[MERGE]

CHANGELOG.md

@@ -2,6 +2,8 @@
 * Guided Tour
 * Sources form highlighting crawlers
 
+## Version 3.0.7
+* Alembic implemented to manage database migrations
 
 ## Version 3.0.6.8
 * REPEC Crawler (connection with https://multivac.iscpif.fr)

alembic.ini

+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# timezone to use when rendering the date
+# within the migration file as well as the filename.
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat alembic/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+# XXX For database access configuration, see alembic/env.py
+#sqlalchemy.url = driver://user:pass@localhost/dbname
+
+[alembic:exclude]
+tables = django_* celery_* djcelery_* auth_*
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

alembic/README

+Alembic must be installed in the virtualenv in order to use right python paths,
+so it's installed with pip. Commands described in this little documentation
+must be executed from gargantext root directory, ie. /srv/gargantext.
+
+Keep in mind that Alembic only handles SQLAlchemy models: tables created from
+Django ORM must be put out of Alembic sight. See [alembic:exclude] section in
+alembic.ini.
+
+
+TELL ALEMBIC TO NOT START FROM SCRATCH
+
+    # To upgrade a database populated before Alembic usage in Gargantext,
+    # don't forget to tell Alembic your current version before to run
+    # "upgrade head" command. If you don't want to do this, you can of course
+    # drop your database and really start from scratch.
+
+    alembic stamp bedce47c9e34
+
+
+UPGRADE TO LATEST DATABASE VERSION
+
+    alembic upgrade head
+
+
+DOWNGRADE TO INITIAL DATABASE STATE
+
+    # /!\ RUNNING THIS COMMAND WILL CAUSE ALL DATA LOST WITHOUT ASKING !!
+
+    alembic downgrade base
+
+
+GENERATE A NEW REVISION
+
+    alembic revision -m "Message for this migration"
+
+    # A migration script is then created in alembic/versions directory. For
+    # example alembic/versions/3adcc9a56557_message_for_this_migration.py
+    # where 3adcc9a56557 is the revision id generated by Alembic.
+    #
+    # This script must be edited to write the migration itself, mainly
+    # in `upgrade` and `downgrade` functions. See Alembic documentation for
+    # further details.
+
+
+GENERATE A REVISION FROM CURRENT STATE
+
+    alembic revision --autogenerate -m "Message for this migration"
+
+    # Alembic should generate a script reflecting changes already made in
+    # database. However it is always a good idea to check it and edit it
+    # manually, Alembic is not always accurate and can't see all alterations.
+    # It should work with basic changes such as model or column creation. See
+    # http://alembic.zzzcomputing.com/en/latest/autogenerate.html#what-does-autogenerate-detect-and-what-does-it-not-detect

alembic/env.py

+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+import re
+
+# Add projet root directory in path and setup Django...
+import os
+import django
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'gargantext.settings')
+django.setup()
+
+# ...to be able to import gargantext.
+
+from gargantext import settings, models
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+config.set_main_option("sqlalchemy.url", settings.DATABASES['default']['URL'])
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = models.Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+# Inspired from https://gist.github.com/utek/6163250
+def exclude_tables_from_config(config):
+    tables = config.get("tables", '').replace('*', '.*').split(' ')
+    pattern = '|'.join(tables)
+    return re.compile(pattern)
+
+exclude_tables = exclude_tables_from_config(config.get_section('alembic:exclude'))
+
+
+def include_object(obj, name, typ, reflected, compare_to):
+    if typ == "table" and exclude_tables.match(name):
+        return False
+    else:
+        return True
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url, target_metadata=target_metadata, literal_binds=True,
+        include_object=include_object)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool)
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            include_object=include_object
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import gargantext
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

alembic/versions/08230100f262_put_a_timezone_on_node_date.py

+"""Put a timezone on Node.date
+
+Revision ID: 08230100f262
+Revises: 601e9d9baa4c
+Create Date: 2017-07-06 13:47:10.788569
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import gargantext
+
+
+# revision identifiers, used by Alembic.
+revision = '08230100f262'
+down_revision = '601e9d9baa4c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.alter_column('nodes', 'date', type_=sa.DateTime(timezone=True))
+
+
+def downgrade():
+    op.alter_column('nodes', 'date', type_=sa.DateTime(timezone=False))

alembic/versions/601e9d9baa4c_add_occ_hist_occ_hist_part_functions.py

+"""Add OCC_HIST & OCC_HIST_PART functions
+
+Revision ID: 601e9d9baa4c
+Revises: 932dbf3e8c43
+Create Date: 2017-07-06 10:52:16.161118
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from gargantext.util.alembic import ReplaceableObject
+
+
+# revision identifiers, used by Alembic.
+revision = '601e9d9baa4c'
+down_revision = '932dbf3e8c43'
+branch_labels = None
+depends_on = None
+
+
+# -- OCC_HIST_PART :: Corpus.id -> GroupList.id -> Start -> End
+
+occ_hist_part = ReplaceableObject(
+    "OCC_HIST_PART(int, int, timestamp, timestamp)",
+    """
+RETURNS TABLE (ng_id int, score float8)
+    AS $$
+-- EXPLAIN ANALYZE
+    SELECT
+    COALESCE(gr.ngram1_id, ng1.ngram_id) as ng_id,
+    SUM(ng1.weight) as score
+
+    from nodes n
+
+    -- BEFORE
+    INNER JOIN nodes as n1 ON n1.id = n.id
+
+    INNER JOIN nodes_ngrams ng1 ON ng1.node_id = n1.id
+
+    -- Limit with timestamps: ]start, end]
+    INNER JOIN nodes_hyperdata nh1 ON nh1.node_id = n1.id
+                                  AND nh1.value_utc >  $3
+                                  AND nh1.value_utc <= $4
+
+    -- Group List
+    LEFT JOIN  nodes_ngrams_ngrams gr ON ng1.ngram_id = gr.ngram2_id
+                               AND gr.node_id = $2
+
+    WHERE
+        n.typename  = 4
+    AND n.parent_id = $1
+    GROUP BY 1
+    $$
+LANGUAGE SQL;
+    """
+)
+
+# -- OCC_HIST :: Corpus.id -> GroupList.id -> MapList.id -> Start -> EndFirst -> EndLast
+# -- EXEMPLE USAGE
+# --    SELECT * FROM OCC_HIST(182856, 183859, 183866, '1800-03-15 17:00:00+01', '2000-03-15 17:00:00+01', '2017-03-15 17:00:00+01')
+
+occ_hist = ReplaceableObject(
+    "OCC_HIST(int, int, int, timestamp, timestamp, timestamp)",
+    """
+RETURNS TABLE (ng_id int, score numeric)
+    AS $$
+    WITH OCC1 as (SELECT * from OCC_HIST_PART($1, $2, $4, $5))
+       , OCC2 as (SELECT * from OCC_HIST_PART($1, $2, $5, $6))
+       , GROWTH as (SELECT ml.ngram_id as ngram_id
+                 , COALESCE(OCC1.score, null) as score1
+                 , COALESCE(OCC2.score, null) as score2
+                    FROM nodes_ngrams ml
+                        LEFT JOIN OCC1 ON OCC1.ng_id = ml.ngram_id
+                        LEFT JOIN OCC2 ON OCC2.ng_id = ml.ngram_id
+                    WHERE ml.node_id = $3
+                    ORDER by score2 DESC)
+    SELECT ngram_id, COALESCE(ROUND(CAST((100 * (score2 - score1) / COALESCE((score2 + score1), 1)) as numeric), 2), 0) from GROWTH
+    $$
+LANGUAGE SQL;
+    """
+)
+
+# -- BEHAVIORAL TEST (should be equal to occ in terms table)
+# --    WITH OCC as (SELECT * from OCC_HIST(182856, 183859, '1800-03-15 17:00:00+01', '2300-03-15 17:00:00+01'))
+# --    SELECT ng_id, score from OCC
+# --            INNER JOIN nodes_ngrams ml on ml.ngram_id = ng_id
+# --                                      AND ml.node_id = 183866
+# --            ORDER BY score DESC;
+
+def upgrade():
+    op.create_sp(occ_hist_part)
+    op.create_sp(occ_hist)
+
+
+def downgrade():
+    op.drop_sp(occ_hist)
+    op.drop_sp(occ_hist_part)

alembic/versions/932dbf3e8c43_initial_migration.py

+"""Initial migration
+
+Revision ID: 932dbf3e8c43
+Revises: 
+Create Date: 2017-07-05 16:41:23.951422
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import gargantext
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '932dbf3e8c43'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('contacts',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('user1_id', sa.Integer(), nullable=True),
+    sa.Column('user2_id', sa.Integer(), nullable=True),
+    sa.Column('is_blocked', sa.Boolean(), nullable=True),
+    sa.Column('date_creation', sa.DateTime(), nullable=True),
+    sa.ForeignKeyConstraint(['user1_id'], ['auth_user.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['user2_id'], ['auth_user.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('user1_id', 'user2_id')
+    )
+    op.create_table('ngrams',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('terms', sa.String(length=255), nullable=True),
+    sa.Column('n', sa.Integer(), nullable=True),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('terms')
+    )
+    op.create_index('ngrams_id_n_idx', 'ngrams', ['id', 'n'], unique=False)
+    op.create_index('ngrams_n_idx', 'ngrams', ['n'], unique=False)
+    op.create_table('nodes',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('typename', gargantext.models.nodes.NodeType(), nullable=True),
+    sa.Column('user_id', sa.Integer(), nullable=True),
+    sa.Column('parent_id', sa.Integer(), nullable=True),
+    sa.Column('name', sa.String(length=255), nullable=True),
+    sa.Column('date', sa.DateTime(), nullable=True),
+    sa.Column('hyperdata', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    sa.ForeignKeyConstraint(['parent_id'], ['nodes.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['user_id'], ['auth_user.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_nodes_typename'), 'nodes', ['typename'], unique=False)
+    op.create_index('nodes_hyperdata_idx', 'nodes', ['hyperdata'], unique=False)
+    op.create_index('nodes_user_id_typename_parent_id_idx', 'nodes', ['user_id', 'typename', 'parent_id'], unique=False)
+    op.create_table('nodes_hyperdata',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('node_id', sa.Integer(), nullable=True),
+    sa.Column('key', gargantext.models.hyperdata.HyperdataKey(), nullable=True),
+    sa.Column('value_int', sa.Integer(), nullable=True),
+    sa.Column('value_flt', postgresql.DOUBLE_PRECISION(), nullable=True),
+    sa.Column('value_utc', sa.DateTime(timezone=True), nullable=True),
+    sa.Column('value_str', sa.String(length=255), nullable=True),
+    sa.Column('value_txt', sa.Text(), nullable=True),
+    sa.ForeignKeyConstraint(['node_id'], ['nodes.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_nodes_hyperdata_value_flt'), 'nodes_hyperdata', ['value_flt'], unique=False)
+    op.create_index(op.f('ix_nodes_hyperdata_value_int'), 'nodes_hyperdata', ['value_int'], unique=False)
+    op.create_index(op.f('ix_nodes_hyperdata_value_str'), 'nodes_hyperdata', ['value_str'], unique=False)
+    op.create_index(op.f('ix_nodes_hyperdata_value_utc'), 'nodes_hyperdata', ['value_utc'], unique=False)
+    op.create_index('nodes_hyperdata_node_id_key_idx', 'nodes_hyperdata', ['node_id', 'key'], unique=False)
+    op.create_index('nodes_hyperdata_node_id_key_value_flt_idx', 'nodes_hyperdata', ['node_id', 'key', 'value_flt'], unique=False)
+    op.create_index('nodes_hyperdata_node_id_key_value_int_idx', 'nodes_hyperdata', ['node_id', 'key', 'value_int'], unique=False)
+    op.create_index('nodes_hyperdata_node_id_key_value_str_idx', 'nodes_hyperdata', ['node_id', 'key', 'value_str'], unique=False)
+    op.create_index('nodes_hyperdata_node_id_key_value_utc_idx', 'nodes_hyperdata', ['node_id', 'key', 'value_utc'], unique=False)
+    op.create_index('nodes_hyperdata_node_id_value_utc_idx', 'nodes_hyperdata', ['node_id', 'value_utc'], unique=False)
+    op.create_table('nodes_ngrams',
+    sa.Column('node_id', sa.Integer(), nullable=False),
+    sa.Column('ngram_id', sa.Integer(), nullable=False),
+    sa.Column('weight', sa.Float(), nullable=True),
+    sa.ForeignKeyConstraint(['ngram_id'], ['ngrams.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['node_id'], ['nodes.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('node_id', 'ngram_id')
+    )
+    op.create_index('nodes_ngrams_ngram_id_idx', 'nodes_ngrams', ['ngram_id'], unique=False)
+    op.create_index('nodes_ngrams_node_id_idx', 'nodes_ngrams', ['node_id'], unique=False)
+    op.create_index('nodes_ngrams_node_id_ngram_id_idx', 'nodes_ngrams', ['node_id', 'ngram_id'], unique=False)
+    op.create_table('nodes_ngrams_ngrams',
+    sa.Column('node_id', sa.Integer(), nullable=False),
+    sa.Column('ngram1_id', sa.Integer(), nullable=False),
+    sa.Column('ngram2_id', sa.Integer(), nullable=False),
+    sa.Column('weight', sa.Float(precision=24), nullable=True),
+    sa.ForeignKeyConstraint(['ngram1_id'], ['ngrams.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['ngram2_id'], ['ngrams.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['node_id'], ['nodes.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('node_id', 'ngram1_id', 'ngram2_id')
+    )
+    op.create_index('nodes_ngrams_ngrams_ngram1_id_idx', 'nodes_ngrams_ngrams', ['ngram1_id'], unique=False)
+    op.create_index('nodes_ngrams_ngrams_ngram2_id_idx', 'nodes_ngrams_ngrams', ['ngram2_id'], unique=False)
+    op.create_index('nodes_ngrams_ngrams_node_id_idx', 'nodes_ngrams_ngrams', ['node_id'], unique=False)
+    op.create_index('nodes_ngrams_ngrams_node_id_ngram1_id_ngram2_id_idx', 'nodes_ngrams_ngrams', ['node_id', 'ngram1_id', 'ngram2_id'], unique=False)
+    op.create_table('nodes_nodes',
+    sa.Column('node1_id', sa.Integer(), nullable=False),
+    sa.Column('node2_id', sa.Integer(), nullable=False),
+    sa.Column('score', sa.Float(precision=24), nullable=True),
+    sa.ForeignKeyConstraint(['node1_id'], ['nodes.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['node2_id'], ['nodes.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('node1_id', 'node2_id')
+    )
+    op.create_index('nodes_nodes_node1_id_node2_id_idx', 'nodes_nodes', ['node1_id', 'node2_id'], unique=False)
+    op.create_table('nodes_nodes_ngrams',
+    sa.Column('node1_id', sa.Integer(), nullable=False),
+    sa.Column('node2_id', sa.Integer(), nullable=False),
+    sa.Column('ngram_id', sa.Integer(), nullable=False),
+    sa.Column('score', sa.Float(precision=24), nullable=True),
+    sa.ForeignKeyConstraint(['ngram_id'], ['ngrams.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['node1_id'], ['nodes.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['node2_id'], ['nodes.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('node1_id', 'node2_id', 'ngram_id')
+    )
+    op.create_index('nodes_nodes_ngrams_node1_id_idx', 'nodes_nodes_ngrams', ['node1_id'], unique=False)
+    op.create_index('nodes_nodes_ngrams_node2_id_idx', 'nodes_nodes_ngrams', ['node2_id'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('nodes_nodes_ngrams_node2_id_idx', table_name='nodes_nodes_ngrams')
+    op.drop_index('nodes_nodes_ngrams_node1_id_idx', table_name='nodes_nodes_ngrams')
+    op.drop_table('nodes_nodes_ngrams')
+    op.drop_index('nodes_nodes_node1_id_node2_id_idx', table_name='nodes_nodes')
+    op.drop_table('nodes_nodes')
+    op.drop_index('nodes_ngrams_ngrams_node_id_ngram1_id_ngram2_id_idx', table_name='nodes_ngrams_ngrams')
+    op.drop_index('nodes_ngrams_ngrams_node_id_idx', table_name='nodes_ngrams_ngrams')
+    op.drop_index('nodes_ngrams_ngrams_ngram2_id_idx', table_name='nodes_ngrams_ngrams')
+    op.drop_index('nodes_ngrams_ngrams_ngram1_id_idx', table_name='nodes_ngrams_ngrams')
+    op.drop_table('nodes_ngrams_ngrams')
+    op.drop_index('nodes_ngrams_node_id_ngram_id_idx', table_name='nodes_ngrams')
+    op.drop_index('nodes_ngrams_node_id_idx', table_name='nodes_ngrams')
+    op.drop_index('nodes_ngrams_ngram_id_idx', table_name='nodes_ngrams')
+    op.drop_table('nodes_ngrams')
+    op.drop_index('nodes_hyperdata_node_id_value_utc_idx', table_name='nodes_hyperdata')
+    op.drop_index('nodes_hyperdata_node_id_key_value_utc_idx', table_name='nodes_hyperdata')
+    op.drop_index('nodes_hyperdata_node_id_key_value_str_idx', table_name='nodes_hyperdata')
+    op.drop_index('nodes_hyperdata_node_id_key_value_int_idx', table_name='nodes_hyperdata')
+    op.drop_index('nodes_hyperdata_node_id_key_value_flt_idx', table_name='nodes_hyperdata')
+    op.drop_index('nodes_hyperdata_node_id_key_idx', table_name='nodes_hyperdata')
+    op.drop_index(op.f('ix_nodes_hyperdata_value_utc'), table_name='nodes_hyperdata')
+    op.drop_index(op.f('ix_nodes_hyperdata_value_str'), table_name='nodes_hyperdata')
+    op.drop_index(op.f('ix_nodes_hyperdata_value_int'), table_name='nodes_hyperdata')
+    op.drop_index(op.f('ix_nodes_hyperdata_value_flt'), table_name='nodes_hyperdata')
+    op.drop_table('nodes_hyperdata')
+    op.drop_index('nodes_user_id_typename_parent_id_idx', table_name='nodes')
+    op.drop_index('nodes_hyperdata_idx', table_name='nodes')
+    op.drop_index(op.f('ix_nodes_typename'), table_name='nodes')
+    op.drop_table('nodes')
+    op.drop_index('ngrams_n_idx', table_name='ngrams')
+    op.drop_index('ngrams_id_n_idx', table_name='ngrams')
+    op.drop_table('ngrams')
+    op.drop_table('contacts')
+    # ### end Alembic commands ###

alembic/versions/bedce47c9e34_fix_issue_with_node_hyperdata_index.py

+"""Fix issue with Node.hyperdata index
+
+Revision ID: bedce47c9e34
+Revises: 08230100f262
+Create Date: 2017-07-10 11:30:59.168190
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import gargantext
+
+
+# revision identifiers, used by Alembic.
+revision = 'bedce47c9e34'
+down_revision = '08230100f262'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_index('nodes_hyperdata_idx', table_name='nodes')
+    op.create_index('nodes_hyperdata_idx', 'nodes', ['hyperdata'], unique=False, postgresql_using="gin")
+
+
+def downgrade():
+    # We won't unfix the bug when downgrading...
+    pass

dbmigrate.py

-#!/usr/bin/env python
-import sys
-import os
-
-if __name__ == "__main__":
-
-    # Django settings
-    dirname = os.path.dirname(os.path.realpath(__file__))
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "gargantext.settings")
-
-    # initialize Django application
-    from django.core.wsgi import get_wsgi_application
-    application = get_wsgi_application()
-
-    # retrieve Django models
-    import django.apps
-    django_models = django.apps.apps.get_models()
-    django_models_names = set(model._meta.db_table for model in django_models)
-
-    # migrate SQLAlchemy models
-    from gargantext import models
-    from gargantext.util.db import Base, engine
-    sqla_models_names = (
-        model for model in Base.metadata.tables.keys()
-        if model not in django_models_names
-    )
-    sqla_models = (
-        Base.metadata.tables[model_name]
-        for model_name in sqla_models_names
-    )
-    print()
-    for model in sqla_models:
-        try:
-            model.create(engine)
-            print('created model: `%s`' % model)
-        except Exception as e:
-            print('could not create model: `%s`, %s' % (model, e))
-        print()

docs/architecture.md

+
+# Definitions and notation for the documentation (!= python notation)
+
+## Node
+
+The table (nodes) is a list of nodes: `[Node]`
+
+Each Node has:
+
+- a typename
+- a parent_id
+- a name
+
+
+### Each Node has a parent_id
+
+    Node A
+    ├── Node B
+    └── Node C
+
+If Node A is Parent of Node B and Node C
+then NodeA.id == NodeB.parent_id == NodeC.parent_id.
+
+### Each Node has a typename
+
+Notation: `Node["FOO"]("bar")` is a Node of typename "FOO" and with name "bar".
+
+Then:
+
+- Then Node[PROJECT] is a project.
+- Then Node[CORPUS] is a corpus.
+- Then Node[DOCUMENT] is a document.
+
+The syntax of the Node here do not follow exactly Python documentation
+(for clarity and to begin with): in Python code, typenames are strings
+represented as UPPERCASE strings (eg. "PROJECT").
+
+### Each Node as a typename and a parent
+
+    Node[USER](name)
+    ├── Node[PROJECT](myProject1)
+    │   ├── Node[CORPUS](myCorpus1)
+    │   ├── Node[CORPUS](myCorpus2)
+    │   └── Node[CORPUS](myCorpus3)
+    └── Node[PROJECT](myProject2)
+
+/!\\ 3 ways to manage rights of the Node:
+
+1. Then Node[User] is a folder containing all User projects and corpus and
+   documents (i.e. Node[user] is the parent_id of the children).
+2. Each node as a user_id (mainly used today)
+3. Right management for the groups (implemented already but not
+   used since not connected to the frontend).
+
+
+## Global Parameters
+
+Global User is Gargantua (Node with typename user).
+This node is the parent of the other nodes for parameters.
+
+    Node[USER](gargantua) (gargantua.id == Node[USER].user_id)
+    ├── Node[TFIDF-Global](global) : without group
+    │   ├── Node[TFIDF](database1)
+    │   ├── Node[TFIDF](database2)
+    │   └── Node[TFIDF](database3)
+    └── Node[ANOTHERMETRIC](global)
+
+
+[//]: # (Are there any plans to add user wide or project wide parameters or metrics?  For example TFIDF nodes related to a normal user -- ie. not Gargantua?)
+
+Yes we can in the future (but we have others priorities before.
+
+[//]: # (What is the purpose of the 3 child nodes of Node[TFIDF-Global]?  Are they TFIDF metrics related to databases 1, 2 and 3? If so, shouldn't they be children of related CORPUS nodes?)
+
+Node placement in the tree indicates the context of the metric: the
+Metrics Node has parent the corpus Node to indicate the context of the
+metrics.
+
+Answer:
+
+    Node[USER](foo)
+    Node[USER](bar)
+    ├── Node[PROJECT](project1)
+    │   ├── Node[CORPUS](corpus1)
+    │   │   ├── Node[DOCUMENT](doc1)
+    │   │   ├── Node[DOCUMENT](doc2)
+    │   │   └── Node[TFIDF-global](name of the metrics)
+    │   ├── Node[CORPUS](corpus2)
+    │   └── Node[CORPUS](corpus3)
+    └── Node[PROJECT](project2)
+
+
+
+## NodeNgram
+
+NodeNgram is a relation of a Node with a ngram:
+
+- documents and ngrams
+- metrics  and ngrams (position of the node metrics indicates the
+  context)
+
+
+
+# Community Parameters
+
+
+# User Parameters
+
+
+

docs/contribution.md

@@ -2,7 +2,7 @@
 
 ## Community
 * [http://gargantext.org/about](http://gargantext.org/about)
-* IRC Chat: (OFTC/FreeNode) #gargantex
+* IRC Chat: (OFTC/FreeNode) #gargantext
 
 ##Tools
 * gogs

docs/index.md

@@ -8,6 +8,9 @@ Gargantext is a web plateform to explore your corpora using text-mining[...](abo
 
 * [Take a tour](demo.md) of the different features offered by Gargantext
 
+## Architecture
+* [Architecture](architecture.md) Architecture of Gargantext
+
 ##Need some help?
 
 Ask the community at:

docs/manual_install.md

@@ -49,11 +49,23 @@ by cloning the repository of gargantext
 ``` bash
 git clone ssh://gitolite@delanoe.org:1979/gargantext /srv/gargantext \
         && cd /srv/gargantext \
-        && git fetch origin refactoring \
-        && git checkout refactoring \
+        && git fetch origin stable \
+        && git checkout stable \
 ```
 
     TODO(soon): git clone https://gogs.iscpif.fr/gargantext.git
 
 
+* Install and configure the virtual environment
+``` bash
+cd /srv/
+pip3 install virtualenv
+virtualenv /srv/env_3-5 -p /usr/bin/python3.5
+pip install -r /srv/gargantext/install
+echo '/srv/gargantext' > /srv/env_3-5/lib/python3.5/site-packages/gargantext.pth
+echo 'alias venv="source /srv/env_3-5/bin/activate"' >> ~/.bashrc
+```
+
+
 See the [next steps of installation procedure](install.md#Install)
+See the [next manual steps of installation procedure](Debian.sh)

docs/ngram_lists.md

+# Gargantext foundations 
+
+Collaborative platform for multi-scale text experiments
+Embrace the past, update the present, forecast the future.
+
+# Main Types of Entity definitions
+
+Documentation valid for 3.0.\* versions of Gargantext.
+
+## Nature of the entities
+
+In Object programming language, it is objects.
+In purely functional language, it is types.
+
+
+## Project
+A project is a list of corpora (a project may have duplicate corpora).
+
+## Corpus
+A corpus is a set of documents: duplicate documents are authorized but
+not recommended for the methodology since it shows artificial repeated content in the corpus. 
+
+Then, in the document view, users may delete duplicates with a specific
+function.
+
+## Document
+A document is the main Entity of Textual Context (ETC) that is composed with:
+    - a title (truncated field name in the database)
+    - the date of publication
+    - a journal (or source)
+    - an abstract
+    - the authors
+Users may add many fields to the document.
+
+The main fields mentioned above are used for the main statistics in Gargantext.
+
+
+### Source Type
+Source Type is the source (database) from where documents have been
+extracted. 
+
+In 3.0.\* versions of Gargantext, each corpus has only one source type
+(i.e database). But user can build his own corpus with CVS format.
+
+
+## Ngrams
+
+### Definitions
+
+### Gram 
+A gram is a contiguous sequence of letters separated by spaces.
+
+### N-gram
+N-gram is a contiguous sequence of n grams separated by spaces (where n
+is a non negative natural number).
+
+
+## N-gram Lists
+
+
+## Main ngrams lists: Stop/Map/Main
+
+Receipe of Gargantext consist of offering the rights ngrams for the map.
+A the better level of complexity in order to unveil its richness
+according to this 2 main rules:
+
+If ngrams are too specifics, then the graph becomes too sparse.
+If ngrams are too generics, then the graph becomes too connected.
+
+As a consequence, finding the right balance of specific and generic
+ngrams is the main target.
+
+In first versions of Gargantext, this balance is solved with linear
+methods. After 3.1.\*, non linear methods trained on dataset of the
+users enable the system to find a better balance at any scale.
+
+
+### Definition
+
+3 main kinds of lists :
+    1. Stop List contains black listed ngrams i.e. the noise or in others words ngrams users do not want to deal with.
+    2. Map List contains ngrams that will be shown in the map.
+    3. Main list or Candidate list contains all other ngrams that are neither in the stop list or in the map list. Then it _could_ be in the map according to the choice of the user or, by default, the default parameters of Gargantext.
+
+### Storage
+
+Relation between the list and the ngram is stored as Node-Ngram
+relation where
+    - Node has type name (STOP|MAIN|MAP) and parent_id the context
+      (CORPUS in version 3.0.*; but could be PROJECT)
+    - Ngrams depend on the context of the Node List where NodeNgrams is
+      not null and Node has typename Document.
+
+
+    Node[USER](name1)
+    ├── Node[PROJECT](project1)
+    │   ├── Node[CORPUS](corpus1)
+    │   │   ├── Node[MAPLIST](list name)
+    │   │   ├── Node[STOPLIST](list name)
+    │   │   ├── Node[MAINLIST](list name)
+    │   │   │  
+    │   │   ├── Node[DOCUMENT](doc1)
+    │   │   ├── Node[DOCUMENT](doc2)
+    │   │   └── Node[DOCUMENT](doc2)
+
+
+### Policy
+
+#### Algo
+
+Let be a set of ngrams where NodeNgram != 0 then
+    find 2 subsets of these ngrams that show a split 
+        - stop ngrams
+        - not stop ngrams
+    then for the subset "not stop ngrams"
+        find 2 subset of ngrams that show a split:
+            - map ngrams
+            - others ngrams
+
+#### Techno algo
+
+A classifier (Support Machine Vector) is used on the following scaled-measures
+for each step:
+    - n (of the "n" gram)
+    - Occurrences : Zip Law (in fact already used in TFICF, this
+      features are correletad, put here for pedagogical purpose)
+    - TFICF-CORPUS-SOURCETYPE
+    - TFICF-SOURCETYPE-ALL
+    - Genericity score
+    - Specificty score
+
+
+
+
+## Metrics
+
+### Term Frequency - Inverse Context Frequency (TF-ICF)
+
+TFICF, short for term frequency-inverse context frequency, is a numerical
+statistic that is intended to reflect how important an ngram is to a
+context of text.
+
+TFICF(ngram,contextLocal,contextGlobal) = TF(ngram,contextLocal) \* ICF(ngram, contextGlobal)
+where
+ * TF(ngram, contextLocal) is the ngram frequency (occurrences) in contextLocal.
+ * ICF(ngram, contextGlobal) is the inverse (log) document frequency (occurrences) in contextGlobal.
+
+
+Others types of TFICF:
+    - TFICF(ngram, DOCUMENT, CORPUS)
+    - TFICF(ngram, CORPUS, PROJECT)
+    - TFICF(ngram, PROJECT, DATABASETYPE)
+    - TFICF(ngram, DATABASETYPE, ALL)
+
+
+If the context is a document in a set of documents (corpus), then it is a TFIDF as usual. 
+Then TFICF-DOCUMENT-CORPUS == TFICF(ngram,DOCUMENT,CORPUS) = TFIDF.
+TFICF is the generalization of [TFIDF, Term Frequency - Inverse Document Frequency](https://en.wikipedia.org/wiki/Tf%E2%80%93idf).
+
+#### Implementation
+
+TFICF = TF * log (ICF)
+
+To prepare the groups, we need to store TF and ICF seperately (in
+NodesNogram via 2 nodes).
+
+Let be TF and ICF typename of Nodes.
+
+
+    Node[USER](gargantua)
+    ├── Node[OCCURRENCES](source)
+    ├── Node[TF](all sourcetype)
+    ├── Node[ICF](all sourcetype)
+    ├── Node[SOURCETYPE](Pubmed)
+    │   ├── Node[OCCURRENCES](all corpora)
+    │   ├── Node[TF](all corpora)
+    │   └── Node[ICF](all corpora)
+    ├── Node[SOURCETYPE](WOS)
+
+
+
+
+## others ngrams lists
+
+### Group List
+
+#### Definition
+
+Group list gives a quantifiable link between two ngrams.
+
+
+#### Policy to build group lists
+
+To group the ngrams:
+- stemming or lemming
+- c-value
+- clustering (see graphs)
+- manually by the user (supervised learning)
+
+The scale is the character.
+
+
+#### Storage
+
+In the table NodeNgramNgram where Node has type name Group for ngram1
+and ngram2.
+
+
+### Favorite List
+
+#### Definition
+
+Fovorite Nodes
+The scale is the node.
+
+#### Building policy
+
+- manually by the user (supervised learning)
+
+#### Storage
+
+NodeNode relation where first Node has type Favorite.
+
+
+
+
+
+
+

docs/tools/about.md

-
-

docs/tools/about/index.md

-#Gargantext
-
-Welcome to Garagentext documentation!

docs/tools/api_urls.md

-List of garg's own JSON API(s) urls
-===================================
-
-2016-05-27
-
-### /api/nodes/2
-
-```
-{
-  "id": 2,
-  "parent_id": 1,
-  "name": "abstract:\"evaporation+loss\"",
-  "typename": "CORPUS"
-}
-```
-------------------------------
-
-### /api/nodes?pagination_limit=-1
-```
-{
-  "records": [
-    {
-      "id": 9,
-      "parent_id": 2,
-      "name": "A recording evaporimeter",
-      "typename": "DOCUMENT"
-    },
-    (...)
-    {
-      "id": 119,
-      "parent_id": 81,
-      "name": "GRAPH EXPLORER COOC (in:81)",
-      "typename": "COOCCURRENCES"
-    }
-  ],
-  "count": 119,
-  "parameters": {
-      "formated": "json","pagination_limit": -1,
-      "fields": ["id","parent_id","name","typename"],
-      "pagination_offset": 0
-  }
-}
-```
-------------------------------
-
-### /api/nodes?types[]=CORPUS
-```
-{
-  "records": [
-    {
-      "id": 2,
-      "parent_id": 1,
-      "name": "abstract:\"evaporation+loss\"",
-      "typename": "CORPUS"
-    },
-    (...)
-    {
-      "id": 8181,
-      "parent_id": 1,
-      "name": "abstract:(astrogeology+OR ((space OR spatial) AND planetary) AND geology)",
-      "typename": "CORPUS"
-    }
-  ],
-  "count": 2,
-  "parameters": {
-        "pagination_limit": 10,
-        "types": ["CORPUS"],
-        "formated": "json",
-        "pagination_offset": 0,
-        "fields": ["id","parent_id","name","typename"]
-  }
-}
-```
-------------------------------
-### /api/nodes/5?fields[]=ngrams
-
-Où <5> représente un doc_id ou list_id
-
-```
-{
-  "ngrams": [
-    [1.0,{"id":2299,"n":1,"terms":designs}],
-    [1.0,{"id":1917,"n":1,"terms":height}],
-    [1.0,{"id":1755,"n":2,"terms":higher speeds}],
-    [1.0,{"id":1940,"n":1,"terms":cylinders}],
-    [1.0,{"id":2221,"n":3,"terms":other synthesized materials}],
-    (...)
-    [2.0,{"id":1970,"n":1,"terms":storms}],
-    [9.0,{"id":1754,"n":2,"terms":spherical gauges}],
-    [1.0,{"id":1895,"n":1,"terms":direction}],
-    [1.0,{"id":2032,"n":1,"terms":testing}],
-    [1.0,{"id":1981,"n":2,"terms":"wind effects"}]
-  ]
-}
-```
-------------------------------
-
-### api/nodes/3?fields[]=id&fields[]=hyperdata&fields[]=typename
-```
-{
-  "id": 3,
-  "typename": "DOCUMENT",
-  "hyperdata": {
-    "language_name": "English",
-    "language_iso3": "eng",
-    "language_iso2": "en",
-    "title": "A blabla analysis of laser treated aluminium blablabla",
-    "name": "A blabla analysis of laser treated aluminium blablabla",
-    "authors": "A K. Jain, V.N. Kulkarni, D.K. Sood"
-    "authorsRAW": [
-    {"name": "....", "affiliations": ["... Research Centre,.. 085, Country"]},
-    {"name": "....", "affiliations": ["... Research Centre,.. 086, Country"]}
-    (...)
-    ],
-    "abstract": "Laser processing of materials, being a rapid melt quenching process, quite often produces a surface which is far from being ideally smooth for ion beam analysis. (...)",
-    "genre": ["research-article"],
-    "doi": "10.1016/0029-554X(81)90998-8",
-    "journal": "Nuclear Instruments and Methods In Physics Research",
-    "publication_year": "1981",
-    "publication_date": "1981-01-01 00:00:00",
-    "publication_month": "01",
-    "publication_day": "01",
-    "publication_hour": "00",
-    "publication_minute": "00",
-    "publication_second": "00",
-    "id": "61076EB1178A97939B1C893904C77FB7DA2276D0",
-    "source": "elsevier",
-    "distributor": "istex"
-  }
-}
-```
-
-## TODO continuer la liste

docs/tools/automatic_install.md

-#

docs/tools/contribute/ngrams/ngram_parsing_flow.dot

-// dot ngram_parsing_flow.dot -Tpng -o ngram_parsing_flow.png
-digraph ngramflow {
-
-    edge [fontsize=10] ;
-    label=<<B><U>gargantext.util.toolchain</U></B><BR/>(ngram extraction flow)>;
-    labelloc="t" ;
-
-    "extracted_ngrams" -> "grouplist" ;
-    "extracted_ngrams" -> "occs+ti_rank" ;
-    "project stoplist (todo)" -> "stoplist" ;
-    "stoplist" -> "mainlist" ;
-    "occs+ti_rank" -> "mainlist" [label="  TI_RANK_LIMIT"];
-    "mainlist" -> "coocs" [label="  COOCS_THRESHOLD"] ;
-    "coocs" -> "specificity" ;
-
-    "specificity" -> "maplist" [label="MAPLIST_LIMIT\nMONOGRAM_PART"];
-
-    "mainlist" -> "tfidf" ;
-    "tfidf" -> "explore" [label="doc relations with all map and candidates"];
-    "maplist" -> "explore" ;
-    "grouplist" -> "occs+ti_rank" ;
-    "grouplist" -> "coocs" ;
-    "grouplist" -> "tfidf" ;
-}

docs/tools/contribution-guide.md

-#Contribution guide
-
-## Community
-* [http://gargantext.org/about](http://gargantext.org/about)
-* IRC Chat: (OFTC/FreeNode) #gargantex
-
-##Tools
-* gogs
-* server access
-* forge
-* gargantext box
-
-##Gargantex
-* Gargantex box install
-
-(S.I.R.= Setup Install & Run procedures)
-
-* Architecture Overview
-
-
-* Database Schema Overview
-
-* Interface design Overview
-
-##To do:
-* Docs
-* Interface deisgn
-* Parsers/scrapers
-* Computing
-
-## How to contribute:
-    1. Clone the repo
-    2. Create a new branch <username>-refactoring
-    3. Run the gargantext-box
-    4. Code
-    5.Test
-    6. Commit
-
-### Exemple1: Adding a parser
-* create your new file cern.py into gargantex/scrapers/
-
-* reference into gargantex/scrapers/urls.py
-add this line:
-import scrapers.cern  as cern
-
-* reference into gargantext/constants
-```
-# type 9
-    {   'name': 'Cern',
-        'parser': CernParser,
-        'default_language': 'en',
-    },
-```
-* add an APIKEY in gargantex/settings
-
-
-### Exemple2: User Interface Design

docs/tools/contribution-guide/contribution.md

-#Contribution guide
-
-* A question or a problem? Ask the community
-* Sources
-* Tools
-* Contribution workflow: for contributions, bugs and features
-* Some examples of contributions
-
-## Community
-Need help? Ask the community
-* [http://gargantext.org/about](http://gargantext.org/about)
-* IRC Chat: (OFTC/FreeNode) #gargantex
-
-## Source
-Source are available throught XXX LICENSE
-
-You can install Gargantext throught the [installation procedure](./install.md)
-
-##Tools
-* gogs
-* forge.iscpif.fr
-* server access
-* gargantext box
-
-
-## Contributing: workflow procedure
-Once you have installed and tested Gargantext
-You
-
-1. Clone the stable release into your project
-    Note: The current stable release <release_branch> is:  refactoring
-
-Inside the repo, clone the reference branch and get the last changes:
-git checkout <ref_branch>
-git pull
-
-It is highly recommended to create a generic branch on a stable release such as
-git checkout -b <username>-<release_branch>
-git pull
-
-
-2. Create your project on stable release
-
-git checkout -b <username>-<release_branch>-<project_name>
-
-Do your modifications and commits as you want it:
-git commit -m "foo/bar/1"
-git commit -m "foo/bar/2"
-git push
-
-
-If you want to save your local change you can merge it into your generic branch <username>-<release_branch>
-git checkout <username>-<release_branch>
-git pull
-git merge <username>-<release_branch>-<project_name>
-git commit -m "[Merge OK] comment"
-
-
-##Technical Overview
-
-* Interface Overview
-
-* Database Schema Overview
-
-* Architecture Overview
-
-
-### Exemple1: Adding a parser
-
-### Exemple2: User Interface Design

docs/tools/contribution-guide/ngrams_lifecycle.md

-Cycle de vie des décomptes ngrammes
------------------------------------
-
-### (schéma actuel et pistes) ###
-
-Dans ce qui crée les décomptes, on peut distinguer deux niveaux ou étapes:
-
-1.  l'extraction initiale et le stockage du poids de la relation ngramme
-    document (appelons ces nodes "1doc")
-
-2.  tout le reste: la préparation des décomptes agrégés pour la table
-    termes ("stats"), et pour les tables de travail des graphes et de la
-    recherche de publications.
-
-On pourrait peut-être parler d'indexation par docs pour le niveau 1 et de "modélisations" pour le niveau 2.
-
-On peut remarquer que le niveau 1 concerne des **formes** ou ngrammes seuls (la forme observée <=> chaine de caractères u-nique après normalisation) tandis que dans le niveau 2 on a des objets plus riches... Au fur et à mesure des traitements on a finalement toujours des ngrammes mais:
-
-  - filtrés (on ne calcule pas tout sur tout)
-
-  - typés avec les listes map, stop, main (et peut-être bientôt des
-    "ownlistes" utilisateur)...
-
-  - groupés (ce qu'on voit avec le `+` de la table terme, et qu'on
-    pourrait peut-être faire apparaître aussi côté graphe?)
-
-On peut dire qu'on manipule plutôt des **termes** au niveau 2 et non plus des **formes**... ils sont toujours des ngrammes mais enrichis par l'inclusion dans une série de mini modèles (agrégations et typologie de ngrammes guidée par les usages).
-
-### Tables en BDD
-
-Si on adopte cette distinction entre formes et termes, ça permet de clarifier à quel moment on doit mettre à jour ce qu'on a dans les tables. Côté structure de données, les décomptes sont toujours stockés via des n-uplets qu'on peut du coup résumer comme cela:
-
--   **1doc**: (doc:node - forme:ngr - poids:float) dans des tables
-    NodeNgram
-
--   **occs/gen/spec/tirank**: (type_mesure:node - terme:ngr -
-    poids:float) dans des tables NodeNgram
-
--   **cooc**: (type_graphe:node - terme1:ngr - terme2:ngr -
-    poids:float) dans des tables NodeNgramNgram
-
--   **tfidf**: (type_lienspublis:node - doc:node - terme:ngr -
-    correlation:float) dans des tables NodeNodeNgram.
-
-Où "type" est le node portant la nature de la stat obtenue, ou bien la
-ref du graphe pour cooc et de l'index lié à la recherche de publis pour
-le tfidf.
-
-Il y a aussi les relations qui ne contiennent pas de décomptes mais sont
-essentielles pour former les décomptes des autres:
-
--   map/main/stopliste: (type_liste:node - forme ou terme:ngr) dans des
-    tables NodeNgram
-
--   "groupes": (mainform:ngr - subform:ngr) dans des tables
-    NodeNgramNgram.
-
-### Scénarios d'actualisation
-
-Alors, dans le déroulé des "scénarios utilisateurs", il y plusieurs
-évenements qui viennent **modifier ces décomptes**:
-
-1.  les créations de termes opérés par l'utilisateur (ex: par
-    sélection/ajout dans la vue annotation)
-
-2.  les imports de termes correspondant à des formes jamais indexées sur
-    ce corpus
-
-3.  les dégroupements de termes opérés par l'utilisateur
-
-4.  le passage d'un terme de la stopliste aux autres listes
-
-5.  tout autre changement de listes et/ou création de nouveaux
-    groupes...
-
-A et B sont les deux seules étapes hormis l'extraction initiale où des
-formes sont rajoutées. Actuellement A et B sont gérés tout de suite pour
-le niveau 1 (tables par doc) : il me semble qu'il est bon d'opérer la
-ré-indexation des 1doc le plus tôt possible après A ou B. Pour la vue
-annotations, l'utilisateur s'attend à voir apparaître le surlignage
-immédiatement sur le doc visualisé. Pour l'import B, c'est pratique car
-on a la liste des nouveaux termes sous la main, ça évite de la stocker
-quelque part en attendant un recalcul ultérieur.
-
-L'autre info mise à jour tout de suite pour A et B est l'appartenance
-aux listes et aux groupes (pour B), qui ne demandent aucun calcul.
-
-C, D et E n'affectent pas le niveau 1 (tables par docs) car ils ne
-rajoutent pas de formes nouvelles, mais constituent des modifications
-sur les listes et les groupes, et devront donc provoquer une
-modification du tfidf (pour cela on doit passer par un re-calcul) et des
-coocs sur map (effet appliqué à la demande d'un nouveau graphe).
-
-C et D demandent aussi une mise à jour des stats par termes
-(occurrences, gen/spec etc) puisque les éléments subforms et les
-éléments de la stopliste ne figurent pas dans les stats.
-
-Donc pour résumer on a dans tous les cas:
-
-=> l'ajout à une liste, à un groupe et tout éventuel décompte de
-nouvelle forme dans les docs sont gérés dès l'action utilisateur
-
-=> mais les modélisations plus "avancées" représentées par les les
-stats occs, gen, spec et les tables de travail "coocs sur map" et
-"tfidf" doivent attendre un recalcul.
-
-Idéalement à l'avenir il seraient tous mis à jour incrémentalement au
-lieu de forcer ce recalcul... mais pour l'instant on en est là.
-
-### Fonctions associées
-
-|       |                          GUI                          |                                       API action → url                                        |                      VIEW                       |                                                            SUBROUTINES                                                             |
-|-------|-------------------------------------------------------|-----------------------------------------------------------------------------------------------|-------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------|
-| A     | "annotations/highlight.js, annotations/ngramlists.js" | "PUT → api/ngrams, PUT/DEL → api/ngramlists/change"                                           | "ApiNgrams, ListChange"                         | util.toolchain.ngrams_addition.index_new_ngrams                                                                                    |
-| B     | NGrams_dyna_chart_and_table                           | POST/PATCH → api/ngramlists/import                                                            | CSVLists                                        | "util.ngramlists_tools.import_ngramlists, util.ngramlists_tools.merge_ngramlists, util.toolchain.ngrams_addition.index_new_ngrams" |
-| C,D,E | NGrams_dyna_chart_and_table                           | "PUT/DEL → api/ngramlists/change,  PUT/DEL → api/ngramlists/groups" "ListChange, GroupChange" | util.toolchain.ngrams_addition.index_new_ngrams |                                                                                                                                    |
-
-L'import B a été remis en route il y a quelques semaines, et je viens de
-reconnecter A dans la vue annotations.

docs/tools/contribution.md

-#Contribution guide
-
-## Community
-* [http://gargantext.org/about](http://gargantext.org/about)
-* IRC Chat: (OFTC/FreeNode) #gargantex
-
-##Tools
-* gogs
-* server access
-* gargantext box
-
-##Gargantex
-* Gargantex box install
-see [install procedure](install.md)
-
-* Architecture Overview
-* Database Schema Overview
-* Interface design Overview
-
-##To do:
-* Docs
-* Interface design
-* [Parsers](./overview/parser.md) / scrappers(./overview/scraper.md)
-* Computing
-
-## How to contribute:
-    1. Clone the repo
-    2. Create a new branch <username>-refactoring
-    3. Run the gargantext-box
-    4. Code
-    5. Test
-    6. Commit

docs/tools/corpus_example/pubmed.md5

-94eb7bdf57557b72dcd1b93a42af044b  pubmed.zip

docs/tools/demo/todo.md

-# API
-
-Be more careful about authorizations.
-
-cf. "ng-resource".
-
-# Projects
-
-## Overview of all projects
-
-- re-implement deletion
-
-## Single project view
-
-- re-implement deletion
-
-
-# Taggers
-
-Path for data used by taggers should be defined in `gargantext.constants`.
-
-
-# Database
-
-# Sharing
-
-Here follows a brief description of how sharing could be implemented.
-
-## Database representation
-
-The database representation of sharing can be distributed among 4 tables:
- - `persons`, of which items represent either a user or a group
- - `relationships` describes the relationships between persons (affiliation
-     of a user to a group, contact between two users, etc.)
- - `nodes` contains the projects, corpora, documents, etc. to share (they shall
-     inherit the sharing properties from their parents)
- - `permissions` stores the relations existing between the three previously
-     described above: it only consists of 2 foreign keys, plus an integer
-     between 1 and 3 representing the level of sharing and the start date
-     (when the sharing has been set) and the end date (when necessary, the time
-     at which sharing has been removed, `NULL` otherwise)
-
-## Python code
-
-The permission levels should be set in `gargantext.constants`, and defined as:
-```python
-PERMISSION_NONE = 0     # 0b0000
-PERMISSION_READ = 1     # 0b0001
-PERMISSION_WRITE = 3    # 0b0011
-PERMISSION_OWNER = 7    # 0b0111
-```
-
-The requests to check for permissions (or add new ones) should not be rewritten
-every time. They should be "hidden" within the models:
- - `Person.owns(node)` returns a boolean
- - `Person.can_read(node)` returns a boolean
- - `Person.can_write(node)` returns a boolean
- - `Person.give_right(node, permission)` gives a right to a given user
- - `Person.remove_right(node, permission)` removes a right from a given user
- - `Person.get_nodes(permission[, type])` returns an iterator on the list of
-    nodes on which the person has at least the given permission (optional
-    argument: type of requested node)
-- `Node.get_persons(permission[, type])` returns an iterator on the list of
-   users who have at least the given permission on the node (optional argument:
-   type of requested persons, such as `USER` or `GROUP`)
-
-## Example
-
-Let's imagine the `persons` table contains the following data:
-
-| id | type  | username  |
-|----|-------|-----------|
-| 1  | USER  | David     |
-| 2  | GROUP | C.N.R.S.  |
-| 3  | USER  | Alexandre |
-| 4  | USER  | Untel     |
-| 5  | GROUP | I.S.C.    |
-| 6  | USER  | Bidule    |
-
-Assume "David" owns the groups "C.N.R.S." and "I.S.C.", "Alexandre" belongs to
-the group "I.S.C.", with "Untel" and "Bidule" belonging to the group "C.N.R.S.".
-"Alexandre" and "David" are in contact.
-The `relationships` table then contains:
-
-| person1_id | person2_id | type    |
-|------------|------------|---------|
-| 1          | 2          | OWNER   |
-| 1          | 5          | OWNER   |
-| 3          | 2          | MEMBER  |
-| 4          | 5          | MEMBER  |
-| 6          | 5          | MEMBER  |
-| 1          | 3          | CONTACT |
-
-The `nodes` table is populated as such:
-
-| id | type     | name                 |
-|----|----------|----------------------|
-| 12 | PROJECT  | My super project     |
-| 13 | CORPUS   | A given corpus       |
-| 13 | CORPUS   | The corpus           |
-| 14 | DOCUMENT | Some document        |
-| 15 | DOCUMENT | Another document     |
-| 16 | DOCUMENT | Yet another document |
-| 17 | DOCUMENT | Last document        |
-| 18 | PROJECT  | Another project      |
-| 19 | PROJECT  | That project         |
-
-If we want to express that "David" created "My super project" (and its children)
-and wants everyone in "C.N.R.S." to be able to view it, but not access it,
-`permissions` should contain:
-
-| person_id | node_id | permission |
-|-----------|---------|------------|
-| 1         | 12      | OWNER      |
-| 2         | 12      | READ       |
-
-If "David" also wanted "Alexandre" (and no one else) to view and modify "The
-corpus" (and its children), we would have:
-
-| person_id | node_id | permission |
-|-----------|---------|------------|
-| 1         | 12      | OWNER      |
-| 2         | 12      | READ       |
-| 3         | 13      | WRITE      |
-
-If "Alexandre" created "That project" and wants "Bidule" (and no one else) to be
-able to view and modify it (and its children), the table should then have:
-
-| person_id | node_id | permission |
-|-----------|---------|------------|
-| 3         | 19      | OWNER      |
-| 6         | 19      | WRITE      |

docs/tools/demo/tuto.md

-#User guide
-
-1. Login
-
-run the gargantex box following the install procedure
-open a webrowser at http://127.0.0.1:8000/
-click on Test Gargantext
-login with:
-```
-Login : gargantua
-Password : autnagrag
-```
-
-2. Create a project
-
-3. Import an existing corpus
-
-4. Create corpus from search
-
-5. Explore stats
-
-6. Explore graphs
-
-7. Query
-
-8. Refine
-* Time periods
-* Nodes
-9. Export

docs/tools/discover.md

-#Architecture Overview
-#Database Schema
-#Website

docs/tools/index.md

-
-
-Gargantext is a web plateform to explore your corpora using text-mining[...](about.md)
-
-## Getting started
-
-* [Install](install.md) the Gargantext box
-
-* [Take a tour](demo.md) of the different features offered by Gargantext
-
-##Need some help?
-
-Ask the community at:
-
-* [http://gargantext.org/about](http://gargantext.org/about)
-* IRC Chat: (OFTC/FreeNode) #gargantex
-
-##Want to contribute?
-
-* take a look at the [architecture overview](overview.md)
-* read the [contribution guide](contribution-guide.md)
-
-
-## News
-
-## Credits and acknowledgments

docs/tools/install.md

-#Install Instructions for Gargamelle:
-
-Gargamelle is the gargantext plateforme toolbox it is a full plateform system
-with minimal modules
-
-First you need to get the source code to install it
-The folder will be /srv/gargantext:
-* docs containes all informations on gargantext
-    /srv/gargantext/docs/
-* install contains all the installation files
-    /srv/gargantext/install/
-
-Help needed ?
-See [http://gargantext.org/about](http://gargantext.org/about) and [tools](./contribution_guide.md) for the community
-
-## Get the source code
-
-by cloning gargantext into /srv/gargantext
-
-``` bash
-git clone ssh://gitolite@delanoe.org:1979/gargantext /srv/gargantext \
-        && cd /srv/gargantext \
-        && git fetch origin stable \
-        && git checkout stable \
-```
-
-
-## Install
- ```bash
- # go into the directory
- user@computer: cd /srv/gargantext/
- #git inside installation folder
- user@computer: cd /install
- #execute the installation
- user@computer: ./install
- ```
-The installation requires to create a user for gargantext,  it will be asked:
-
-```bash
-Username (leave blank to use 'gargantua'):
-#email is not mandatory
-Email address:
-Password:
-Password (again):
-```
-If successfully done this step you should see:
-```bash
-Superuser created successfully.
-[ ok ] Stopping PostgreSQL 9.5 database server: main.
-```
-
-
-## Run
-Once you proceed to installation Gargantext plateforme will be available at localhost:8000
-to start gargantext plateform:
- ``` bash
- # go into the directory
- user@computer: cd /srv/gargantext/
- #git inside installation folder
- user@computer: ./start
- #type ctrl+d to exit or simply type exit in terminal;
- ```
-
-Then open up a chromium browser and go to localhost:8000
-Click on "Enter Gargantext"
-Login in with you created username and pasword
-
-Enjoy! ;)

docs/tools/manual_install.md

-* Create user gargantua
-Main user of Gargantext is Gargantua (role of Pantagruel soon)!
-``` bash
-sudo adduser --disabled-password --gecos "" gargantua
-```
-
-* Create the directories you need
-
-here for the example gargantext package will be installed in /srv/
-``` bash
-for dir in "/srv/gargantext"
-           "/srv/gargantext_lib"
-           "/srv/gargantext_static"
-           "/srv/gargantext_media"
-           "/srv/env_3-5"; do
-    sudo mkdir -p $dir ;
-    sudo chown gargantua:gargantua $dir ;
-done
-```
-
-You should see:
-
-```bash
-$tree /srv
-/srv
-├── gargantext
-├── gargantext_lib
-├── gargantext_media
-│   └── srv
-│       └── env_3-5
-└── gargantext_static
-```
-* Get the main libraries
-
-Download uncompress and make main user access to it.
-PLease, Be patient due to the size of the packages libraries (27GO)
-this step can be long....
-
-``` bash
-wget http://dl.gargantext.org/gargantext_lib.tar.bz2 \
-&& tar xvjf gargantext_lib.tar.bz2 -o /srv/gargantext_lib \
-&& sudo chown -R gargantua:gargantua /srv/gargantext_lib \
-&& echo "Libs installed"
-```
-
-* Get the source code of Gargantext
-
-by cloning the repository of gargantext
-``` bash
-git clone ssh://gitolite@delanoe.org:1979/gargantext /srv/gargantext \
-        && cd /srv/gargantext \
-        && git fetch origin refactoring \
-        && git checkout refactoring \
-```
-
-    TODO(soon): git clone https://gogs.iscpif.fr/gargantext.git
-
-
-See the [next steps of installation procedure](install.md#Install)

docs/tools/overview.md

-#Architecture Overview
-#Database Schema
-#Website

docs/tools/overview/parser.md

-# HOW TO: Reference a new webscrapper/API + parser
-
-## Global scope
-Three main mooves to do:
-- develop and index parser
-in gargantext.util.parsers
-- developp and index a scrapper
-in gargantext.moissonneurs
-
-- adapt forms for a new source
-in templates and views
-
-## Reference parser into gargantext website
-gargantext website is stored in gargantext/gargantext
-
-### reference your new parser into contants.py
-* import your parser l.125
-```
-from gargantext.util.parsers import \
-    EuropressParser, RISParser, PubmedParser, ISIParser, CSVParser, ISTexParser, CernParser
-```
-The parser corresponds to the name of the parser referenced in gargantext/util/parser
-here  name is CernParser
-
-
-* index your RESOURCETYPE
-int RESOURCETYPES (l.145) **at the end of the list**
-```
-# type 10
-   {    "name": 'SCOAP (XML MARC21 Format)',
-        "parser": CernParser,
-        "default_language": "en",
-        'accepted_formats':["zip","xml"],
-   },
-```
-    A noter le nom ici est composé de l'API_name(SCOAP) + (GENERICFILETYPE FORMAT_XML Format)
-    La complexité du nommage correspond à trois choses:
-        * le nom de l'API (different de l'organisme de production)
-        * le type de format: XML
-        * la norme XML de ce format : MARC21 (cf. CernParser in gargantext/util/parser/Cern.py )
-
-The default_langage corresponds to the default accepted lang that **should load** the default corresponding tagger
-
-
-```
-from gargantext.util.taggers import NltkTagger
-```
-    TO DO: charger à la demander les types de taggers en fonction des langues et de l'install
-    TO DO: proposer un module pour télécharger des parsers supplémentaires
-    TO DO: provide install tagger module scripts inside lib
-
-Les formats correspondent aux types de fichiers acceptées lors de l'envoi du fichier dans le formulaire de
-parsing disponible dans `gargantext/view/pages/projects.py` et
-exposé dans `/templates/pages/projects/project.html`
-
-## reference your parser script
-
-## add your parser script into folder gargantext/util/parser/
-here my filename was Cern.py
-
-##declare it into gargantext/util/parser/__init__.py
-from .Cern  import CernParser
-
-At this step, you will be able to see your parser and add a file with the form
-but nothing will occur
-
-## the good way to write the scrapper script
-
-Three main and only requirements:
-* your parser class should inherit from the base class _Parser()
-`gargantext/gargantext/util/parser/_Parser`
-* your parser class must have a parse method that take a **file buffer** as input
-* you parser must structure and store data into **hyperdata_list** variable name
-to be properly indexed by toolchain
-! Be careful of date format: provide a publication_date in  a string format YYYY-mm-dd HH:MM:SS
-
-# Adding a scrapper API to offer search option:
-En cours
-* Add pop up question Do you have a corpus
-option search in /templates/pages/projects/project.html line 181
-
-
-## Reference a scrapper (moissonneur) into gargantext
-
-* adding accepted_formats in constants
-* adding check_file routine in Form check ==> but should inherit from utils/files.py
-that also have implmented the size upload limit check
-
-# Suggestion 4 next steps:
-* XML parser MARC21 UNIMARC ...
-* A project type is qualified by the first element add i.e:
-the first element determine the type of corpus of all the corpora within the project
-

docs/tools/resource.md

-#resources
-
-Adding a new source into Gargantext requires a previous declaration
-of the source inside constants.py
-
-```python
-RESOURCETYPES= [
-{    "type":9, #give a unique type int
-      "name": 'SCOAP [XML]', #resource name as proposed into the add corpus FORM [generic format]
-      "parser": "CernParser", #name of the new parser class inside a CERN.py file (set to None if not implemented)
-      "format": 'MARC21', #specific format
-      'file_formats':["zip","xml"],# accepted file format
-      "crawler": "CernCrawler", #name of the new crawler class inside a CERN.py file (set to None if no Crawler implemented)
-      'default_languages': ['en', 'fr'], #supported defaut languages of the source
- },
- ...
- ]
-```
-## adding a new parser
-
-Once you declared your new parser inside constants.py
-
-add your new crawler file into /srv/gargantext/utils/parsers/
-following this naming convention:
-
-* Filename must be in uppercase without the Crawler mention.
-  eg. MailParser => MAIL.py
-* Inside this file the Parser must be called following the exact typo declared as parser in constants.py
-* Your new crawler shall inherit from baseclasse Parser and provide a parse(filebuffer) method
-
-```python
-  #!/usr/bin/python3 env
-  #filename:/srv/gargantext/util/parser/MAIL.py:
-  from ._Parser import Parser
-  class MailParser(Parser):
-      def parse(self, file):
-          ...
-```
-## adding a new crawler
-
-Once you declared your new parser inside constants.py
-add your new crawler file into /srv/gargantext/utils/parsers/
-following this naming convention:
-
-* Filename must be in uppercase without the Crawler mention.
-  eg. MailCrawler => MAIL.py
-* Inside this file the Crawler must be called following the exact typo declared as crawler in constants.py
-* Your new crawler shall inherit from baseclasse Crawler and provide three method:
-  * scan_results => ids
-  * sample = > yes/no
-  * fetch
-
-```python
-  #!/usr/bin/python3 env
-  #filename:/srv/gargantext/util/crawler/MAIL.py:
-  from ._Crawler import Crawler
-  class MailCrawler(Crawler):
-      def scan_results(self, query):
-        ...
-        self.ids = set()
-      def sample(self, results_nb):
-        ...
-      def fetch(self, ids):
-        
-```

docs/tools/schemas/ngram_parsing_flow.dot

-// dot ngram_parsing_flow.dot -Tpng -o ngram_parsing_flow.png
-digraph ngramflow {
-
-    edge [fontsize=10] ;
-    label=<<B><U>gargantext.util.toolchain</U></B><BR/>(ngram extraction flow)>;
-    labelloc="t" ;
-
-    "extracted_ngrams" -> "grouplist" ;
-    "extracted_ngrams" -> "occs+tfidfs" ;
-    "main_user_stoplist" -> "stoplist" ;
-    "stoplist" -> "mainlist" ;
-    "occs+tfidfs" -> "mainlist" [label="  TFIDF_LIMIT"];
-    "mainlist" -> "coocs" [label="  COOCS_THRESHOLD"] ;
-    "coocs" -> "specificity" ;
-
-    "specificity" -> "maplist" [label="MAPLIST_LIMIT\nMONOGRAM_PART"];
-
-    "maplist" -> "explore" ;
-    "grouplist" -> "maplist" ;
-}

gargantext/__init__.py

+from .celery import app as celery_app

gargantext/celery.py

+"""
+Setup the Celery instance (see also gargantext/__init__.py) that will be
+used by all shared_task.
+
+This is the recommended way:
+http://docs.celeryproject.org/en/3.1/django/first-steps-with-django.html
+"""
+
+import os
+from celery import Celery
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'gargantext.settings')
+
+from django.conf import settings #noqa
+
+app = Celery('gargantext')
+app.config_from_object('django.conf:settings')
+app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

gargantext/constants.py

@@ -36,7 +36,7 @@ import os
 import re
 import importlib
 from gargantext.util.lists import *
-from gargantext.util.tools import datetime, convert_to_date
+from gargantext.util import datetime, convert_to_datetime
 from .settings import BASE_DIR
 
 # types & models (nodes, lists, hyperdata, resource) ---------------------------------------------
@@ -59,25 +59,25 @@ LISTTYPES = {
 NODETYPES = [
     # TODO separate id not array index, read by models.node
     None,                    # 0
-    # documents hierarchy
+    # node/file hierarchy
     'USER',                  # 1
     'PROJECT',               # 2
     #RESOURCE should be here but last
     'CORPUS',                # 3
     'DOCUMENT',              # 4
-    # lists
+    # lists of ngrams
     'STOPLIST',              # 5
     'GROUPLIST',             # 6
     'MAINLIST',              # 7
     'MAPLIST',               # 8
     'COOCCURRENCES',         # 9
-    # scores
+    # scores for ngrams
     'OCCURRENCES',           # 10
     'SPECCLUSION',           # 11
     'CVALUE',                # 12
     'TFIDF-CORPUS',          # 13
     'TFIDF-GLOBAL',          # 14
-    # docs subset
+    # node subset
     'FAVORITES',             # 15
     # more scores (sorry!)
 
@@ -108,9 +108,9 @@ INDEXED_HYPERDATA = {
 
     'publication_date':
         { 'id'             : 2
-        , 'type'           : datetime.datetime
-        , 'convert_to_db'  : convert_to_date
-        , 'convert_from_db': datetime.datetime.fromtimestamp
+        , 'type'           : datetime
+        , 'convert_to_db'  : convert_to_datetime
+        , 'convert_from_db': convert_to_datetime
         },
 
     'title':
@@ -197,7 +197,7 @@ RESOURCETYPES = [
         'crawler': None,
     },
     {   'type': 3,
-        'name': 'Pubmed [CRAWLER/XML]',
+        'name': 'Pubmed [XML]',
         'format': 'Pubmed',
         'parser': "PubmedParser",
         'file_formats':["zip", "xml"],
@@ -233,14 +233,14 @@ RESOURCETYPES = [
         'crawler': None,
     },
     {   'type': 8,
-        'name': 'ISTex [CRAWLER]',
+        'name': 'ISTex',
         'format': 'json',
         'parser': "ISTexParser",
         'file_formats':["zip", "txt"],
         'crawler': None,
     },
    {    "type": 9,
-        "name": 'SCOAP [CRAWLER/XML]',
+        "name": 'SCOAP [API/XML]',
         "parser": "CernParser",
         "format": 'MARC21',
         'file_formats':["zip","xml"],
@@ -255,7 +255,7 @@ RESOURCETYPES = [
 #   },
 #
    {    "type": 10,
-        "name": 'REPEC [CRAWLER]',
+        "name": 'REPEC [MULTIVAC API]',
         "parser": "MultivacParser",
         "format": 'JSON',
         'file_formats':["zip","json"],
@@ -263,13 +263,21 @@ RESOURCETYPES = [
    },
 
    {    "type": 11,
-        "name": 'HAL [CRAWLER]',
+        "name": 'HAL (english) [API]',
         "parser": "HalParser",
         "format": 'JSON',
         'file_formats':["zip","json"],
         "crawler": "HalCrawler",
    },
 
+   {    "type": 12,
+        "name": 'ISIDORE [SPARQLE API /!\ BETA]',
+        "parser": "IsidoreParser",
+        "format": 'JSON',
+        'file_formats':["zip","json"],
+        "crawler": "IsidoreCrawler",
+   },
+
 ]
 #shortcut for resources declaration in template
 PARSERS = [(n["type"],n["name"]) for n in RESOURCETYPES if n["parser"] is not None]

gargantext/management/commands/nodes.py

+from django.core.management.base import BaseCommand, CommandError
+from gargantext.util.show_nodes import tree_show, nodes
+import colorama
+
+
+class Command(BaseCommand):
+    help = 'Nodes'
+
+    def add_arguments(self, parser):
+        parser.add_argument(dest='action', default='show')
+
+    def handle(self, *args, **options):
+        action = options.get('action')
+
+        if action == 'show':
+            colorama.init(strip=False)
+            for root in nodes():
+                tree_show(root)

gargantext/models/__init__.py

+from .base import Base
 from .nodes import *
 from .hyperdata import *
 from .users import *

gargantext/models/base.py

+from sqlalchemy.schema import Column, ForeignKey, UniqueConstraint, Index
+from sqlalchemy.orm import relationship, validates
+from sqlalchemy.types import TypeDecorator, \
+                             Integer, Float, Boolean, DateTime, String, Text
+from sqlalchemy.dialects.postgresql import JSONB, DOUBLE_PRECISION as Double
+from sqlalchemy.ext.mutable import MutableDict, MutableList
+from sqlalchemy.ext.declarative import declarative_base
+
+__all__ = ["Column", "ForeignKey", "UniqueConstraint", "relationship",
+           "validates", "ValidatorMixin",
+           "Integer", "Float", "Boolean", "DateTime", "String", "Text",
+           "TypeDecorator",
+           "JSONB", "Double",
+           "MutableDict", "MutableList",
+           "Base", "DjangoBase"]
+
+
+# All the models should derive from this base class, so Base.metadata keeps
+# all tables handled by Alembic migration scripts.
+Base = declarative_base()
+
+
+# To be used by tables already handled by Django ORM, such as User model. We
+# separate them in order to keep those out of Alembic sight.
+DjangoBase = declarative_base()
+
+
+class ValidatorMixin(object):
+    def enforce_length(self, key, value):
+        """Truncate a string according to its column length
+
+        Usage example:
+
+        .. code-block:: python
+
+            @validates('some_column')
+            def validate_some_column(self, key, value):
+                self.enforce_length(key, value)
+        """
+        max_len = getattr(self.__class__, key).prop.columns[0].type.length
+        if value and len(value) > max_len:
+            return value[:max_len]
+        return value

gargantext/models/hyperdata.py

-from gargantext.util.db import *
 from gargantext.constants import INDEXED_HYPERDATA
 
+from .base import Base, Column, ForeignKey, TypeDecorator, Index, \
+                  Integer, Double, DateTime, String, Text
 from .nodes import Node
 import datetime
 
@@ -64,6 +65,14 @@ class NodeHyperdata(Base):
         )
     """
     __tablename__ = 'nodes_hyperdata'
+    __table_args__ = (
+            Index('nodes_hyperdata_node_id_value_utc_idx', 'node_id', 'value_utc'),
+            Index('nodes_hyperdata_node_id_key_value_utc_idx', 'node_id', 'key', 'value_utc'),
+            Index('nodes_hyperdata_node_id_key_value_str_idx', 'node_id', 'key', 'value_str'),
+            Index('nodes_hyperdata_node_id_key_value_int_idx', 'node_id', 'key', 'value_int'),
+            Index('nodes_hyperdata_node_id_key_value_flt_idx', 'node_id', 'key', 'value_flt'),
+            Index('nodes_hyperdata_node_id_key_idx', 'node_id', 'key'))
+
     id        = Column( Integer, primary_key=True )
     node_id   = Column( Integer, ForeignKey(Node.id, ondelete='CASCADE'))
     key       = Column( HyperdataKey )

gargantext/models/migration.py

-from gargantext.util.db import *
-from gargantext.util.files import upload
-from gargantext.constants import *
-from gargantext.util.toolchain.main import parse_extract_indexhyperdata
-
-from datetime import datetime
-
-from .users import User
-from .nodes import Node
-
-#__all__ = ['Node', 'NodeType', 'Language']
-
-
-class NodeType_v2(Base):
-    __table_args__ = {'extend_existing': True}
-    __tablename__  = 'node_nodetype'
-    id             = Column(Integer, primary_key=True)
-    name           = Column(String(255))
-
-class Language_v2(Base):
-    __table_args__ = {'extend_existing': True}
-    __tablename__  = 'node_language'
-    id             = Column(Integer, primary_key=True)
-    iso2           = Column(String(2))
-    iso3           = Column(String(3))
-    fullname       = Column(String(255))
-    implemented    = Column(Boolean)
-
-
-class Node_v2(Base):
-    __table_args__ = {'extend_existing': True}
-    __tablename__  = 'node_node'
-    id             = Column(Integer, primary_key=True)
-    parent_id      = Column(Integer, ForeignKey('node_node.id'))
-    user_id        = Column(Integer, ForeignKey(User.id))
-    type_id        = Column(ForeignKey(NodeType_v2.id))
-    name           = Column(String(255))
-    language_id    = Column(Integer, ForeignKey(Language_v2.id))
-    date           = Column(DateTime(), default=datetime.now)
-    hyperdata      = Column(JSONB, default=dict)
-
-
-class ResourceType(Base):
-    __table_args__ = {'extend_existing': True}
-    __tablename__  = 'node_resourcetype'
-    id             = Column(Integer, primary_key=True)
-    name           = Column(String(255))
-
-class NodeResource(Base):
-    __table_args__ = {'extend_existing': True}
-    __tablename__  = 'node_node_resource'
-    id             = Column(Integer, primary_key=True)
-    node_id        = Column(ForeignKey(Node_v2.id))
-    resource_id    = Column(ForeignKey(ResourceType.id))
-    parsed         = Column(Boolean)
-
-
-
-def nodes_list(user_id, nodetype, parent_id=None, count=False):
-    """
-    nodes_list :: Int -> String -> Maybe Int -> Maybe Bool -> [(Int, String)]
-    """
-    nodes = ( session.query(Node_v2.id, Node_v2.name)
-                     .join(NodeType_v2, NodeType_v2.id == Node_v2.type_id)
-                     .filter(NodeType_v2.name == nodetype)
-               )
-    if parent_id is not None:
-        nodes = nodes.filter(Node_v2.parent_id == parent_id)
-
-    if count is True:
-        return nodes.count()
-    else:
-        return nodes.all()
-
-def nodes_tree(user_id):
-    """
-    nodes_tree :: Int -> Tree Nodes
-    """
-    for project_id, project_name in nodes_list(user_id, 'Project'):
-        print("* Project (%d, %s)" % (project_id, project_name))
-        for corpus_id, corpus_name in nodes_list(user_id, 'Corpus', parent_id=project_id):
-
-            count =  nodes_list( user_id
-                               , 'Document'
-                               , parent_id=corpus_id
-                               , count=True
-                               )
-
-            if count > 1:
-                print("|__ %d %s" % ( corpus_id, corpus_name ))
-                print("   |___ %s docs" % count)
-
-
-def copy_nodes(node_id, to_parent_id=None, enabled=['PROJECT', 'CORPUS', 'DOCUMENT']):
-    node = session.query(Node_v2).filter(Node_v2.id==node_id).first()
-    
-    nodetype = session.query(NodeType_v2).filter(NodeType_v2.id == node.type_id).first()
-    
-    resource = (session.query(ResourceType)
-                           .join(NodeResource, NodeResource.resource_id == ResourceType.id)
-                           .filter(NodeResource.node_id == node.id)
-                           .first()
-                           )
-    
-    nodetype_proj_id = session.query(NodeType_v2.id).filter(NodeType_v2.name == 'Project' ).first()
-    nodetype_corp_id = session.query(NodeType_v2.id).filter(NodeType_v2.name == 'Corpus'  ).first()
-    nodetype_docu_id = session.query(NodeType_v2.id).filter(NodeType_v2.name == 'Document').first()
-
-    typename = nodetype.name.upper()
-
-# Import a project:
-#         new_project = Node(
-#             user_id = user.id,
-#             typename = 'PROJECT',
-#             name = name,
-#         )
-#         session.add(new_project)
-#         session.commit()
-
-    if typename in enabled:
-        parent_node = session.query(Node).filter(Node.id==to_parent_id).first()
-        if parent_node is not None:
-            corpus = parent_node.add_child(
-                name = node.name,
-                typename = typename
-                )
-            
-            corpus.hyperdata['languages'] = {'fr' : 100}
-            try:
-                corpus.add_resource(
-                    type = resourcetype(resource.name)
-                    )
-            except:
-                corpus.add_resource(
-                    type = resourcetype('Europress (French)')
-                    )
-            session.add(corpus)
-            session.commit()
-            print("%s copied" % corpus.name)
-
-
-            nodes = (session.query(Node_v2)
-                             .filter(Node_v2.parent_id == node.id)
-                             .filter(Node_v2.type_id == nodetype_docu_id)
-                    .all()
-                    )
-
-            for n in nodes:
-                print(n.name)
-                doc = corpus.add_child( name      = n.name
-                                      , typename  = "DOCUMENT"
-                                      , hyperdata = n.hyperdata
-                                      )
-                session.add(doc)
-                session.commit()
-            
-
-#        else:
-#            print("%d is None" % parent_id)
-
-    else:
-        print('%s is not enabled' % typename)
-
-
-
-

gargantext/models/ngrams.py

-from gargantext.util.db import *
-
+from .base import Base, Column, ForeignKey, relationship, Index, \
+                  Integer, Float, String
 from .nodes import Node
 
 __all__ = ['Ngram', 'NodeNgram', 'NodeNodeNgram', 'NodeNgramNgram']
@@ -7,17 +7,39 @@ __all__ = ['Ngram', 'NodeNgram', 'NodeNodeNgram', 'NodeNgramNgram']
 
 class Ngram(Base):
     __tablename__ = 'ngrams'
+    __table_args__ = (
+            Index('ngrams_id_n_idx', 'id', 'n'),
+            Index('ngrams_n_idx', 'n'))
+
     id = Column(Integer, primary_key=True)
     terms = Column(String(255), unique=True)
     n = Column(Integer)
 
+    def __str__(self):
+        return '<{0.terms}>#{0.n}'.format(self)
+
+    def __repr__(self):
+        return '<Ngram(id={0.id}, terms={0.terms!r}, n={0.n})>'.format(self)
+
 
 class NodeNgram(Base):
     __tablename__ = 'nodes_ngrams'
+    __table_args__ = (
+            Index('nodes_ngrams_node_id_ngram_id_idx', 'node_id', 'ngram_id'),
+            Index('nodes_ngrams_node_id_idx', 'node_id'),
+            Index('nodes_ngrams_ngram_id_idx', 'ngram_id'))
+
     node_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
     ngram_id = Column(Integer, ForeignKey(Ngram.id, ondelete='CASCADE'), primary_key=True)
     weight = Column(Float)
 
+    node = relationship(Node)
+    ngram = relationship(Ngram)
+
+    def __repr__(self):
+        return '<NodeNgram(node_id={0.node_id}, ngram={0.ngram}, weight={0.weight})>'.format(self)
+
+
 class NodeNodeNgram(Base):
     """ for instance for TFIDF
     (
@@ -28,6 +50,10 @@ class NodeNodeNgram(Base):
     )
     """
     __tablename__ = 'nodes_nodes_ngrams'
+    __table_args__ = (
+            Index('nodes_nodes_ngrams_node2_id_idx', 'node2_id'),
+            Index('nodes_nodes_ngrams_node1_id_idx', 'node1_id'))
+
     node1_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
     node2_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
     ngram_id = Column(Integer, ForeignKey(Ngram.id, ondelete='CASCADE'), primary_key=True)
@@ -36,6 +62,14 @@ class NodeNodeNgram(Base):
     # sinon par défaut on aurait un type sql "double_precision" (soit 15 chiffres)
     # (cf. www.postgresql.org/docs/9.4/static/datatype-numeric.html#DATATYPE-FLOAT)
 
+    node1 = relationship(Node, foreign_keys=[node1_id])
+    node2 = relationship(Node, foreign_keys=[node2_id])
+    ngram = relationship(Ngram)
+
+    def __repr__(self):
+        return '<NodeNodeNgram(node1_id={0.node1_id}, node2_id={0.node2_id}, ngram={0.ngram}, score={0.score})>'.format(self)
+
+
 class NodeNgramNgram(Base):
     """ for instance for COOCCURRENCES and GROUPLIST
     (
@@ -46,7 +80,20 @@ class NodeNgramNgram(Base):
     )
     """
     __tablename__ = 'nodes_ngrams_ngrams'
+    __table_args__ = (
+            Index('nodes_ngrams_ngrams_node_id_ngram1_id_ngram2_id_idx', 'node_id', 'ngram1_id', 'ngram2_id'),
+            Index('nodes_ngrams_ngrams_node_id_idx', 'node_id'),
+            Index('nodes_ngrams_ngrams_ngram1_id_idx', 'ngram1_id'),
+            Index('nodes_ngrams_ngrams_ngram2_id_idx', 'ngram2_id'))
+
     node_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
     ngram1_id = Column(Integer, ForeignKey(Ngram.id, ondelete='CASCADE'), primary_key=True)
     ngram2_id = Column(Integer, ForeignKey(Ngram.id, ondelete='CASCADE'), primary_key=True)
     weight = Column(Float(precision=24))  # see comment for NodeNodeNgram.score
+
+    node = relationship(Node)
+    ngram1 = relationship(Ngram, foreign_keys=[ngram1_id])
+    ngram2 = relationship(Ngram, foreign_keys=[ngram2_id])
+
+    def __repr__(self):
+        return '<NodeNgramNgram(node_id={0.node_id}, ngram1={0.ngram1}, ngram2={0.ngram2}, weight={0.weight})>'.format(self)

gargantext/models/nodes.py

-from gargantext.util.db import *
+from gargantext.util.db import session
 from gargantext.util.files import upload
 from gargantext.constants import *
 
+# Uncomment to make column full text searchable
+#from sqlalchemy_utils.types import TSVectorType
+
 from datetime import datetime
 
+from .base import Base, Column, ForeignKey, relationship, TypeDecorator, Index, \
+                  Integer, Float, String, DateTime, JSONB, \
+                  MutableList, MutableDict, validates, ValidatorMixin
 from .users import User
 
-__all__ = ['Node', 'NodeNode']
+__all__ = ['Node', 'NodeNode', 'CorpusNode']
 
 class NodeType(TypeDecorator):
     """Define a new type of column to describe a Node's type.
@@ -19,23 +25,69 @@ class NodeType(TypeDecorator):
     def process_result_value(self, typeindex, dialect):
         return NODETYPES[typeindex]
 
-class Node(Base):
-    """This model can fit many purposes.
+
+class Node(ValidatorMixin, Base):
+    """This model can fit many purposes:
+
+    myFirstCorpus = session.query(CorpusNode).first()
+
     It intends to provide a generic model, allowing hierarchical structure
     and NoSQL-like data structuring.
     The possible types are defined in `gargantext.constants.NODETYPES`.
+
+    Thanks to __new__ overriding and SQLAlchemy's polymorphism, every Node
+    instance is automagically casted to its sub-class, assuming a typename
+    is specified.
+
+    >>> Node(name='without-type')
+    <Node(id=None, typename=None, user_id=None, parent_id=None, name='without-type', date=None)>
+    >>> Node(typename='CORPUS')
+    <CorpusNode(id=None, typename='CORPUS', user_id=None, parent_id=None, name=None, date=None)>
+    >>> from gargantext.util.db import session
+    >>> session.query(Node).filter_by(typename='USER').first() # doctest: +ELLIPSIS
+    <UserNode(...)>
+
+    But beware, there are some caveats with bulk queries. In this case typename
+    MUST be specified manually.
+
+    >>> session.query(UserNode).delete() # doctest: +SKIP
+    # Wrong: all nodes are deleted!
+    >>> session.query(UserNode).filter_by(typename='USER').delete() # doctest: +SKIP
+    # Right: only user nodes are deleted.
     """
     __tablename__ = 'nodes'
+    __table_args__ = (
+            Index('nodes_user_id_typename_parent_id_idx', 'user_id', 'typename', 'parent_id'),
+            Index('nodes_hyperdata_idx', 'hyperdata', postgresql_using='gin'))
+
+    # TODO
+    # create INDEX full_text_idx on nodes using gin(to_tsvector('english', hyperdata ->> 'abstract' || 'title'));
+
     id = Column(Integer, primary_key=True)
+
     typename = Column(NodeType, index=True)
+    __mapper_args__ = { 'polymorphic_on': typename }
+
     # foreign keys
-    user_id = Column(Integer, ForeignKey(User.id, ondelete='CASCADE'))
-    parent_id = Column(Integer, ForeignKey('nodes.id', ondelete='CASCADE'))
-    # main data
+    user_id       = Column(Integer, ForeignKey(User.id, ondelete='CASCADE'))
+    user          = relationship(User)
+
+    parent_id     = Column(Integer, ForeignKey('nodes.id', ondelete='CASCADE'))
+    parent        = relationship('Node', remote_side=[id])
+
     name = Column(String(255))
-    date  = Column(DateTime(), default=datetime.now)
+    date  = Column(DateTime(timezone=True), default=datetime.now)
+
+    hyperdata     = Column(JSONB, default=dict)
     # metadata (see https://bashelton.com/2014/03/updating-postgresql-json-fields-via-sqlalchemy/)
-    hyperdata = Column(JSONB, default=dict)
+    # To make search possible uncomment the line below
+    #search_vector = Column(TSVectorType('hyperdata'))
+
+    def __new__(cls, *args, **kwargs):
+        if cls is Node and kwargs.get('typename'):
+            typename = kwargs.pop('typename')
+            return _NODE_MODELS[typename](*args, **kwargs)
+        return super(Node, cls).__new__(cls)
 
     def __init__(self, **kwargs):
         """Node's constructor.
@@ -55,6 +107,15 @@ class Node(Base):
         """
         self.hyperdata[key] = value
 
+    def __repr__(self):
+        return '<{0.__class__.__name__}(id={0.id}, typename={0.typename!r}, ' \
+               'user_id={0.user_id}, parent_id={0.parent_id}, ' \
+               'name={0.name!r}, date={0.date})>'.format(self)
+
+    @validates('name')
+    def validate_name(self, key, value):
+        return self.enforce_length(key, value)
+
     @property
     def ngrams(self):
         """Pseudo-attribute allowing to retrieve a node's ngrams.
@@ -120,36 +181,6 @@ class Node(Base):
             **kwargs
         )
 
-    def resources(self):
-        """Return all the resources attached to a given node.
-        Mainly used for corpora.
-
-        example:
-        [{'extracted': True,
-          'path': '/home/me/gargantext/uploads/corpora/0c/0c5b/0c5b50/0c5b50ad8ebdeb2ae33d8e54141a52ee_Corpus_Europresse-Français-2015-12-11.zip',
-          'type': 1,
-          'url': None}]
-        """
-        if 'resources' not in self.hyperdata:
-            self['resources'] = MutableList()
-        return self['resources']
-
-    def add_resource(self, type, path=None, url=None):
-        """Attach a resource to a given node.
-        Mainly used for corpora.
-
-        this just adds metadata to the CORPUS node (NOT for adding documents)
-
-        example:
-        {'extracted': True,
-          'path': '/home/me/gargantext/uploads/corpora/0c/0c5b/0c5b50/0c5b50ad8ebdeb2ae33d8e54141a52ee_Corpus_Europresse-Français-2015-12-11.zip',
-          'type': 1,
-          'url': None}
-        """
-        self.resources().append(MutableDict(
-            {'type': type, 'path':path, 'url':url, 'extracted': False}
-        ))
-
     def status(self, action=None, progress=0, complete=False, error=None):
         """Get or update the status of the given action.
         If no action is given, the status of the first uncomplete or last item
@@ -187,8 +218,86 @@ class Node(Base):
         ))
         return self['statuses'][-1]
 
+
+class CorpusNode(Node):
+    __mapper_args__ = {
+        'polymorphic_identity': 'CORPUS'
+    }
+
+    def resources(self):
+        """Return all the resources attached to a given node.
+
+        example:
+        [{'extracted': True,
+          'path': '/home/me/gargantext/uploads/corpora/0c/0c5b/0c5b50/0c5b50ad8ebdeb2ae33d8e54141a52ee_Corpus_Europresse-Français-2015-12-11.zip',
+          'type': 1,
+          'url': None}]
+        """
+        if 'resources' not in self.hyperdata:
+            self['resources'] = MutableList()
+        return self['resources']
+
+    def add_resource(self, type, path=None, url=None):
+        """Attach a resource to a given node.
+
+        this just adds metadata to the CORPUS node (NOT for adding documents)
+
+        example:
+        {'extracted': True,
+          'path': '/home/me/gargantext/uploads/corpora/0c/0c5b/0c5b50/0c5b50ad8ebdeb2ae33d8e54141a52ee_Corpus_Europresse-Français-2015-12-11.zip',
+          'type': 1,
+          'url': None}
+        """
+        self.resources().append(MutableDict(
+            {'type': type, 'path':path, 'url':url, 'extracted': False}
+        ))
+
+
 class NodeNode(Base):
     __tablename__ = 'nodes_nodes'
+    __table_args__ = (
+            Index('nodes_nodes_node1_id_node2_id_idx', 'node1_id', 'node2_id'),)
+
     node1_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
     node2_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
     score    = Column(Float(precision=24))
+
+    node1 = relationship(Node, foreign_keys=[node1_id])
+    node2 = relationship(Node, foreign_keys=[node2_id])
+
+    def __repr__(self):
+        return '<NodeNode(node1_id={0.node1_id}, node2_id={0.node2_id}, score={0.score})>'.format(self)
+
+
+# --8<-- Begin hack ------
+
+# XXX Hack to automatically defines subclasses of Node for every NODETYPES,
+#     in order to avoid SQLAlchemy complaints -- and subsequent exceptions.
+#
+#     We could manually write a class for every NodeType, or find a way to
+#     tell SQLAlchemy that it should stick to instantiate a Node when a
+#     class is not defined for the wanted typename.
+
+_ALREADY_IMPLEMENTED_NODE_TYPES = \
+    set(cls.__mapper_args__.get('polymorphic_identity') for cls in Node.__subclasses__())
+
+for nodetype in NODETYPES:
+    if nodetype and nodetype not in _ALREADY_IMPLEMENTED_NODE_TYPES:
+        # Convert nodetype to a CamelCase class name, assuming it's possible...
+        class_name = ''.join(nodetype.title().split("-")) + 'Node'
+        # Create new class and add it to global scope
+        globals()[class_name] = type(class_name, (Node,), {
+            "__mapper_args__": {
+                "polymorphic_identity": nodetype
+            }
+        })
+        # Add class to exports
+        __all__.append(class_name)
+
+# ------ End of hack ------
+
+_NODE_MODELS = {
+    mapper.polymorphic_identity: mapper.class_
+    for mapper in Node.__mapper__.self_and_descendants
+    if mapper.class_ is not Node
+}

gargantext/models/users.py

 from django.contrib.auth import models
-from gargantext.util.db import *
+from gargantext.util.db import session, aliased
 
 from datetime import datetime
 
+from .base import DjangoBase, Base, Column, ForeignKey, UniqueConstraint, \
+                  Integer, Boolean, DateTime, String
+
 __all__ = ['User', 'Contact']
 
-class User(Base):
+class User(DjangoBase):
     # The properties below are a reflection of Django's auth module's models.
     __tablename__ = models.User._meta.db_table
     id           = Column(Integer, primary_key=True)
@@ -60,7 +63,7 @@ class User(Base):
         """check if a given node is owned by the user"""
         return (node.user_id == self.id) or \
                 node.id in (contact.id for contact in self.contacts())
-    
+
     def get_params(self, username=None):
         print(self.__dict__.items())
         return self.hyperdata

gargantext/util/__init__.py

+from .dates import datetime, convert_to_datetime, MINYEAR

gargantext/util/alembic.py

+"""Define ReplaceableObject and related operations
+
+Implements operations to create/drop SQL objects such as views, stored
+procedures and triggers that can't be "altered" but can be replaced -- hence
+the name of "ReplaceableObject" class.
+
+This recipe is directly borrowed from Alembic documentation, see
+http://alembic.zzzcomputing.com/en/latest/cookbook.html#replaceable-objects
+
+"""
+
+from alembic.operations import Operations, MigrateOperation
+
+
+__all__ = ['ReplaceableObject']
+
+
+class ReplaceableObject(object):
+    def __init__(self, name, sqltext):
+        self.name = name
+        self.sqltext = sqltext
+
+
+class ReversibleOp(MigrateOperation):
+    def __init__(self, target):
+        self.target = target
+
+    @classmethod
+    def invoke_for_target(cls, operations, target):
+        op = cls(target)
+        return operations.invoke(op)
+
+    def reverse(self):
+        raise NotImplementedError()
+
+    @classmethod
+    def _get_object_from_version(cls, operations, ident):
+        version, objname = ident.split(".")
+
+        module = operations.get_context().script.get_revision(version).module
+        obj = getattr(module, objname)
+        return obj
+
+    @classmethod
+    def replace(cls, operations, target, replaces=None, replace_with=None):
+
+        if replaces:
+            old_obj = cls._get_object_from_version(operations, replaces)
+            drop_old = cls(old_obj).reverse()
+            create_new = cls(target)
+        elif replace_with:
+            old_obj = cls._get_object_from_version(operations, replace_with)
+            drop_old = cls(target).reverse()
+            create_new = cls(old_obj)
+        else:
+            raise TypeError("replaces or replace_with is required")
+
+        operations.invoke(drop_old)
+        operations.invoke(create_new)
+
+
+@Operations.register_operation("create_view", "invoke_for_target")
+@Operations.register_operation("replace_view", "replace")
+class CreateViewOp(ReversibleOp):
+    def reverse(self):
+        return DropViewOp(self.target)
+
+
+@Operations.register_operation("drop_view", "invoke_for_target")
+class DropViewOp(ReversibleOp):
+    def reverse(self):
+        return CreateViewOp(self.view)
+
+
+@Operations.register_operation("create_sp", "invoke_for_target")
+@Operations.register_operation("replace_sp", "replace")
+class CreateSPOp(ReversibleOp):
+    def reverse(self):
+        return DropSPOp(self.target)
+
+
+@Operations.register_operation("drop_sp", "invoke_for_target")
+class DropSPOp(ReversibleOp):
+    def reverse(self):
+        return CreateSPOp(self.target)
+
+
+@Operations.implementation_for(CreateViewOp)
+def create_view(operations, operation):
+    operations.execute("CREATE VIEW %s AS %s" % (
+        operation.target.name,
+        operation.target.sqltext
+    ))
+
+
+@Operations.implementation_for(DropViewOp)
+def drop_view(operations, operation):
+    operations.execute("DROP VIEW %s" % operation.target.name)
+
+
+@Operations.implementation_for(CreateSPOp)
+def create_sp(operations, operation):
+    operations.execute(
+        "CREATE FUNCTION %s %s" % (
+            operation.target.name, operation.target.sqltext
+        )
+    )
+
+
+@Operations.implementation_for(DropSPOp)
+def drop_sp(operations, operation):
+    operations.execute("DROP FUNCTION %s" % operation.target.name)

gargantext/util/crawlers/HAL.py

@@ -29,16 +29,18 @@ class HalCrawler(Crawler):
         '''formating the query'''
 
         #search_field="title_t"
-        search_field="abstract_t"
+        #search_field="abstract_t"
 
-        return (search_field + ":" + "(" + query  + ")")
+        #return (search_field + ":" + "(" + query  + ")")
+        return "(" + query + ")"
 
 
     def _get(self, query, fromPage=1, count=10, lang=None):
         # Parameters
 
-        fl = """ title_s
-               , abstract_s
+        fl = """ en_title_s
+               , en_title_s
+               , en_abstract_s
                , submittedDate_s
                , journalDate_s
                , authFullName_s
@@ -53,6 +55,7 @@ class HalCrawler(Crawler):
                , deptStructId_i
                , labStructId_i
                , rteamStructId_i
+               , docType_s
              """
                #, authUrl_s
                #, type_s
@@ -119,7 +122,7 @@ class HalCrawler(Crawler):
             msg = "Invalid sample size N = %i (max = %i)" % ( self.query_max
                                                             , QUERY_SIZE_N_MAX
                                                             )
-            print("ERROR (scrap: Multivac d/l ): " , msg)
+            print("ERROR (scrap: HAL d/l ): " , msg)
             self.query_max = QUERY_SIZE_N_MAX
         
         #for page in range(1, trunc(self.query_max / 100) + 2):

gargantext/util/crawlers/ISIDORE.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# ****************************
+# ****  ISIDORE  Scrapper  ***
+# ****************************
+# CNRS COPYRIGHTS
+# SEE LEGAL LICENCE OF GARGANTEXT.ORG
+
+from ._Crawler import *
+import json
+from gargantext.constants  import UPLOAD_DIRECTORY
+from math                  import trunc
+from gargantext.util.files import save
+from gargantext.util.crawlers.sparql.bool2sparql import bool2sparql, isidore
+
+
+class IsidoreCrawler(Crawler):
+    ''' ISIDORE SPARQL API CLIENT'''
+    
+    def __init__(self):
+        # Main EndPoints
+        self.BASE_URL = "https://www.rechercheisidore.fr"
+        self.API_URL  = "sparql"
+        
+        # Final EndPoints
+        # TODO : Change endpoint according type of database
+        self.URL   = self.BASE_URL + "/" + self.API_URL
+        self.status = []
+
+    def __format_query__(self, query=None, count=False, offset=None, limit=None):
+        '''formating the query'''
+
+        return (bool2sparql(query, count=count, offset=offset, limit=limit))
+
+
+    def _get(self, query, offset=0, limit=None, lang=None):
+        '''Parameters to download data'''
+        
+        isidore(query, count=False, offset=offset, limit=limit)
+
+    def scan_results(self, query):
+        '''
+        scan_results : Returns the number of results
+        Query String -> Int
+        '''
+        self.results_nb = [n for n in isidore(query, count=True)][0]
+        return self.results_nb
+
+    def download(self, query):
+        
+        downloaded = False
+        
+        self.status.append("fetching results")
+
+        corpus = []
+        limit = 1000
+        self.query_max = self.scan_results(query)
+        print("self.query_max : %s" % self.query_max)
+
+        if self.query_max > QUERY_SIZE_N_MAX:
+            msg = "Invalid sample size N = %i (max = %i)" % ( self.query_max
+                                                            , QUERY_SIZE_N_MAX
+                                                            )
+            print("WARNING (scrap: ISIDORE d/l ): " , msg)
+            self.query_max = QUERY_SIZE_N_MAX
+        
+        for offset in range(0, self.query_max, limit):
+            print("Downloading result %s to %s" % (offset, self.query_max))
+
+            for doc in isidore(query, offset=offset, limit=limit) :
+                corpus.append(doc)
+
+        self.path = save( json.dumps(corpus).encode("utf-8")
+                        , name='ISIDORE.json'
+                        , basedir=UPLOAD_DIRECTORY
+                        )
+        downloaded = True
+        
+        return downloaded

gargantext/util/crawlers/sparql/bool2sparql.py

+
+import subprocess
+import re
+from .sparql import Service
+from gargantext.settings import BOOL_TOOLS_PATH
+#from sparql import Service
+
+def bool2sparql(rawQuery, count=False, offset=None, limit=None):
+    """
+    bool2sparql :: String -> Bool -> Int -> String
+    Translate a boolean query into a Sparql request
+    You need to build bool2sparql binaries before
+    See: https://github.com/delanoe/bool2sparql
+    """
+    query = re.sub("\"", "\'", rawQuery)
+    bashCommand = [BOOL_TOOLS_PATH + "/bool2sparql-exe","-q",query]
+
+    if count is True :
+        bashCommand.append("-c")
+    else :
+        if offset is not None :
+            for command in ["--offset", str(offset)] :
+                bashCommand.append(command)
+        
+        if limit is not None :
+            for command in ["--limit", str(limit)] :
+                bashCommand.append(command)
+
+
+    process = subprocess.Popen(bashCommand, stdout=subprocess.PIPE)
+    output, error = process.communicate()
+    
+    if error is not None :
+        raise(error)
+    else :
+        print(output)
+        return(output.decode("utf-8"))
+
+def isidore(query, count=False, offset=None, limit=None):
+    """
+    isidore :: String -> Bool -> Int -> Either (Dict String) Int
+    use sparql-client either to search or to scan
+    """
+
+    query = bool2sparql(query, count=count, offset=offset, limit=limit)
+    
+    go = Service("https://www.rechercheisidore.fr/sparql/", "utf-8", "GET")
+    results = go.query(query)
+
+    if count is False:
+        for r in results:
+            doc        = dict()
+            doc_values = dict()
+            doc["url"], doc["title"], doc["date"], doc["abstract"], doc["source"] = r
+            
+            for k in doc.keys():
+                doc_values[k] = doc[k].value
+            
+            yield(doc_values)
+
+
+    else :
+        count = []
+        for r in results:
+            n, = r
+            count.append(int(n.value))
+        yield count[0]
+
+
+def test():
+    query = "delanoe"
+    limit  = 100
+    offset = 10
+
+    for d in isidore(query, offset=offset, limit=limit):
+        print(d["date"])
+    #print([n for n in isidore(query, count=True)])
+
+if __name__ == '__main__':
+    test()

gargantext/util/dates.py

+import os
+from gargantext.settings import MEDIA_ROOT
+
+from datetime import MINYEAR
+from django.utils.dateparse import parse_datetime
+from django.utils.timezone import datetime as _datetime, utc as UTC, now as utcnow
+
+__all__ = ['convert_to_datetime', 'datetime', 'MINYEAR']
+
+
+class datetime(_datetime):
+    @staticmethod
+    def now():
+        return utcnow()
+
+    @staticmethod
+    def utcfromtimestamp(ts):
+        return _datetime.utcfromtimestamp(ts).replace(tzinfo=UTC)
+
+    @staticmethod
+    def parse(s):
+        dt = parse_datetime(s)
+        return dt.astimezone(UTC) if dt.tzinfo else dt.replace(tzinfo=UTC)
+
+
+def convert_to_datetime(dt):
+    if isinstance(dt, (int, float)):
+        return datetime.utcfromtimestamp(dt)
+
+    elif isinstance(dt, str):
+        return datetime.parse(dt)
+
+    elif isinstance(dt, _datetime):
+        args = (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
+        return datetime(*args, tzinfo=dt.tzinfo or UTC).astimezone(UTC)
+
+    else:
+        raise ValueError("Can't convert to datetime: %r" % dt)

gargantext/util/db.py

@@ -6,15 +6,11 @@ from gargantext.util.json import json_dumps
 # get engine, session, etc.
 ########################################################################
 from sqlalchemy.orm import sessionmaker, scoped_session
-from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import delete
 
 def get_engine():
     from sqlalchemy import create_engine
-    url = 'postgresql+psycopg2://{USER}:{PASSWORD}@{HOST}:{PORT}/{NAME}'.format(
-        **settings.DATABASES['default']
-    )
-    return create_engine( url
+    return create_engine( settings.DATABASES['default']['URL']
                         , use_native_hstore = True
                         , json_serializer = json_dumps
                         , pool_size=20, max_overflow=0
@@ -22,20 +18,8 @@ def get_engine():
 
 engine = get_engine()
 
-Base = declarative_base()
-
 session = scoped_session(sessionmaker(bind=engine))
 
-
-########################################################################
-# tools to build models
-########################################################################
-from sqlalchemy.types import *
-from sqlalchemy.schema import Column, ForeignKey, UniqueConstraint
-from sqlalchemy.dialects.postgresql import JSONB, DOUBLE_PRECISION
-from sqlalchemy.ext.mutable import MutableDict, MutableList
-Double = DOUBLE_PRECISION
-
 ########################################################################
 # useful for queries
 ########################################################################

gargantext/util/db_cache.py

@@ -29,6 +29,7 @@ class ModelCache(dict):
                 continue
         if formatted_key in self:
             self[key] = self[formatted_key]
+            element = self[key]
         else:
             element = session.query(self._model).filter(or_(*conditions)).first()
             if element is None:

gargantext/util/group_tools.py

@@ -7,7 +7,7 @@ from gargantext.util.db  import session, aliased
 from gargantext.models   import Ngram, NodeNgramNgram
 from igraph              import Graph  # for group_union
 
-def query_groups(groupings_id, details=False):
+def query_groups(groupings_id, details=False, sort=False):
     """
     Listing of couples (mainform,   subform)
                  aka   (ngram1_id, ngram2_id)
@@ -15,24 +15,27 @@ def query_groups(groupings_id, details=False):
     Parameter:
       - details: if False, just send the array of couples
                  if True, send quadruplets with (ngram1_id, term1, ngram2_id, term2)
+      - sort: order results by terms of ngram1 then ngram2
     """
+    if details or sort:
+        Ngram1, Ngram2 = Ngram, aliased(Ngram)
+
     if not details:
         # simple contents
-        query = session.query(NodeNgramNgram.ngram1_id, NodeNgramNgram.ngram2_id)
+        columns = (NodeNgramNgram.ngram1_id, NodeNgramNgram.ngram2_id)
     else:
         # detailed contents (id + terms)
-        Ngram1 = aliased(Ngram)
-        Ngram2 = aliased(Ngram)
-        query = (session
-                    .query(
-                        NodeNgramNgram.ngram1_id,
-                        Ngram1.terms,
-                        NodeNgramNgram.ngram2_id,
-                        Ngram2.terms,
-                     )
-                    .join(Ngram1, NodeNgramNgram.ngram1_id == Ngram1.id)
-                    .join(Ngram2, NodeNgramNgram.ngram2_id == Ngram2.id)
-                )
+        columns = (Ngram1.id, Ngram1.terms,
+                   Ngram2.id, Ngram2.terms)
+
+    query = session.query(*columns)
+
+    if details or sort:
+        query = (query.join(Ngram1, NodeNgramNgram.ngram1_id == Ngram1.id)
+                      .join(Ngram2, NodeNgramNgram.ngram2_id == Ngram2.id))
+
+    if sort:
+        query = query.order_by(Ngram1.terms, Ngram2.terms)
 
     # main filter
     # -----------

gargantext/util/http.py

@@ -73,7 +73,8 @@ from rest_framework.views import APIView
 from gargantext.util.json import json_encoder
 def JsonHttpResponse(data, status=200):
     return HttpResponse(
-        content      = json_encoder.encode(data),
+        content      = data.encode('utf-8') if isinstance(data, str) else \
+                       json_encoder.encode(data),
         content_type = 'application/json; charset=utf-8',
         status       = status
     )

gargantext/util/json.py

@@ -10,7 +10,7 @@ __all__ = ['json_encoder', 'json_dumps']
 
 class JSONEncoder(json.JSONEncoder):
     def default(self, obj):
-        from gargantext.util.db import Base
+        from gargantext.models import Base
         if isinstance(obj, Base):
             return {
                 key: value

gargantext/util/lists.py

@@ -50,6 +50,9 @@ class _BaseClass:
         else:
             return NotImplemented
 
+    def __len__(self):
+        return len(self.items)
+
     def __repr__(self):
         items = self.items
         if isinstance(items, defaultdict):

gargantext/util/parsers/CSV.py

@@ -4,128 +4,67 @@ import sys
 import csv
 csv.field_size_limit(sys.maxsize)
 import numpy as np
-import os
+
 
 class CSVParser(Parser):
+    DELIMITERS = ", \t;|:"
 
-    def CSVsample( self, small_contents , delim) :
-        reader = csv.reader(small_contents, delimiter=delim)
+    def detect_delimiter(self, lines, sample_size=10):
+        sample = lines[:sample_size]
 
-        Freqs = []
-        for row in reader:
-            Freqs.append(len(row))
+        # Compute frequency of each delimiter on each input line
+        delimiters_freqs = {
+            d: [line.count(d) for line in sample]
+            for d in self.DELIMITERS
+        }
 
-        return Freqs
+        # Select delimiters with a standard deviation of zero, ie. delimiters
+        # for which we have the same number of fields on each line
+        selected_delimiters = [
+            (d, np.sum(freqs))
+            for d, freqs in delimiters_freqs.items()
+            if any(freqs) and np.std(freqs) == 0
+        ]
 
+        if selected_delimiters:
+            # Choose the delimiter with highest frequency amongst selected ones
+            sorted_delimiters = sorted(selected_delimiters, key=lambda x: x[1])
+            return sorted_delimiters[-1][0]
 
     def parse(self, filebuf):
-
         print("CSV: parsing (assuming UTF-8 and LF line endings)")
 
         contents = filebuf.read().decode("UTF-8").split("\n")
 
-        sample_size = 10
-        sample_contents = contents[0:sample_size]
-
-        hyperdata_list = []
-
-        # # = = = = [ Getting delimiters frequency ] = = = = #
-        PossibleDelimiters = [ ',',' ','\t', ';', '|', ':' ]
-        AllDelimiters = {}
-        for delim in PossibleDelimiters:
-            AllDelimiters[delim] = self.CSVsample( sample_contents , delim )
-        # # = = = = [ / Getting delimiters frequency ] = = = = #
-        # # OUTPUT example:
-        # #  AllDelimiters = {
-        # #   '\t': [1, 1, 1, 1, 1],
-        # #   ' ': [1, 13, 261, 348, 330],
-        # #   ',': [15, 15, 15, 15, 15],
-        # #   ';': [1, 1, 1, 1, 1],
-        # #   '|': [1, 1, 1, 1, 1]
-        # #  }
-
-        # # = = = = [ Stand.Dev=0 & Sum of delimiters ] = = = = #
-        Delimiters = []
-        for d in AllDelimiters:
-            freqs = AllDelimiters[d]
-            suma = np.sum( freqs )
-            if suma >0:
-                std = np.std( freqs )
-                # print [ d , suma , len(freqs) , std]
-                if std == 0:
-                    Delimiters.append ( [ d , suma , len(freqs) , std] )
-        # # = = = = [ / Stand.Dev=0 & Sum of delimiters ] = = = = #
-        # # OUTPUT example:
-        # #  Delimiters = [
-        # #     ['\t', 5, 5, 0.0],
-        # #     [',', 75, 5, 0.0],
-        # #     ['|', 5, 5, 0.0]
-        # #  ]
-
-
-        # # = = = = [ Delimiter selection ] = = = = #
-        Sorted_Delims = sorted(Delimiters, key=lambda x: x[1], reverse=True)
-        HighestDelim = Sorted_Delims[0][0]
-        # HighestDelim = ","
-        print("CSV selected delimiter:",[HighestDelim])
-        # # = = = = [ / Delimiter selection ] = = = = #
-
-
-        # # = = = = [ First data coordinate ] = = = = #
-        Coords = {
-            "row": -1,
-            "column": -1
-        }
+        # Filter out empty lines
+        contents = [line for line in contents if line.strip()]
+
+        # Delimiter auto-detection
+        delimiter = self.detect_delimiter(contents, sample_size=10)
+
+        if delimiter is None:
+            raise ValueError("CSV: couldn't detect delimiter, bug or malformed data")
+
+        print("CSV: selected delimiter: %r" % delimiter)
+
+        # Parse CSV
+        reader = csv.reader(contents, delimiter=delimiter)
+
+        # Get first not empty row and its fields (ie. header row), or (0, [])
+        first_row, headers = \
+            next(((i, fields) for i, fields in enumerate(reader) if any(fields)),
+                 (0, []))
+
+        # Get first not empty column of the first row, or 0
+        first_col = next((i for i, field in enumerate(headers) if field), 0)
+
+        # Strip out potential empty fields in headers
+        headers = headers[first_col:]
 
-        reader = csv.reader(contents, delimiter=HighestDelim)
-
-        for rownum, tokens in enumerate(reader):
-            if rownum % 250 == 0:
-                print("CSV row: ", rownum)
-            joined_tokens = "".join (tokens)
-            if Coords["row"]<0 and len( joined_tokens )>0 :
-                Coords["row"] = rownum
-                for columnum in range(len(tokens)):
-                    t = tokens[columnum]
-                    if len(t)>0:
-                        Coords["column"] = columnum
-                        break
-        # # = = = = [ / First data coordinate ] = = = = #
-
-
-
-        # # = = = = [ Setting Headers ] = = = = #
-        Headers_Int2Str = {}
-        reader = csv.reader(contents, delimiter=HighestDelim)
-        for rownum, tokens in enumerate(reader):
-            if rownum>=Coords["row"]:
-                for columnum in range( Coords["column"],len(tokens) ):
-                    t = tokens[columnum]
-                    Headers_Int2Str[columnum] = t
-                break
-        # print("Headers_Int2Str")
-        # print(Headers_Int2Str)
-        # # = = = = [ / Setting Headers ] = = = = #
-        # # OUTPUT example:
-        # #  Headers_Int2Str = {
-        # #     0: 'publication_date',
-        # #      1: 'publication_month',
-        # #      2: 'publication_second',
-        # #      3: 'abstract'
-        # #  }
-
-
-        # # = = = = [ Reading the whole CSV and saving ] = = = = #
-        hyperdata_list = []
-        reader = csv.reader(contents, delimiter=HighestDelim)
-        for rownum, tokens in enumerate(reader):
-            if rownum>Coords["row"]:
-                RecordDict = {}
-                for columnum in range( Coords["column"],len(tokens) ):
-                    data = tokens[columnum]
-                    RecordDict[ Headers_Int2Str[columnum] ] = data
-                if len(RecordDict.keys())>0:
-                    hyperdata_list.append( RecordDict )
-        # # = = = = [ / Reading the whole CSV and saving ] = = = = #
-
-        return hyperdata_list
+        # Return a generator of dictionaries with column labels as keys,
+        # filtering out empty rows
+        for i, fields in enumerate(reader):
+            if i % 500 == 0:
+                print("CSV: parsing row #%s..." % (i+1))
+            if any(fields):
+                yield dict(zip(headers, fields[first_col:]))

gargantext/util/parsers/HAL.py

@@ -3,7 +3,7 @@
 # ****************************
 # ****  HAL Parser    ***
 # ****************************
-# CNRS COPYRIGHTS
+# CNRS COPYRIGHTS 2017
 # SEE LEGAL LICENCE OF GARGANTEXT.ORG
 
 from ._Parser import Parser
@@ -11,22 +11,15 @@ from datetime import datetime
 import json
 
 class HalParser(Parser):
-
-    def parse(self, filebuf):
-        '''
-        parse :: FileBuff -> [Hyperdata]
-        '''
-        contents = filebuf.read().decode("UTF-8")
-        data = json.loads(contents)
+    def _parse(self, json_docs):
         
-        filebuf.close()
-        
-        json_docs = data
         hyperdata_list = []
         
         hyperdata_path = { "id"              : "isbn_s"
                          , "title"           : "title_s"
                          , "abstract"        : "abstract_s"
+                         , "title"           : "en_title_s"
+                         , "abstract"        : "en_abstract_s"
                          , "source"          : "journalTitle_s"
                          , "url"             : "uri_s"
                          , "authors"         : "authFullName_s"
@@ -39,6 +32,7 @@ class HalParser(Parser):
                          , "deptStructId_i"  : "deptStructId_i"
                          , "labStructId_i"   : "labStructId_i"
                          , "rteamStructId_i" : "rteamStructId_i" 
+                         , "docType_s"       : "docType_s" 
                          }
 
         uris = set()
@@ -82,3 +76,13 @@ class HalParser(Parser):
                 hyperdata_list.append(hyperdata)
         
         return hyperdata_list
+
+    def parse(self, filebuf):
+        '''
+        parse :: FileBuff -> [Hyperdata]
+        '''
+        contents = filebuf.read().decode("UTF-8")
+        data = json.loads(contents)
+
+        return self._parse(data)
+

gargantext/util/parsers/ISIDORE.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# ****************************
+# ****  ISIDORE Parser    ***
+# ****************************
+# CNRS COPYRIGHTS
+# SEE LEGAL LICENCE OF GARGANTEXT.ORG
+
+from ._Parser import Parser
+from datetime import datetime
+import json
+
+class IsidoreParser(Parser):
+
+    def parse(self, filebuf):
+        '''
+        parse :: FileBuff -> [Hyperdata]
+        '''
+        contents = filebuf.read().decode("UTF-8")
+        data = json.loads(contents)
+        
+        filebuf.close()
+        
+        json_docs = data
+        hyperdata_list = []
+        
+        hyperdata_path = { "title"    : "title"
+                         , "abstract" : "abstract"
+                         , "authors"  : "authors"
+                         , "url"      : "url"
+                         , "source"   : "source"
+                         }
+        
+        uniq_id = set()
+
+        for doc in json_docs:
+
+            hyperdata = {}
+            
+            for key, path in hyperdata_path.items():
+                    hyperdata[key] = doc.get(path, "")
+            
+            if hyperdata["url"] not in uniq_id:
+                # Removing the duplicates implicitly
+                uniq_id.add(hyperdata["url"])
+                
+                # Source is the Journal Name 
+                hyperdata["source"] = doc.get("source", "ISIDORE Database")
+                
+                # Working on the date
+                maybeDate = doc.get("date"  , None)
+
+                if maybeDate is None:
+                    date = datetime.now()
+                else:
+                    try :
+                        # Model of date: 1958-01-01T00:00:00
+                        date = datetime.strptime(maybeDate, '%Y-%m-%dT%H:%M:%S')
+                    except :
+                        print("FIX DATE ISIDORE please >%s<" % maybeDate)
+                        date = datetime.now()
+
+                hyperdata["publication_date"] = date
+                hyperdata["publication_year"]  = str(date.year)
+                hyperdata["publication_month"] = str(date.month)
+                hyperdata["publication_day"]   = str(date.day)
+                
+                hyperdata_list.append(hyperdata)
+        
+        return hyperdata_list

gargantext/util/parsers/MULTIVAC.py

@@ -18,30 +18,30 @@ class MultivacParser(Parser):
         '''
         contents = filebuf.read().decode("UTF-8")
         data = json.loads(contents)
-        
+
         filebuf.close()
-        
+
         json_docs = data
         hyperdata_list = []
-        
+
         hyperdata_path = { "id"       : "id"
                          , "title"    : "title"
                          , "abstract" : "abstract"
                          , "type"     : "type"
                          }
-        
+
         for json_doc in json_docs:
 
             hyperdata = {}
-            
+
             doc = json_doc["_source"]
 
             for key, path in hyperdata_path.items():
                     hyperdata[key] = doc.get(path, "")
-            
+
             hyperdata["source"] = doc.get("serial"      , {})\
                                      .get("journaltitle", "REPEC Database")
-            
+
             try:
                 hyperdata["url"]    = doc.get("file", {})\
                                          .get("url" , "")
@@ -51,15 +51,15 @@ class MultivacParser(Parser):
             hyperdata["authors"] = ", ".join(
                                              [ p.get("person", {})
                                                 .get("name"  , "")
-                          
+
                                                for p in doc.get("hasauthor", [])
                                              ]
                                             )
-            
+
 
             year = doc.get("serial"  , {})\
                       .get("issuedate", None)
-            
+
             if year == "Invalide date":
                 year = doc.get("issuedate"  , None)
 
@@ -73,10 +73,7 @@ class MultivacParser(Parser):
                     date = datetime.now()
 
             hyperdata["publication_date"] = date
-            hyperdata["publication_year"]  = str(date.year)
-            hyperdata["publication_month"] = str(date.month)
-            hyperdata["publication_day"]   = str(date.day)
-            
+
             hyperdata_list.append(hyperdata)
-        
+
         return hyperdata_list

gargantext/util/parsers/_Parser.py

-import datetime
 import dateutil.parser
 import zipfile
 import re
 import dateparser as date_parser
 from gargantext.util.languages import languages
+from gargantext.util import datetime, convert_to_datetime, MINYEAR
 
 
-DEFAULT_DATE = datetime.datetime(datetime.MINYEAR, 1, 1)
+DEFAULT_DATE = datetime(MINYEAR, 1, 1)
 
 
 class Parser:
@@ -34,29 +34,29 @@ class Parser:
     def format_hyperdata_dates(self, hyperdata):
         """Format the dates found in the hyperdata.
         Examples:
-            {"publication_date": "2014-10-23 09:57:42"}
-            -> {"publication_date": "2014-10-23 09:57:42", "publication_year": "2014", ...}
+            {"publication_date": "2014-10-23 09:57:42+00:00"}
+            -> {"publication_date": "2014-10-23 09:57:42+00:00", "publication_year": "2014", ...}
             {"publication_year": "2014"}
-            -> {"publication_date": "2014-01-01 00:00:00", "publication_year": "2014", ...}
+            -> {"publication_date": "2014-01-01 00:00:00+00:00", "publication_year": "2014", ...}
         """
 
         # First, check the split dates...
         # This part mainly deal with Zotero data but can be usefull for others
         # parts
-        date_string = hyperdata.get('publication_date_to_parse', None)
+        date_string = hyperdata.get('publication_date_to_parse')
         if date_string is not None:
             date_string = re.sub(r'\/\/+(\w*|\d*)', '', date_string)
             try:
-                hyperdata['publication' + "_date"] = dateutil.parser.parse(
+                hyperdata['publication_date'] = dateutil.parser.parse(
                     date_string,
                     default=DEFAULT_DATE
-                ).strftime("%Y-%m-%d %H:%M:%S")
+                )
             except Exception as error:
                 print(error, 'Date not parsed for:', date_string)
-                hyperdata['publication_date'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                hyperdata['publication_date'] = datetime.now()
 
 
-        elif hyperdata.get('publication_year', None) is not None:
+        elif hyperdata.get('publication_year') is not None:
 
             prefixes = [key[:-5] for key in hyperdata.keys() if key[-5:] == "_year"]
             # eg prefixes : ['publication']
@@ -64,56 +64,45 @@ class Parser:
             for prefix in prefixes:
                 date_string = hyperdata[prefix + "_year"]
 
-                # FIXME: except for year is it necessary to test that key exists
-                #        when we have a default value in .get(key, "01") ??
-                key = prefix + "_month"
-                if key in hyperdata:
-                    date_string += " " + hyperdata.get(key, "01")
-                    key = prefix + "_day"
-                    if key in hyperdata:
-                        date_string += " " + hyperdata.get(key, "01")
-                        key = prefix + "_hour"
-                        if key in hyperdata:
-                            date_string += " " + hyperdata.get(key, "01")
-                            key = prefix + "_minute"
-                            if key in hyperdata:
-                                date_string += ":" + hyperdata.get(key, "01")
-                                key = prefix + "_second"
-                                if key in hyperdata:
-                                    date_string += ":" + hyperdata.get(key, "01")
+                for part in ('month', 'day', 'hour', 'minute', 'second'):
+                    key = prefix + '_' + part
+                    if key not in hyperdata:
+                        break
+
+                    sep = ":" if key in ('minute', 'second') else " "
+                    date_string += sep + hyperdata.get(key, '01')
+
                 try:
-                    hyperdata[prefix + "_date"] = dateutil.parser.parse(date_string).strftime("%Y-%m-%d %H:%M:%S")
+                    hyperdata[prefix + "_date"] = dateutil.parser.parse(date_string)
                 except Exception as error:
                     try:
                         print("_Parser: error in full date parse", error, date_string)
                         # Date format:  1994 NOV-DEC
-                        hyperdata[prefix + "_date"] = date_parser.parse(str(date_string)[:8]).strftime("%Y-%m-%d %H:%M:%S")
+                        hyperdata[prefix + "_date"] = date_parser.parse(str(date_string)[:8])
 
                     except Exception as error:
                         try:
                             print("_Parser: error in short date parse", error)
                             # FIXME Date format:  1994 SPR
                             # By default, we take the year only
-                            hyperdata[prefix + "_date"] = date_parser.parse(str(date_string)[:4]).strftime("%Y-%m-%d %H:%M:%S")
+                            hyperdata[prefix + "_date"] = date_parser.parse(str(date_string)[:4])
 
                         except Exception as error:
                             print("_Parser:", error)
         else:
             print("WARNING: Date unknown at _Parser level, using now()")
-            hyperdata['publication_date'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+            hyperdata['publication_date'] = datetime.now()
 
         # ...then parse all the "date" fields, to parse it into separate elements
         prefixes = [key[:-5] for key in hyperdata.keys() if key[-5:] == "_date"]
         for prefix in prefixes:
-            date = dateutil.parser.parse(hyperdata[prefix + "_date"])
-            #print(date)
-
-            hyperdata[prefix + "_year"]      = date.strftime("%Y")
-            hyperdata[prefix + "_month"]     = date.strftime("%m")
-            hyperdata[prefix + "_day"]       = date.strftime("%d")
-            hyperdata[prefix + "_hour"]      = date.strftime("%H")
-            hyperdata[prefix + "_minute"]    = date.strftime("%M")
-            hyperdata[prefix + "_second"]    = date.strftime("%S")
+            name = prefix + "_date"
+            date = hyperdata[name]
+            hyperdata[name] = str(convert_to_datetime(date))
+
+            for part in ('year', 'month', 'day', 'hour', 'minute', 'second'):
+                hyperdata[prefix + '_' + part] = getattr(date, part)
+
         # print("line 116", hyperdata['publication_date'])
         # finally, return the transformed result!
         return hyperdata

gargantext/util/show_nodes.py

+# Make this a standalone script...
+# Can be called this way: python3 gargantext/util/show_nodes.py
+
+import os
+import django
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'gargantext.settings')
+django.setup()
+
+# ...End of jiberish.
+
+import itertools
+import colorama
+from colorama import Fore
+from sqlalchemy.sql.expression import literal_column
+
+from gargantext.util.db import session, func, aliased
+from gargantext.models import Node
+
+
+NODE_BULLET = '‣'
+
+# https://en.wikipedia.org/wiki/Box-drawing_character
+TREE_ROOT = '╾'
+TREE_VERT = '│'
+TREE_HORI = '─'
+TREE_FORK = '├'
+TREE_CORN = '└'
+
+FIRST  = 0x01
+LAST   = 0x02
+
+
+def nodes(parent=None, group_by='typename', order_by='typename', has_child='check'):
+    if group_by or has_child is not None:
+        select = [func.min(Node.id).label('id'),
+                  func.min(Node.name).label('name'),
+                  func.min(Node.typename).label('typename'),
+                  func.count(Node.id).label('cnt')]
+    else:
+        select = [Node.id.label('id'),
+                  Node.name.label('name'),
+                  Node.typename.label('typename'),
+                  literal_column('1').label('cnt')]
+
+    if has_child is not None:
+        N = aliased(Node)
+        select.append(func.count(N.id).label('children'))
+    else:
+        select.append(literal_column('NULL').label('children'))
+
+    parent_id = getattr(parent, 'id', parent)
+    q = session.query(*select).filter_by(parent_id=parent_id) \
+               .group_by(getattr(Node, group_by if group_by else 'id'))
+
+    if has_child is not None:
+        q = q.outerjoin(N, N.parent_id == Node.id).group_by(N.parent_id)
+
+    return q.order_by(order_by)
+
+
+def node_show(node, prefix='', maxlen=60):
+    if node.children > 0 or node.cnt == 1:
+        name = node.name[:maxlen] + '..' if len(node.name) > maxlen else node.name
+        label = Fore.CYAN + name + Fore.RESET
+    else:
+        label = Fore.MAGENTA + str(node.cnt) + Fore.RESET
+
+    print(prefix, '%s%s %s' % (Fore.GREEN, node.typename, label), sep='')
+
+
+def tree_show(node, pos=FIRST|LAST, level=0, prefix='', maxlen=60, compact=True):
+    #print('%02d %x' % (level, pos), end='')
+
+    branch = TREE_ROOT if pos&FIRST and level == 0 else TREE_FORK if not pos&LAST else TREE_CORN
+    node_prefix = prefix + branch + 2*TREE_HORI + ' '
+    node_show(node, node_prefix, maxlen)
+
+    childs = iter(nodes(parent=node, group_by=compact and 'typename'))
+
+    try:
+        node = next(childs)
+    except StopIteration:
+        return
+
+    prefix = prefix + (' ' if pos&LAST else TREE_VERT) + '   '
+
+    for i, next_node in enumerate(itertools.chain(childs, [None])):
+        pos = (FIRST if i == 0 else 0) | (LAST if next_node is None else 0)
+        tree_show(node, pos, level + 1, prefix, maxlen, compact)
+        node = next_node
+
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) == 1:
+        compact = True
+    elif len(sys.argv) == 2 and sys.argv[1] in ('-a', '--all'):
+        compact = False
+    else:
+        print("Usage: %s [-a|--all]" % sys.argv[0], file=sys.stderr)
+        sys.exit(1)
+
+    colorama.init(strip=False)
+    for root in nodes():
+        tree_show(root, compact=compact)

gargantext/util/toolchain/hyperdata_indexing.py

@@ -43,8 +43,7 @@ def _nodes_hyperdata_generator(corpus):
                             key['id'],
                             None,
                             None,
-                            value.strftime("%Y-%m-%d %H:%M:%S"), 
-                            # FIXME check timestamp +%Z
+                            str(value),
                             None,
                             None,
                         )

gargantext/util/toolchain/list_map.py

@@ -12,15 +12,19 @@ from gargantext.constants     import DEFAULT_MAPLIST_MAX,\
                                      DEFAULT_MAPLIST_GENCLUSION_RATIO,\
                                      DEFAULT_MAPLIST_MONOGRAMS_RATIO
 
+def do_maplist_query():
+    return None
+
+
 def do_maplist(corpus,
-               overwrite_id = None,
-               mainlist_id  = None,
-               specclusion_id = None,
-               genclusion_id = None,
-               grouplist_id = None,
-               limit=DEFAULT_MAPLIST_MAX,
-               genclusion_part=DEFAULT_MAPLIST_GENCLUSION_RATIO,
-               monograms_part=DEFAULT_MAPLIST_MONOGRAMS_RATIO
+               overwrite_id    = None,
+               mainlist_id     = None,
+               specclusion_id  = None,
+               genclusion_id   = None,
+               grouplist_id    = None,
+               limit           = DEFAULT_MAPLIST_MAX,
+               genclusion_part = DEFAULT_MAPLIST_GENCLUSION_RATIO,
+               monograms_part  = DEFAULT_MAPLIST_MONOGRAMS_RATIO
                ):
     '''
     According to Genericity/Specificity and mainlist
@@ -28,9 +32,9 @@ def do_maplist(corpus,
     Parameters:
       - mainlist_id (starting point, already cleaned of stoplist terms)
       - specclusion_id (ngram inclusion by cooc specificity -- ranking factor)
-      - genclusion_id (ngram inclusion by cooc genericity -- ranking factor)
+      - genclusion_id  (ngram inclusion by cooc genericity  -- ranking factor)
       - grouplist_id (filtering grouped ones)
-      - overwrite_id: optional if preexisting MAPLIST node to overwrite
+      - overwrite_id: optional. Overwrite if preexisting MAPLIST node
 
       + 3 params to modulate the terms choice
         - limit for the amount of picked terms
@@ -77,6 +81,7 @@ def do_maplist(corpus,
                         )
                 .join(Ngram, Ngram.id == ScoreSpec.ngram_id)
                 .join(ScoreGen, ScoreGen.ngram_id == ScoreSpec.ngram_id)
+                
                 .filter(ScoreSpec.node_id == specclusion_id)
                 .filter(ScoreGen.node_id == genclusion_id)
 
@@ -155,10 +160,10 @@ def do_maplist(corpus,
         # at the end of the first loop we just need to sort all by the second ranker (gen)
         scored_ngrams = sorted(scored_ngrams, key=lambda ng_infos: ng_infos[2], reverse=True)
 
-    obtained_spec_mono = len(chosen_ngrams['topspec']['monograms'])
+    obtained_spec_mono  = len(chosen_ngrams['topspec']['monograms'])
     obtained_spec_multi = len(chosen_ngrams['topspec']['multigrams'])
-    obtained_gen_mono = len(chosen_ngrams['topgen']['monograms'])
-    obtained_gen_multi = len(chosen_ngrams['topgen']['multigrams'])
+    obtained_gen_mono   = len(chosen_ngrams['topgen']['monograms'])
+    obtained_gen_multi  = len(chosen_ngrams['topgen']['multigrams'])
     obtained_total = obtained_spec_mono   \
                     + obtained_spec_multi \
                     + obtained_gen_mono   \

gargantext/util/toolchain/main.py

@@ -185,7 +185,7 @@ def parse_extract_indexhyperdata(corpus):
     session.commit()
 
 @shared_task
-def recount(corpus):
+def recount(corpus_id):
     """
     Recount essential metrics of the toolchain after group modifications.
 
@@ -203,6 +203,7 @@ def recount(corpus):
 
     NB: no new extraction, no list change, just the metrics
     """
+    corpus = session.query(Node).filter(Node.id==corpus_id).first()
     # 1) we'll need the new groups and mainlist as basis
     group_id = corpus.children("GROUPLIST").first().id
     mainlist_id = corpus.children("MAINLIST").first().id

gargantext/util/toolchain/ngram_coocs.py

@@ -9,7 +9,6 @@ from gargantext.util.db        import get_engine
 from gargantext.util.db_cache  import cache
 from gargantext.constants      import DEFAULT_COOC_THRESHOLD, NODETYPES
 from gargantext.constants      import INDEXED_HYPERDATA
-from gargantext.util.tools     import datetime, convert_to_date
 
 def compute_coocs(  corpus,
                     overwrite_id    = None,
@@ -95,7 +94,7 @@ def compute_coocs(  corpus,
 
     # 2b) stating the filters
     cooc_filter_sql = """
-        WHERE 
+        WHERE
             n.typename  = {nodetype_id}
         AND n.parent_id = {corpus_id}
         GROUP BY 1,2
@@ -105,7 +104,7 @@ def compute_coocs(  corpus,
         """.format( nodetype_id = NODETYPES.index('DOCUMENT')
                   , corpus_id=corpus.id
                   )
-    
+
     # 3) taking the cooccurrences of ngram x2
     ngram_filter_A_sql += """
         -- STEP 1: X axis of the matrix
@@ -162,25 +161,25 @@ def compute_coocs(  corpus,
     # 4) prepare the synonyms
     if groupings_id:
         ngram_filter_A_sql += """
-        LEFT JOIN  nodes_ngrams_ngrams 
-               AS grA  ON wlA.ngram_id = grA.ngram1_id 
+        LEFT JOIN  nodes_ngrams_ngrams
+               AS grA  ON wlA.ngram_id = grA.ngram1_id
                       AND grA.node_id  = {groupings_id}
         -- \--> adding (joining) ngrams that are grouped
         LEFT JOIN  nodes_ngrams
                AS wlAA ON grA.ngram2_id = wlAA.ngram_id
-                      AND wlAA.node_id  = wlA.node_id 
+                      AND wlAA.node_id  = wlA.node_id
         -- \--> adding (joining) ngrams that are not grouped
         --LEFT JOIN  ngrams        AS wlAA ON grA.ngram2_id = wlAA.id
         -- \--> for joining all synonyms even if they are not in the main list (white list)
 
         """.format(groupings_id = groupings_id)
-        
+
         ngram_filter_B_sql += """
         LEFT JOIN  nodes_ngrams_ngrams
-               AS grB  ON wlB.ngram_id = grB.ngram1_id 
+               AS grB  ON wlB.ngram_id = grB.ngram1_id
                       AND grB.node_id  = {groupings_id}
         -- \--> adding (joining) ngrams that are grouped
-        LEFT JOIN  nodes_ngrams 
+        LEFT JOIN  nodes_ngrams
                AS wlBB ON grB.ngram2_id = wlBB.ngram_id
                       AND wlBB.node_id   = wlB.node_id
         -- \--> adding (joining) ngrams that are not grouped

gargantext/util/toolchain/ngrams_extraction.py

@@ -81,44 +81,45 @@ def extract_ngrams(corpus, keys=DEFAULT_INDEX_FIELDS, do_subngrams = DEFAULT_IND
                     corpus.hyperdata["skipped_docs"].append(document.id)
                     corpus.save_hyperdata()
                     continue
-                else:
-                    # ready !
-                    tagger = tagger_bots[language_iso2]
-
-                    # to do verify if document has no KEYS to index
-                    # eg: use set intersect (+ loop becomes direct! with no continue)
-                    for key in keys:
-                        try:
-                            value = document.hyperdata[str(key)]
-                            if not isinstance(value, str):
-                                #print("DBG wrong content in doc for key", key)
-                                continue
-                                # get ngrams
-                            for ngram in tagger.extract(value):
-                                tokens = tuple(normalize_forms(token[0]) for token in ngram)
-                                if do_subngrams:
-                                    # ex tokens = ["very", "cool", "exemple"]
-                                    #    subterms = [['very', 'cool'],...]
-
-                                    subterms = subsequences(tokens)
-                                else:
-                                    subterms = [tokens]
-
-                                for seqterm in subterms:
-                                    ngram = ' '.join(seqterm)
-                                    nbwords = len(seqterm)
-                                    nbchars = len(ngram)
-                                    if nbchars > 1:
-                                        if nbchars > 255:
-                                            # max ngram length (DB constraint)
-                                            ngram = ngram[:255]
-                                        # doc <=> ngram index
-                                        nodes_ngrams_count[(document.id, ngram)] += 1
-                                        # add fields :   terms          n
-                                        ngrams_data.add((ngram, nbwords, ))
-                        except:
-                            #value not in doc
+
+                # ready !
+                tagger = tagger_bots[language_iso2]
+
+                # to do verify if document has no KEYS to index
+                # eg: use set intersect (+ loop becomes direct! with no continue)
+                for key in keys:
+                    try:
+                        value = document.hyperdata[str(key)]
+                        if not isinstance(value, str):
+                            #print("DBG wrong content in doc for key", key)
                             continue
+                            # get ngrams
+                        for ngram in tagger.extract(value):
+                            normal_forms = (normalize_forms(t[0]) for t in ngram)
+                            tokens = tuple(nf for nf in normal_forms if nf)
+                            if do_subngrams:
+                                # ex tokens = ["very", "cool", "exemple"]
+                                #    subterms = [['very', 'cool'],...]
+
+                                subterms = subsequences(tokens)
+                            else:
+                                subterms = [tokens]
+
+                            for seqterm in subterms:
+                                ngram = ' '.join(seqterm)
+                                nbwords = len(seqterm)
+                                nbchars = len(ngram)
+                                if nbchars > 1:
+                                    if nbchars > 255:
+                                        # max ngram length (DB constraint)
+                                        ngram = ngram[:255]
+                                    # doc <=> ngram index
+                                    nodes_ngrams_count[(document.id, ngram)] += 1
+                                    # add fields :   terms          n
+                                    ngrams_data.add((ngram, nbwords, ))
+                    except:
+                        #value not in doc
+                        continue
 
             # integrate ngrams and nodes-ngrams
             if len(nodes_ngrams_count) >= BATCH_NGRAMSEXTRACTION_SIZE:

gargantext/util/toolchain/parsing.py

@@ -175,7 +175,6 @@ def parse(corpus):
                         hyperdata = hyperdata,
                     )
                     session.add(document)
-                    session.commit()
                     documents_count += 1
 
                     if pending_add_error_stats:
@@ -190,6 +189,9 @@ def parse(corpus):
                         session.add(corpus)
                         session.commit()
 
+                # Commit any pending document
+                session.commit()
+
                 # update info about the resource
                 resource['extracted'] = True
                 #print( "resource n°",i, ":", d, "docs inside this file")

gargantext/util/tools.py

-import os
-from gargantext.settings import MEDIA_ROOT
-
-import datetime
-import dateutil
-
-def convert_to_date(date):
-    if isinstance(date, (int, float)):
-        return datetime.datetime.timestamp(date)
-    else:
-        return dateutil.parser.parse(date)
-
-
-def ensure_dir(user):
-    '''
-    If user is new, folder does not exist yet, create it then
-    '''
-    dirpath = '%s/corpora/%s' % (MEDIA_ROOT, user.username)
-    if not os.path.exists(dirpath):
-        print("Creating folder %s" % dirpath)
-        os.makedirs(dirpath)

gargantext/views/api/metrics.py

@@ -2,6 +2,7 @@ from gargantext.util.db_cache   import cache
 from gargantext.util.http       import ValidationException, APIView \
                                      , HttpResponse, JsonHttpResponse
 from gargantext.util.toolchain.main import recount
+from gargantext.util.scheduling import scheduled
 from datetime                   import datetime
 
 class CorpusMetrics(APIView):
@@ -32,7 +33,7 @@ class CorpusMetrics(APIView):
         else:
             t_before = datetime.now()
             # =============
-            recount(corpus)
+            scheduled(recount)(corpus.id)
             # =============
             t_after = datetime.now()
 

gargantext/views/api/ngramlists.py

@@ -90,10 +90,11 @@ class CSVLists(APIView):
         # import the csv
         # try:
         log_msg = "Async generation"
-        
+
         corpus_node_id = corpus_node.id
-        scheduled(import_and_merge_ngramlists)(csv_contents, corpus_node_id)
-        
+        scheduled(import_and_merge_ngramlists)(csv_contents, corpus_node_id,
+                                               overwrite=bool(params.get('overwrite')))
+
         return JsonHttpResponse({
             'log': log_msg,
             }, 200)
@@ -153,7 +154,13 @@ class CSVLists(APIView):
         # attempt to merge and send response
         try:
             # merge the source_lists onto those of the target corpus
-            log_msg = merge_ngramlists(source_lists, onto_corpus=corpus_node)
+            delete = todo_lists if bool(params.get('overwrite')) else []
+
+            if len(delete) == len(list_types):
+                delete.append('groupings')
+
+            log_msg = merge_ngramlists(source_lists, onto_corpus=corpus_node, del_originals=delete)
+
             return JsonHttpResponse({
                 'log': log_msg,
                 }, 200)

gargantext_notebook.py

+install/notebook/gargantext_notebook.py
\ No newline at end of file

install/notebook.install

+#!/bin/bash
+
+sudo adduser --disabled-password --gecos "" notebooks
+sudo docker rm $(sudo docker ps -a | grep sh | awk '{print $1}')
+sudo docker build -t garg-notebook:latest ./notebook