Merge branch 'testing-notebook' into stable-notebook

CHANGELOG.md

@@ -2,6 +2,8 @@
 * Guided Tour
 * Sources form highlighting crawlers
 
+## Version 3.0.7
+* Alembic implemented to manage database migrations
 
 ## Version 3.0.6.8
 * REPEC Crawler (connection with https://multivac.iscpif.fr)

alembic.ini

+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# timezone to use when rendering the date
+# within the migration file as well as the filename.
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat alembic/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+# XXX For database access configuration, see alembic/env.py
+#sqlalchemy.url = driver://user:pass@localhost/dbname
+
+[alembic:exclude]
+tables = django_* celery_* djcelery_* auth_*
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

alembic/README

+Alembic must be installed in the virtualenv in order to use right python paths,
+so it's installed with pip. Commands described in this little documentation
+must be executed from gargantext root directory, ie. /srv/gargantext.
+
+Keep in mind that Alembic only handles SQLAlchemy models: tables created from
+Django ORM must be put out of Alembic sight. See [alembic:exclude] section in
+alembic.ini.
+
+
+TELL ALEMBIC TO NOT START FROM SCRATCH
+
+    # To upgrade a database populated before Alembic usage in Gargantext,
+    # don't forget to tell Alembic your current version before to run
+    # "upgrade head" command. If you don't want to do this, you can of course
+    # drop your database and really start from scratch.
+
+    alembic stamp bedce47c9e34
+
+
+UPGRADE TO LATEST DATABASE VERSION
+
+    alembic upgrade head
+
+
+DOWNGRADE TO INITIAL DATABASE STATE
+
+    # /!\ RUNNING THIS COMMAND WILL CAUSE ALL DATA LOST WITHOUT ASKING !!
+
+    alembic downgrade base
+
+
+GENERATE A NEW REVISION
+
+    alembic revision -m "Message for this migration"
+
+    # A migration script is then created in alembic/versions directory. For
+    # example alembic/versions/3adcc9a56557_message_for_this_migration.py
+    # where 3adcc9a56557 is the revision id generated by Alembic.
+    #
+    # This script must be edited to write the migration itself, mainly
+    # in `upgrade` and `downgrade` functions. See Alembic documentation for
+    # further details.
+
+
+GENERATE A REVISION FROM CURRENT STATE
+
+    alembic revision --autogenerate -m "Message for this migration"
+
+    # Alembic should generate a script reflecting changes already made in
+    # database. However it is always a good idea to check it and edit it
+    # manually, Alembic is not always accurate and can't see all alterations.
+    # It should work with basic changes such as model or column creation. See
+    # http://alembic.zzzcomputing.com/en/latest/autogenerate.html#what-does-autogenerate-detect-and-what-does-it-not-detect

alembic/env.py

+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+import re
+
+# Add projet root directory in path and setup Django...
+import os
+import django
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'gargantext.settings')
+django.setup()
+
+# ...to be able to import gargantext.
+
+from gargantext import settings, models
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+config.set_main_option("sqlalchemy.url", settings.DATABASES['default']['URL'])
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = models.Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+# Inspired from https://gist.github.com/utek/6163250
+def exclude_tables_from_config(config):
+    tables = config.get("tables", '').replace('*', '.*').split(' ')
+    pattern = '|'.join(tables)
+    return re.compile(pattern)
+
+exclude_tables = exclude_tables_from_config(config.get_section('alembic:exclude'))
+
+
+def include_object(obj, name, typ, reflected, compare_to):
+    if typ == "table" and exclude_tables.match(name):
+        return False
+    else:
+        return True
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url, target_metadata=target_metadata, literal_binds=True,
+        include_object=include_object)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool)
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            include_object=include_object
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import gargantext
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

alembic/versions/08230100f262_put_a_timezone_on_node_date.py

+"""Put a timezone on Node.date
+
+Revision ID: 08230100f262
+Revises: 601e9d9baa4c
+Create Date: 2017-07-06 13:47:10.788569
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import gargantext
+
+
+# revision identifiers, used by Alembic.
+revision = '08230100f262'
+down_revision = '601e9d9baa4c'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.alter_column('nodes', 'date', type_=sa.DateTime(timezone=True))
+
+
+def downgrade():
+    op.alter_column('nodes', 'date', type_=sa.DateTime(timezone=False))

alembic/versions/601e9d9baa4c_add_occ_hist_occ_hist_part_functions.py

+"""Add OCC_HIST & OCC_HIST_PART functions
+
+Revision ID: 601e9d9baa4c
+Revises: 932dbf3e8c43
+Create Date: 2017-07-06 10:52:16.161118
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from gargantext.util.alembic import ReplaceableObject
+
+
+# revision identifiers, used by Alembic.
+revision = '601e9d9baa4c'
+down_revision = '932dbf3e8c43'
+branch_labels = None
+depends_on = None
+
+
+# -- OCC_HIST_PART :: Corpus.id -> GroupList.id -> Start -> End
+
+occ_hist_part = ReplaceableObject(
+    "OCC_HIST_PART(int, int, timestamp, timestamp)",
+    """
+RETURNS TABLE (ng_id int, score float8)
+    AS $$
+-- EXPLAIN ANALYZE
+    SELECT
+    COALESCE(gr.ngram1_id, ng1.ngram_id) as ng_id,
+    SUM(ng1.weight) as score
+
+    from nodes n
+
+    -- BEFORE
+    INNER JOIN nodes as n1 ON n1.id = n.id
+
+    INNER JOIN nodes_ngrams ng1 ON ng1.node_id = n1.id
+
+    -- Limit with timestamps: ]start, end]
+    INNER JOIN nodes_hyperdata nh1 ON nh1.node_id = n1.id
+                                  AND nh1.value_utc >  $3
+                                  AND nh1.value_utc <= $4
+
+    -- Group List
+    LEFT JOIN  nodes_ngrams_ngrams gr ON ng1.ngram_id = gr.ngram2_id
+                               AND gr.node_id = $2
+
+    WHERE
+        n.typename  = 4
+    AND n.parent_id = $1
+    GROUP BY 1
+    $$
+LANGUAGE SQL;
+    """
+)
+
+# -- OCC_HIST :: Corpus.id -> GroupList.id -> MapList.id -> Start -> EndFirst -> EndLast
+# -- EXEMPLE USAGE
+# --    SELECT * FROM OCC_HIST(182856, 183859, 183866, '1800-03-15 17:00:00+01', '2000-03-15 17:00:00+01', '2017-03-15 17:00:00+01')
+
+occ_hist = ReplaceableObject(
+    "OCC_HIST(int, int, int, timestamp, timestamp, timestamp)",
+    """
+RETURNS TABLE (ng_id int, score numeric)
+    AS $$
+    WITH OCC1 as (SELECT * from OCC_HIST_PART($1, $2, $4, $5))
+       , OCC2 as (SELECT * from OCC_HIST_PART($1, $2, $5, $6))
+       , GROWTH as (SELECT ml.ngram_id as ngram_id
+                 , COALESCE(OCC1.score, null) as score1
+                 , COALESCE(OCC2.score, null) as score2
+                    FROM nodes_ngrams ml
+                        LEFT JOIN OCC1 ON OCC1.ng_id = ml.ngram_id
+                        LEFT JOIN OCC2 ON OCC2.ng_id = ml.ngram_id
+                    WHERE ml.node_id = $3
+                    ORDER by score2 DESC)
+    SELECT ngram_id, COALESCE(ROUND(CAST((100 * (score2 - score1) / COALESCE((score2 + score1), 1)) as numeric), 2), 0) from GROWTH
+    $$
+LANGUAGE SQL;
+    """
+)
+
+# -- BEHAVIORAL TEST (should be equal to occ in terms table)
+# --    WITH OCC as (SELECT * from OCC_HIST(182856, 183859, '1800-03-15 17:00:00+01', '2300-03-15 17:00:00+01'))
+# --    SELECT ng_id, score from OCC
+# --            INNER JOIN nodes_ngrams ml on ml.ngram_id = ng_id
+# --                                      AND ml.node_id = 183866
+# --            ORDER BY score DESC;
+
+def upgrade():
+    op.create_sp(occ_hist_part)
+    op.create_sp(occ_hist)
+
+
+def downgrade():
+    op.drop_sp(occ_hist)
+    op.drop_sp(occ_hist_part)

alembic/versions/932dbf3e8c43_initial_migration.py

+"""Initial migration
+
+Revision ID: 932dbf3e8c43
+Revises: 
+Create Date: 2017-07-05 16:41:23.951422
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import gargantext
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '932dbf3e8c43'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('contacts',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('user1_id', sa.Integer(), nullable=True),
+    sa.Column('user2_id', sa.Integer(), nullable=True),
+    sa.Column('is_blocked', sa.Boolean(), nullable=True),
+    sa.Column('date_creation', sa.DateTime(), nullable=True),
+    sa.ForeignKeyConstraint(['user1_id'], ['auth_user.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['user2_id'], ['auth_user.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('user1_id', 'user2_id')
+    )
+    op.create_table('ngrams',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('terms', sa.String(length=255), nullable=True),
+    sa.Column('n', sa.Integer(), nullable=True),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('terms')
+    )
+    op.create_index('ngrams_id_n_idx', 'ngrams', ['id', 'n'], unique=False)
+    op.create_index('ngrams_n_idx', 'ngrams', ['n'], unique=False)
+    op.create_table('nodes',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('typename', gargantext.models.nodes.NodeType(), nullable=True),
+    sa.Column('user_id', sa.Integer(), nullable=True),
+    sa.Column('parent_id', sa.Integer(), nullable=True),
+    sa.Column('name', sa.String(length=255), nullable=True),
+    sa.Column('date', sa.DateTime(), nullable=True),
+    sa.Column('hyperdata', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    sa.ForeignKeyConstraint(['parent_id'], ['nodes.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['user_id'], ['auth_user.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_nodes_typename'), 'nodes', ['typename'], unique=False)
+    op.create_index('nodes_hyperdata_idx', 'nodes', ['hyperdata'], unique=False)
+    op.create_index('nodes_user_id_typename_parent_id_idx', 'nodes', ['user_id', 'typename', 'parent_id'], unique=False)
+    op.create_table('nodes_hyperdata',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('node_id', sa.Integer(), nullable=True),
+    sa.Column('key', gargantext.models.hyperdata.HyperdataKey(), nullable=True),
+    sa.Column('value_int', sa.Integer(), nullable=True),
+    sa.Column('value_flt', postgresql.DOUBLE_PRECISION(), nullable=True),
+    sa.Column('value_utc', sa.DateTime(timezone=True), nullable=True),
+    sa.Column('value_str', sa.String(length=255), nullable=True),
+    sa.Column('value_txt', sa.Text(), nullable=True),
+    sa.ForeignKeyConstraint(['node_id'], ['nodes.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_nodes_hyperdata_value_flt'), 'nodes_hyperdata', ['value_flt'], unique=False)
+    op.create_index(op.f('ix_nodes_hyperdata_value_int'), 'nodes_hyperdata', ['value_int'], unique=False)
+    op.create_index(op.f('ix_nodes_hyperdata_value_str'), 'nodes_hyperdata', ['value_str'], unique=False)
+    op.create_index(op.f('ix_nodes_hyperdata_value_utc'), 'nodes_hyperdata', ['value_utc'], unique=False)
+    op.create_index('nodes_hyperdata_node_id_key_idx', 'nodes_hyperdata', ['node_id', 'key'], unique=False)
+    op.create_index('nodes_hyperdata_node_id_key_value_flt_idx', 'nodes_hyperdata', ['node_id', 'key', 'value_flt'], unique=False)
+    op.create_index('nodes_hyperdata_node_id_key_value_int_idx', 'nodes_hyperdata', ['node_id', 'key', 'value_int'], unique=False)
+    op.create_index('nodes_hyperdata_node_id_key_value_str_idx', 'nodes_hyperdata', ['node_id', 'key', 'value_str'], unique=False)
+    op.create_index('nodes_hyperdata_node_id_key_value_utc_idx', 'nodes_hyperdata', ['node_id', 'key', 'value_utc'], unique=False)
+    op.create_index('nodes_hyperdata_node_id_value_utc_idx', 'nodes_hyperdata', ['node_id', 'value_utc'], unique=False)
+    op.create_table('nodes_ngrams',
+    sa.Column('node_id', sa.Integer(), nullable=False),
+    sa.Column('ngram_id', sa.Integer(), nullable=False),
+    sa.Column('weight', sa.Float(), nullable=True),
+    sa.ForeignKeyConstraint(['ngram_id'], ['ngrams.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['node_id'], ['nodes.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('node_id', 'ngram_id')
+    )
+    op.create_index('nodes_ngrams_ngram_id_idx', 'nodes_ngrams', ['ngram_id'], unique=False)
+    op.create_index('nodes_ngrams_node_id_idx', 'nodes_ngrams', ['node_id'], unique=False)
+    op.create_index('nodes_ngrams_node_id_ngram_id_idx', 'nodes_ngrams', ['node_id', 'ngram_id'], unique=False)
+    op.create_table('nodes_ngrams_ngrams',
+    sa.Column('node_id', sa.Integer(), nullable=False),
+    sa.Column('ngram1_id', sa.Integer(), nullable=False),
+    sa.Column('ngram2_id', sa.Integer(), nullable=False),
+    sa.Column('weight', sa.Float(precision=24), nullable=True),
+    sa.ForeignKeyConstraint(['ngram1_id'], ['ngrams.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['ngram2_id'], ['ngrams.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['node_id'], ['nodes.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('node_id', 'ngram1_id', 'ngram2_id')
+    )
+    op.create_index('nodes_ngrams_ngrams_ngram1_id_idx', 'nodes_ngrams_ngrams', ['ngram1_id'], unique=False)
+    op.create_index('nodes_ngrams_ngrams_ngram2_id_idx', 'nodes_ngrams_ngrams', ['ngram2_id'], unique=False)
+    op.create_index('nodes_ngrams_ngrams_node_id_idx', 'nodes_ngrams_ngrams', ['node_id'], unique=False)
+    op.create_index('nodes_ngrams_ngrams_node_id_ngram1_id_ngram2_id_idx', 'nodes_ngrams_ngrams', ['node_id', 'ngram1_id', 'ngram2_id'], unique=False)
+    op.create_table('nodes_nodes',
+    sa.Column('node1_id', sa.Integer(), nullable=False),
+    sa.Column('node2_id', sa.Integer(), nullable=False),
+    sa.Column('score', sa.Float(precision=24), nullable=True),
+    sa.ForeignKeyConstraint(['node1_id'], ['nodes.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['node2_id'], ['nodes.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('node1_id', 'node2_id')
+    )
+    op.create_index('nodes_nodes_node1_id_node2_id_idx', 'nodes_nodes', ['node1_id', 'node2_id'], unique=False)
+    op.create_table('nodes_nodes_ngrams',
+    sa.Column('node1_id', sa.Integer(), nullable=False),
+    sa.Column('node2_id', sa.Integer(), nullable=False),
+    sa.Column('ngram_id', sa.Integer(), nullable=False),
+    sa.Column('score', sa.Float(precision=24), nullable=True),
+    sa.ForeignKeyConstraint(['ngram_id'], ['ngrams.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['node1_id'], ['nodes.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['node2_id'], ['nodes.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('node1_id', 'node2_id', 'ngram_id')
+    )
+    op.create_index('nodes_nodes_ngrams_node1_id_idx', 'nodes_nodes_ngrams', ['node1_id'], unique=False)
+    op.create_index('nodes_nodes_ngrams_node2_id_idx', 'nodes_nodes_ngrams', ['node2_id'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('nodes_nodes_ngrams_node2_id_idx', table_name='nodes_nodes_ngrams')
+    op.drop_index('nodes_nodes_ngrams_node1_id_idx', table_name='nodes_nodes_ngrams')
+    op.drop_table('nodes_nodes_ngrams')
+    op.drop_index('nodes_nodes_node1_id_node2_id_idx', table_name='nodes_nodes')
+    op.drop_table('nodes_nodes')
+    op.drop_index('nodes_ngrams_ngrams_node_id_ngram1_id_ngram2_id_idx', table_name='nodes_ngrams_ngrams')
+    op.drop_index('nodes_ngrams_ngrams_node_id_idx', table_name='nodes_ngrams_ngrams')
+    op.drop_index('nodes_ngrams_ngrams_ngram2_id_idx', table_name='nodes_ngrams_ngrams')
+    op.drop_index('nodes_ngrams_ngrams_ngram1_id_idx', table_name='nodes_ngrams_ngrams')
+    op.drop_table('nodes_ngrams_ngrams')
+    op.drop_index('nodes_ngrams_node_id_ngram_id_idx', table_name='nodes_ngrams')
+    op.drop_index('nodes_ngrams_node_id_idx', table_name='nodes_ngrams')
+    op.drop_index('nodes_ngrams_ngram_id_idx', table_name='nodes_ngrams')
+    op.drop_table('nodes_ngrams')
+    op.drop_index('nodes_hyperdata_node_id_value_utc_idx', table_name='nodes_hyperdata')
+    op.drop_index('nodes_hyperdata_node_id_key_value_utc_idx', table_name='nodes_hyperdata')
+    op.drop_index('nodes_hyperdata_node_id_key_value_str_idx', table_name='nodes_hyperdata')
+    op.drop_index('nodes_hyperdata_node_id_key_value_int_idx', table_name='nodes_hyperdata')
+    op.drop_index('nodes_hyperdata_node_id_key_value_flt_idx', table_name='nodes_hyperdata')
+    op.drop_index('nodes_hyperdata_node_id_key_idx', table_name='nodes_hyperdata')
+    op.drop_index(op.f('ix_nodes_hyperdata_value_utc'), table_name='nodes_hyperdata')
+    op.drop_index(op.f('ix_nodes_hyperdata_value_str'), table_name='nodes_hyperdata')
+    op.drop_index(op.f('ix_nodes_hyperdata_value_int'), table_name='nodes_hyperdata')
+    op.drop_index(op.f('ix_nodes_hyperdata_value_flt'), table_name='nodes_hyperdata')
+    op.drop_table('nodes_hyperdata')
+    op.drop_index('nodes_user_id_typename_parent_id_idx', table_name='nodes')
+    op.drop_index('nodes_hyperdata_idx', table_name='nodes')
+    op.drop_index(op.f('ix_nodes_typename'), table_name='nodes')
+    op.drop_table('nodes')
+    op.drop_index('ngrams_n_idx', table_name='ngrams')
+    op.drop_index('ngrams_id_n_idx', table_name='ngrams')
+    op.drop_table('ngrams')
+    op.drop_table('contacts')
+    # ### end Alembic commands ###

alembic/versions/bedce47c9e34_fix_issue_with_node_hyperdata_index.py

+"""Fix issue with Node.hyperdata index
+
+Revision ID: bedce47c9e34
+Revises: 08230100f262
+Create Date: 2017-07-10 11:30:59.168190
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import gargantext
+
+
+# revision identifiers, used by Alembic.
+revision = 'bedce47c9e34'
+down_revision = '08230100f262'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_index('nodes_hyperdata_idx', table_name='nodes')
+    op.create_index('nodes_hyperdata_idx', 'nodes', ['hyperdata'], unique=False, postgresql_using="gin")
+
+
+def downgrade():
+    # We won't unfix the bug when downgrading...
+    pass

dbmigrate.py

-#!/usr/bin/env python
-import sys
-import os
-
-if __name__ == "__main__":
-
-    # Django settings
-    dirname = os.path.dirname(os.path.realpath(__file__))
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "gargantext.settings")
-
-    # initialize Django application
-    from django.core.wsgi import get_wsgi_application
-    application = get_wsgi_application()
-
-    # retrieve Django models
-    import django.apps
-    django_models = django.apps.apps.get_models()
-    django_models_names = set(model._meta.db_table for model in django_models)
-
-    # migrate SQLAlchemy models
-    from gargantext import models
-    from gargantext.util.db import Base, engine
-    sqla_models_names = (
-        model for model in Base.metadata.tables.keys()
-        if model not in django_models_names
-    )
-    sqla_models = (
-        Base.metadata.tables[model_name]
-        for model_name in sqla_models_names
-    )
-    print()
-    for model in sqla_models:
-        try:
-            model.create(engine)
-            print('created model: `%s`' % model)
-        except Exception as e:
-            print('could not create model: `%s`, %s' % (model, e))
-        print()

docs/architecture.md

@@ -3,9 +3,10 @@
 
 ## Node
 
-The table (nodes) is a list of nodes: [Node]
+The table (nodes) is a list of nodes: `[Node]`
 
 Each Node has:
+
 - a typename
 - a parent_id
 - a name
@@ -13,63 +14,90 @@ Each Node has:
 
 ### Each Node has a parent_id
 
-Node A
-├── Node B
-└── Node C
+    Node A
+    ├── Node B
+    └── Node C
 
 If Node A is Parent of Node B and Node C
 then NodeA.id == NodeB.parent_id == NodeC.parent_id.
 
 ### Each Node has a typename
 
-Notation: Node[foo](bar) is a Node of typename "foo" and with name "bar".
+Notation: `Node["FOO"]("bar")` is a Node of typename "FOO" and with name "bar".
 
 Then:
-    - Then Node[project] is a project.
-    - Then Node[corpus] is a corpus.
-    - Then Node[document] is a document.
 
+- Then Node[PROJECT] is a project.
+- Then Node[CORPUS] is a corpus.
+- Then Node[DOCUMENT] is a document.
 
-### Each Node as a typename and a parent
+The syntax of the Node here do not follow exactly Python documentation
+(for clarity and to begin with): in Python code, typenames are strings
+represented as UPPERCASE strings (eg. "PROJECT").
 
+### Each Node as a typename and a parent
 
-Node[user](name)
-├── Node[project](myProject1)
-│   ├── Node[corpus](myCorpus1)
-│   ├── Node[corpus](myCorpus2)
-│   └── Node[corpus](myCorpus3)
-└── Node[project](myProject2)
+    Node[USER](name)
+    ├── Node[PROJECT](myProject1)
+    │   ├── Node[CORPUS](myCorpus1)
+    │   ├── Node[CORPUS](myCorpus2)
+    │   └── Node[CORPUS](myCorpus3)
+    └── Node[PROJECT](myProject2)
 
+/!\\ 3 ways to manage rights of the Node:
 
-/!\ 3 way to manage rights of the Node:
-    1) Then Node[User] is a folder containing all User projects and corpus and
-documents (i.e. Node[user] is the parent_id of the children).
-    2) Each node as a user_id (mainly used today)
-    3) Right management for the groups (implemented already but not
-    used since not connected to the frontend).
+1. Then Node[User] is a folder containing all User projects and corpus and
+   documents (i.e. Node[user] is the parent_id of the children).
+2. Each node as a user_id (mainly used today)
+3. Right management for the groups (implemented already but not
+   used since not connected to the frontend).
 
 
 ## Global Parameters
 
-Global User is Gargantua (Node with typename User).
-This node is the parent of the others Nodes for parameters.
+Global User is Gargantua (Node with typename user).
+This node is the parent of the other nodes for parameters.
+
+    Node[USER](gargantua) (gargantua.id == Node[USER].user_id)
+    ├── Node[TFIDF-Global](global) : without group
+    │   ├── Node[TFIDF](database1)
+    │   ├── Node[TFIDF](database2)
+    │   └── Node[TFIDF](database3)
+    └── Node[ANOTHERMETRIC](global)
+
+
+[//]: # (Are there any plans to add user wide or project wide parameters or metrics?  For example TFIDF nodes related to a normal user -- ie. not Gargantua?)
+
+Yes we can in the future (but we have others priorities before.
+
+[//]: # (What is the purpose of the 3 child nodes of Node[TFIDF-Global]?  Are they TFIDF metrics related to databases 1, 2 and 3? If so, shouldn't they be children of related CORPUS nodes?)
+
+Node placement in the tree indicates the context of the metric: the
+Metrics Node has parent the corpus Node to indicate the context of the
+metrics.
+
+Answer:
 
-Node[user](gargantua) (gargantua.id == Node[user].user_id)
-├── Node[TFIDF-Global](global) : without group
-│   ├── Node[tfidf](database1)
-│   ├── Node[tfidf](database2)
-│   └── Node[tfidf](database2)
-└── Node[anotherMetric](global)
+    Node[USER](foo)
+    Node[USER](bar)
+    ├── Node[PROJECT](project1)
+    │   ├── Node[CORPUS](corpus1)
+    │   │   ├── Node[DOCUMENT](doc1)
+    │   │   ├── Node[DOCUMENT](doc2)
+    │   │   └── Node[TFIDF-global](name of the metrics)
+    │   ├── Node[CORPUS](corpus2)
+    │   └── Node[CORPUS](corpus3)
+    └── Node[PROJECT](project2)
 
 
 
 ## NodeNgram
 
 NodeNgram is a relation of a Node with a ngram:
-    - document and ngrams
-    - metrics  and ngrams (position of the node metrics indicates the
-      context)
 
+- documents and ngrams
+- metrics  and ngrams (position of the node metrics indicates the
+  context)
 
 
 

docs/contribution.md

@@ -2,7 +2,7 @@
 
 ## Community
 * [http://gargantext.org/about](http://gargantext.org/about)
-* IRC Chat: (OFTC/FreeNode) #gargantex
+* IRC Chat: (OFTC/FreeNode) #gargantext
 
 ##Tools
 * gogs

docs/manual_install.md

-tools/manual_install.md
\ No newline at end of file

docs/manual_install.md

+* Create user gargantua
+Main user of Gargantext is Gargantua (role of Pantagruel soon)!
+``` bash
+sudo adduser --disabled-password --gecos "" gargantua
+```
+
+* Create the directories you need
+
+here for the example gargantext package will be installed in /srv/
+``` bash
+for dir in "/srv/gargantext"
+           "/srv/gargantext_lib"
+           "/srv/gargantext_static"
+           "/srv/gargantext_media"
+           "/srv/env_3-5"; do
+    sudo mkdir -p $dir ;
+    sudo chown gargantua:gargantua $dir ;
+done
+```
+
+You should see:
+
+```bash
+$tree /srv
+/srv
+├── gargantext
+├── gargantext_lib
+├── gargantext_media
+│   └── srv
+│       └── env_3-5
+└── gargantext_static
+```
+* Get the main libraries
+
+Download uncompress and make main user access to it.
+PLease, Be patient due to the size of the packages libraries (27GO)
+this step can be long....
+
+``` bash
+wget http://dl.gargantext.org/gargantext_lib.tar.bz2 \
+&& tar xvjf gargantext_lib.tar.bz2 -o /srv/gargantext_lib \
+&& sudo chown -R gargantua:gargantua /srv/gargantext_lib \
+&& echo "Libs installed"
+```
+
+* Get the source code of Gargantext
+
+by cloning the repository of gargantext
+``` bash
+git clone ssh://gitolite@delanoe.org:1979/gargantext /srv/gargantext \
+        && cd /srv/gargantext \
+        && git fetch origin stable \
+        && git checkout stable \
+```
+
+    TODO(soon): git clone https://gogs.iscpif.fr/gargantext.git
+
+
+* Install and configure the virtual environment
+``` bash
+cd /srv/
+pip3 install virtualenv
+virtualenv /srv/env_3-5 -p /usr/bin/python3.5
+pip install -r /srv/gargantext/install
+echo '/srv/gargantext' > /srv/env_3-5/lib/python3.5/site-packages/gargantext.pth
+echo 'alias venv="source /srv/env_3-5/bin/activate"' >> ~/.bashrc
+```
+
+
+See the [next steps of installation procedure](install.md#Install)
+See the [next manual steps of installation procedure](Debian.sh)

docs/ngram_lists.md

+# Gargantext foundations 
+
+Collaborative platform for multi-scale text experiments
+Embrace the past, update the present, forecast the future.
+
+# Main Types of Entity definitions
+
+Documentation valid for 3.0.\* versions of Gargantext.
+
+## Nature of the entities
+
+In Object programming language, it is objects.
+In purely functional language, it is types.
+
+
+## Project
+A project is a list of corpora (a project may have duplicate corpora).
+
+## Corpus
+A corpus is a set of documents: duplicate documents are authorized but
+not recommended for the methodology since it shows artificial repeated content in the corpus. 
+
+Then, in the document view, users may delete duplicates with a specific
+function.
+
+## Document
+A document is the main Entity of Textual Context (ETC) that is composed with:
+    - a title (truncated field name in the database)
+    - the date of publication
+    - a journal (or source)
+    - an abstract
+    - the authors
+Users may add many fields to the document.
+
+The main fields mentioned above are used for the main statistics in Gargantext.
+
+
+### Source Type
+Source Type is the source (database) from where documents have been
+extracted. 
+
+In 3.0.\* versions of Gargantext, each corpus has only one source type
+(i.e database). But user can build his own corpus with CVS format.
+
+
+## Ngrams
+
+### Definitions
+
+### Gram 
+A gram is a contiguous sequence of letters separated by spaces.
+
+### N-gram
+N-gram is a contiguous sequence of n grams separated by spaces (where n
+is a non negative natural number).
+
+
+## N-gram Lists
+
+
+## Main ngrams lists: Stop/Map/Main
+
+Receipe of Gargantext consist of offering the rights ngrams for the map.
+A the better level of complexity in order to unveil its richness
+according to this 2 main rules:
+
+If ngrams are too specifics, then the graph becomes too sparse.
+If ngrams are too generics, then the graph becomes too connected.
+
+As a consequence, finding the right balance of specific and generic
+ngrams is the main target.
+
+In first versions of Gargantext, this balance is solved with linear
+methods. After 3.1.\*, non linear methods trained on dataset of the
+users enable the system to find a better balance at any scale.
+
+
+### Definition
+
+3 main kinds of lists :
+    1. Stop List contains black listed ngrams i.e. the noise or in others words ngrams users do not want to deal with.
+    2. Map List contains ngrams that will be shown in the map.
+    3. Main list or Candidate list contains all other ngrams that are neither in the stop list or in the map list. Then it _could_ be in the map according to the choice of the user or, by default, the default parameters of Gargantext.
+
+### Storage
+
+Relation between the list and the ngram is stored as Node-Ngram
+relation where
+    - Node has type name (STOP|MAIN|MAP) and parent_id the context
+      (CORPUS in version 3.0.*; but could be PROJECT)
+    - Ngrams depend on the context of the Node List where NodeNgrams is
+      not null and Node has typename Document.
+
+
+    Node[USER](name1)
+    ├── Node[PROJECT](project1)
+    │   ├── Node[CORPUS](corpus1)
+    │   │   ├── Node[MAPLIST](list name)
+    │   │   ├── Node[STOPLIST](list name)
+    │   │   ├── Node[MAINLIST](list name)
+    │   │   │  
+    │   │   ├── Node[DOCUMENT](doc1)
+    │   │   ├── Node[DOCUMENT](doc2)
+    │   │   └── Node[DOCUMENT](doc2)
+
+
+### Policy
+
+#### Algo
+
+Let be a set of ngrams where NodeNgram != 0 then
+    find 2 subsets of these ngrams that show a split 
+        - stop ngrams
+        - not stop ngrams
+    then for the subset "not stop ngrams"
+        find 2 subset of ngrams that show a split:
+            - map ngrams
+            - others ngrams
+
+#### Techno algo
+
+A classifier (Support Machine Vector) is used on the following scaled-measures
+for each step:
+    - n (of the "n" gram)
+    - Occurrences : Zip Law (in fact already used in TFICF, this
+      features are correletad, put here for pedagogical purpose)
+    - TFICF-CORPUS-SOURCETYPE
+    - TFICF-SOURCETYPE-ALL
+    - Genericity score
+    - Specificty score
+
+
+
+
+## Metrics
+
+### Term Frequency - Inverse Context Frequency (TF-ICF)
+
+TFICF, short for term frequency-inverse context frequency, is a numerical
+statistic that is intended to reflect how important an ngram is to a
+context of text.
+
+TFICF(ngram,contextLocal,contextGlobal) = TF(ngram,contextLocal) \* ICF(ngram, contextGlobal)
+where
+ * TF(ngram, contextLocal) is the ngram frequency (occurrences) in contextLocal.
+ * ICF(ngram, contextGlobal) is the inverse (log) document frequency (occurrences) in contextGlobal.
+
+
+Others types of TFICF:
+    - TFICF(ngram, DOCUMENT, CORPUS)
+    - TFICF(ngram, CORPUS, PROJECT)
+    - TFICF(ngram, PROJECT, DATABASETYPE)
+    - TFICF(ngram, DATABASETYPE, ALL)
+
+
+If the context is a document in a set of documents (corpus), then it is a TFIDF as usual. 
+Then TFICF-DOCUMENT-CORPUS == TFICF(ngram,DOCUMENT,CORPUS) = TFIDF.
+TFICF is the generalization of [TFIDF, Term Frequency - Inverse Document Frequency](https://en.wikipedia.org/wiki/Tf%E2%80%93idf).
+
+#### Implementation
+
+TFICF = TF * log (ICF)
+
+To prepare the groups, we need to store TF and ICF seperately (in
+NodesNogram via 2 nodes).
+
+Let be TF and ICF typename of Nodes.
+
+
+    Node[USER](gargantua)
+    ├── Node[OCCURRENCES](source)
+    ├── Node[TF](all sourcetype)
+    ├── Node[ICF](all sourcetype)
+    ├── Node[SOURCETYPE](Pubmed)
+    │   ├── Node[OCCURRENCES](all corpora)
+    │   ├── Node[TF](all corpora)
+    │   └── Node[ICF](all corpora)
+    ├── Node[SOURCETYPE](WOS)
+
+
+
+
+## others ngrams lists
+
+### Group List
+
+#### Definition
+
+Group list gives a quantifiable link between two ngrams.
+
+
+#### Policy to build group lists
+
+To group the ngrams:
+- stemming or lemming
+- c-value
+- clustering (see graphs)
+- manually by the user (supervised learning)
+
+The scale is the character.
+
+
+#### Storage
+
+In the table NodeNgramNgram where Node has type name Group for ngram1
+and ngram2.
+
+
+### Favorite List
+
+#### Definition
+
+Fovorite Nodes
+The scale is the node.
+
+#### Building policy
+
+- manually by the user (supervised learning)
+
+#### Storage
+
+NodeNode relation where first Node has type Favorite.
+
+
+
+
+
+
+

docs/tools/about.md

-
-

docs/tools/about/index.md

-#Gargantext
-
-Welcome to Garagentext documentation!

docs/tools/api_urls.md

-List of garg's own JSON API(s) urls
-===================================
-
-2016-05-27
-
-### /api/nodes/2
-
-```
-{
-  "id": 2,
-  "parent_id": 1,
-  "name": "abstract:\"evaporation+loss\"",
-  "typename": "CORPUS"
-}
-```
-------------------------------
-
-### /api/nodes?pagination_limit=-1
-```
-{
-  "records": [
-    {
-      "id": 9,
-      "parent_id": 2,
-      "name": "A recording evaporimeter",
-      "typename": "DOCUMENT"
-    },
-    (...)
-    {
-      "id": 119,
-      "parent_id": 81,
-      "name": "GRAPH EXPLORER COOC (in:81)",
-      "typename": "COOCCURRENCES"
-    }
-  ],
-  "count": 119,
-  "parameters": {
-      "formated": "json","pagination_limit": -1,
-      "fields": ["id","parent_id","name","typename"],
-      "pagination_offset": 0
-  }
-}
-```
-------------------------------
-
-### /api/nodes?types[]=CORPUS
-```
-{
-  "records": [
-    {
-      "id": 2,
-      "parent_id": 1,
-      "name": "abstract:\"evaporation+loss\"",
-      "typename": "CORPUS"
-    },
-    (...)
-    {
-      "id": 8181,
-      "parent_id": 1,
-      "name": "abstract:(astrogeology+OR ((space OR spatial) AND planetary) AND geology)",
-      "typename": "CORPUS"
-    }
-  ],
-  "count": 2,
-  "parameters": {
-        "pagination_limit": 10,
-        "types": ["CORPUS"],
-        "formated": "json",
-        "pagination_offset": 0,
-        "fields": ["id","parent_id","name","typename"]
-  }
-}
-```
-------------------------------
-### /api/nodes/5?fields[]=ngrams
-
-Où <5> représente un doc_id ou list_id
-
-```
-{
-  "ngrams": [
-    [1.0,{"id":2299,"n":1,"terms":designs}],
-    [1.0,{"id":1917,"n":1,"terms":height}],
-    [1.0,{"id":1755,"n":2,"terms":higher speeds}],
-    [1.0,{"id":1940,"n":1,"terms":cylinders}],
-    [1.0,{"id":2221,"n":3,"terms":other synthesized materials}],
-    (...)
-    [2.0,{"id":1970,"n":1,"terms":storms}],
-    [9.0,{"id":1754,"n":2,"terms":spherical gauges}],
-    [1.0,{"id":1895,"n":1,"terms":direction}],
-    [1.0,{"id":2032,"n":1,"terms":testing}],
-    [1.0,{"id":1981,"n":2,"terms":"wind effects"}]
-  ]
-}
-```
-------------------------------
-
-### api/nodes/3?fields[]=id&fields[]=hyperdata&fields[]=typename
-```
-{
-  "id": 3,
-  "typename": "DOCUMENT",
-  "hyperdata": {
-    "language_name": "English",
-    "language_iso3": "eng",
-    "language_iso2": "en",
-    "title": "A blabla analysis of laser treated aluminium blablabla",
-    "name": "A blabla analysis of laser treated aluminium blablabla",
-    "authors": "A K. Jain, V.N. Kulkarni, D.K. Sood"
-    "authorsRAW": [
-    {"name": "....", "affiliations": ["... Research Centre,.. 085, Country"]},
-    {"name": "....", "affiliations": ["... Research Centre,.. 086, Country"]}
-    (...)
-    ],
-    "abstract": "Laser processing of materials, being a rapid melt quenching process, quite often produces a surface which is far from being ideally smooth for ion beam analysis. (...)",
-    "genre": ["research-article"],
-    "doi": "10.1016/0029-554X(81)90998-8",
-    "journal": "Nuclear Instruments and Methods In Physics Research",
-    "publication_year": "1981",
-    "publication_date": "1981-01-01 00:00:00",
-    "publication_month": "01",
-    "publication_day": "01",
-    "publication_hour": "00",
-    "publication_minute": "00",
-    "publication_second": "00",
-    "id": "61076EB1178A97939B1C893904C77FB7DA2276D0",
-    "source": "elsevier",
-    "distributor": "istex"
-  }
-}
-```
-
-## TODO continuer la liste

docs/tools/automatic_install.md

-#

docs/tools/contribute/ngrams/ngram_parsing_flow.dot

-// dot ngram_parsing_flow.dot -Tpng -o ngram_parsing_flow.png
-digraph ngramflow {
-
-    edge [fontsize=10] ;
-    label=<<B><U>gargantext.util.toolchain</U></B><BR/>(ngram extraction flow)>;
-    labelloc="t" ;
-
-    "extracted_ngrams" -> "grouplist" ;
-    "extracted_ngrams" -> "occs+ti_rank" ;
-    "project stoplist (todo)" -> "stoplist" ;
-    "stoplist" -> "mainlist" ;
-    "occs+ti_rank" -> "mainlist" [label="  TI_RANK_LIMIT"];
-    "mainlist" -> "coocs" [label="  COOCS_THRESHOLD"] ;
-    "coocs" -> "specificity" ;
-
-    "specificity" -> "maplist" [label="MAPLIST_LIMIT\nMONOGRAM_PART"];
-
-    "mainlist" -> "tfidf" ;
-    "tfidf" -> "explore" [label="doc relations with all map and candidates"];
-    "maplist" -> "explore" ;
-    "grouplist" -> "occs+ti_rank" ;
-    "grouplist" -> "coocs" ;
-    "grouplist" -> "tfidf" ;
-}

docs/tools/contribution-guide.md

-#Contribution guide
-
-## Community
-* [http://gargantext.org/about](http://gargantext.org/about)
-* IRC Chat: (OFTC/FreeNode) #gargantex
-
-##Tools
-* gogs
-* server access
-* forge
-* gargantext box
-
-##Gargantex
-* Gargantex box install
-
-(S.I.R.= Setup Install & Run procedures)
-
-* Architecture Overview
-
-
-* Database Schema Overview
-
-* Interface design Overview
-
-##To do:
-* Docs
-* Interface deisgn
-* Parsers/scrapers
-* Computing
-
-## How to contribute:
-    1. Clone the repo
-    2. Create a new branch <username>-refactoring
-    3. Run the gargantext-box
-    4. Code
-    5.Test
-    6. Commit
-
-### Exemple1: Adding a parser
-* create your new file cern.py into gargantex/scrapers/
-
-* reference into gargantex/scrapers/urls.py
-add this line:
-import scrapers.cern  as cern
-
-* reference into gargantext/constants
-```
-# type 9
-    {   'name': 'Cern',
-        'parser': CernParser,
-        'default_language': 'en',
-    },
-```
-* add an APIKEY in gargantex/settings
-
-
-### Exemple2: User Interface Design

docs/tools/contribution-guide/contribution.md

-#Contribution guide
-
-* A question or a problem? Ask the community
-* Sources
-* Tools
-* Contribution workflow: for contributions, bugs and features
-* Some examples of contributions
-
-## Community
-Need help? Ask the community
-* [http://gargantext.org/about](http://gargantext.org/about)
-* IRC Chat: (OFTC/FreeNode) #gargantex
-
-## Source
-Source are available throught XXX LICENSE
-
-You can install Gargantext throught the [installation procedure](./install.md)
-
-##Tools
-* gogs
-* forge.iscpif.fr
-* server access
-* gargantext box
-
-
-## Contributing: workflow procedure
-Once you have installed and tested Gargantext
-You
-
-1. Clone the stable release into your project
-    Note: The current stable release <release_branch> is:  refactoring
-
-Inside the repo, clone the reference branch and get the last changes:
-git checkout <ref_branch>
-git pull
-
-It is highly recommended to create a generic branch on a stable release such as
-git checkout -b <username>-<release_branch>
-git pull
-
-
-2. Create your project on stable release
-
-git checkout -b <username>-<release_branch>-<project_name>
-
-Do your modifications and commits as you want it:
-git commit -m "foo/bar/1"
-git commit -m "foo/bar/2"
-git push
-
-
-If you want to save your local change you can merge it into your generic branch <username>-<release_branch>
-git checkout <username>-<release_branch>
-git pull
-git merge <username>-<release_branch>-<project_name>
-git commit -m "[Merge OK] comment"
-
-
-##Technical Overview
-
-* Interface Overview
-
-* Database Schema Overview
-
-* Architecture Overview
-
-
-### Exemple1: Adding a parser
-
-### Exemple2: User Interface Design

docs/tools/contribution-guide/ngrams_lifecycle.md

-Cycle de vie des décomptes ngrammes
------------------------------------
-
-### (schéma actuel et pistes) ###
-
-Dans ce qui crée les décomptes, on peut distinguer deux niveaux ou étapes:
-
-1.  l'extraction initiale et le stockage du poids de la relation ngramme
-    document (appelons ces nodes "1doc")
-
-2.  tout le reste: la préparation des décomptes agrégés pour la table
-    termes ("stats"), et pour les tables de travail des graphes et de la
-    recherche de publications.
-
-On pourrait peut-être parler d'indexation par docs pour le niveau 1 et de "modélisations" pour le niveau 2.
-
-On peut remarquer que le niveau 1 concerne des **formes** ou ngrammes seuls (la forme observée <=> chaine de caractères u-nique après normalisation) tandis que dans le niveau 2 on a des objets plus riches... Au fur et à mesure des traitements on a finalement toujours des ngrammes mais:
-
-  - filtrés (on ne calcule pas tout sur tout)
-
-  - typés avec les listes map, stop, main (et peut-être bientôt des
-    "ownlistes" utilisateur)...
-
-  - groupés (ce qu'on voit avec le `+` de la table terme, et qu'on
-    pourrait peut-être faire apparaître aussi côté graphe?)
-
-On peut dire qu'on manipule plutôt des **termes** au niveau 2 et non plus des **formes**... ils sont toujours des ngrammes mais enrichis par l'inclusion dans une série de mini modèles (agrégations et typologie de ngrammes guidée par les usages).
-
-### Tables en BDD
-
-Si on adopte cette distinction entre formes et termes, ça permet de clarifier à quel moment on doit mettre à jour ce qu'on a dans les tables. Côté structure de données, les décomptes sont toujours stockés via des n-uplets qu'on peut du coup résumer comme cela:
-
--   **1doc**: (doc:node - forme:ngr - poids:float) dans des tables
-    NodeNgram
-
--   **occs/gen/spec/tirank**: (type_mesure:node - terme:ngr -
-    poids:float) dans des tables NodeNgram
-
--   **cooc**: (type_graphe:node - terme1:ngr - terme2:ngr -
-    poids:float) dans des tables NodeNgramNgram
-
--   **tfidf**: (type_lienspublis:node - doc:node - terme:ngr -
-    correlation:float) dans des tables NodeNodeNgram.
-
-Où "type" est le node portant la nature de la stat obtenue, ou bien la
-ref du graphe pour cooc et de l'index lié à la recherche de publis pour
-le tfidf.
-
-Il y a aussi les relations qui ne contiennent pas de décomptes mais sont
-essentielles pour former les décomptes des autres:
-
--   map/main/stopliste: (type_liste:node - forme ou terme:ngr) dans des
-    tables NodeNgram
-
--   "groupes": (mainform:ngr - subform:ngr) dans des tables
-    NodeNgramNgram.
-
-### Scénarios d'actualisation
-
-Alors, dans le déroulé des "scénarios utilisateurs", il y plusieurs
-évenements qui viennent **modifier ces décomptes**:
-
-1.  les créations de termes opérés par l'utilisateur (ex: par
-    sélection/ajout dans la vue annotation)
-
-2.  les imports de termes correspondant à des formes jamais indexées sur
-    ce corpus
-
-3.  les dégroupements de termes opérés par l'utilisateur
-
-4.  le passage d'un terme de la stopliste aux autres listes
-
-5.  tout autre changement de listes et/ou création de nouveaux
-    groupes...
-
-A et B sont les deux seules étapes hormis l'extraction initiale où des
-formes sont rajoutées. Actuellement A et B sont gérés tout de suite pour
-le niveau 1 (tables par doc) : il me semble qu'il est bon d'opérer la
-ré-indexation des 1doc le plus tôt possible après A ou B. Pour la vue
-annotations, l'utilisateur s'attend à voir apparaître le surlignage
-immédiatement sur le doc visualisé. Pour l'import B, c'est pratique car
-on a la liste des nouveaux termes sous la main, ça évite de la stocker
-quelque part en attendant un recalcul ultérieur.
-
-L'autre info mise à jour tout de suite pour A et B est l'appartenance
-aux listes et aux groupes (pour B), qui ne demandent aucun calcul.
-
-C, D et E n'affectent pas le niveau 1 (tables par docs) car ils ne
-rajoutent pas de formes nouvelles, mais constituent des modifications
-sur les listes et les groupes, et devront donc provoquer une
-modification du tfidf (pour cela on doit passer par un re-calcul) et des
-coocs sur map (effet appliqué à la demande d'un nouveau graphe).
-
-C et D demandent aussi une mise à jour des stats par termes
-(occurrences, gen/spec etc) puisque les éléments subforms et les
-éléments de la stopliste ne figurent pas dans les stats.
-
-Donc pour résumer on a dans tous les cas:
-
-=> l'ajout à une liste, à un groupe et tout éventuel décompte de
-nouvelle forme dans les docs sont gérés dès l'action utilisateur
-
-=> mais les modélisations plus "avancées" représentées par les les
-stats occs, gen, spec et les tables de travail "coocs sur map" et
-"tfidf" doivent attendre un recalcul.
-
-Idéalement à l'avenir il seraient tous mis à jour incrémentalement au
-lieu de forcer ce recalcul... mais pour l'instant on en est là.
-
-### Fonctions associées
-
-|       |                          GUI                          |                                       API action → url                                        |                      VIEW                       |                                                            SUBROUTINES                                                             |
-|-------|-------------------------------------------------------|-----------------------------------------------------------------------------------------------|-------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------|
-| A     | "annotations/highlight.js, annotations/ngramlists.js" | "PUT → api/ngrams, PUT/DEL → api/ngramlists/change"                                           | "ApiNgrams, ListChange"                         | util.toolchain.ngrams_addition.index_new_ngrams                                                                                    |
-| B     | NGrams_dyna_chart_and_table                           | POST/PATCH → api/ngramlists/import                                                            | CSVLists                                        | "util.ngramlists_tools.import_ngramlists, util.ngramlists_tools.merge_ngramlists, util.toolchain.ngrams_addition.index_new_ngrams" |
-| C,D,E | NGrams_dyna_chart_and_table                           | "PUT/DEL → api/ngramlists/change,  PUT/DEL → api/ngramlists/groups" "ListChange, GroupChange" | util.toolchain.ngrams_addition.index_new_ngrams |                                                                                                                                    |
-
-L'import B a été remis en route il y a quelques semaines, et je viens de
-reconnecter A dans la vue annotations.

docs/tools/contribution.md

-#Contribution guide
-
-## Community
-* [http://gargantext.org/about](http://gargantext.org/about)
-* IRC Chat: (OFTC/FreeNode) #gargantex
-
-##Tools
-* gogs
-* server access
-* gargantext box
-
-##Gargantex
-* Gargantex box install
-see [install procedure](install.md)
-
-* Architecture Overview
-* Database Schema Overview
-* Interface design Overview
-
-##To do:
-* Docs
-* Interface design
-* [Parsers](./overview/parser.md) / scrappers(./overview/scraper.md)
-* Computing
-
-## How to contribute:
-    1. Clone the repo
-    2. Create a new branch <username>-refactoring
-    3. Run the gargantext-box
-    4. Code
-    5. Test
-    6. Commit

docs/tools/corpus_example/pubmed.md5

-94eb7bdf57557b72dcd1b93a42af044b  pubmed.zip

docs/tools/demo/todo.md

-# API
-
-Be more careful about authorizations.
-
-cf. "ng-resource".
-
-# Projects
-
-## Overview of all projects
-
-- re-implement deletion
-
-## Single project view
-
-- re-implement deletion
-
-
-# Taggers
-
-Path for data used by taggers should be defined in `gargantext.constants`.
-
-
-# Database
-
-# Sharing
-
-Here follows a brief description of how sharing could be implemented.
-
-## Database representation
-
-The database representation of sharing can be distributed among 4 tables:
- - `persons`, of which items represent either a user or a group
- - `relationships` describes the relationships between persons (affiliation
-     of a user to a group, contact between two users, etc.)
- - `nodes` contains the projects, corpora, documents, etc. to share (they shall
-     inherit the sharing properties from their parents)
- - `permissions` stores the relations existing between the three previously
-     described above: it only consists of 2 foreign keys, plus an integer
-     between 1 and 3 representing the level of sharing and the start date
-     (when the sharing has been set) and the end date (when necessary, the time
-     at which sharing has been removed, `NULL` otherwise)
-
-## Python code
-
-The permission levels should be set in `gargantext.constants`, and defined as:
-```python
-PERMISSION_NONE = 0     # 0b0000
-PERMISSION_READ = 1     # 0b0001
-PERMISSION_WRITE = 3    # 0b0011
-PERMISSION_OWNER = 7    # 0b0111
-```
-
-The requests to check for permissions (or add new ones) should not be rewritten
-every time. They should be "hidden" within the models:
- - `Person.owns(node)` returns a boolean
- - `Person.can_read(node)` returns a boolean
- - `Person.can_write(node)` returns a boolean
- - `Person.give_right(node, permission)` gives a right to a given user
- - `Person.remove_right(node, permission)` removes a right from a given user
- - `Person.get_nodes(permission[, type])` returns an iterator on the list of
-    nodes on which the person has at least the given permission (optional
-    argument: type of requested node)
-- `Node.get_persons(permission[, type])` returns an iterator on the list of
-   users who have at least the given permission on the node (optional argument:
-   type of requested persons, such as `USER` or `GROUP`)
-
-## Example
-
-Let's imagine the `persons` table contains the following data:
-
-| id | type  | username  |
-|----|-------|-----------|
-| 1  | USER  | David     |
-| 2  | GROUP | C.N.R.S.  |
-| 3  | USER  | Alexandre |
-| 4  | USER  | Untel     |
-| 5  | GROUP | I.S.C.    |
-| 6  | USER  | Bidule    |
-
-Assume "David" owns the groups "C.N.R.S." and "I.S.C.", "Alexandre" belongs to
-the group "I.S.C.", with "Untel" and "Bidule" belonging to the group "C.N.R.S.".
-"Alexandre" and "David" are in contact.
-The `relationships` table then contains:
-
-| person1_id | person2_id | type    |
-|------------|------------|---------|
-| 1          | 2          | OWNER   |
-| 1          | 5          | OWNER   |
-| 3          | 2          | MEMBER  |
-| 4          | 5          | MEMBER  |
-| 6          | 5          | MEMBER  |
-| 1          | 3          | CONTACT |
-
-The `nodes` table is populated as such:
-
-| id | type     | name                 |
-|----|----------|----------------------|
-| 12 | PROJECT  | My super project     |
-| 13 | CORPUS   | A given corpus       |
-| 13 | CORPUS   | The corpus           |
-| 14 | DOCUMENT | Some document        |
-| 15 | DOCUMENT | Another document     |
-| 16 | DOCUMENT | Yet another document |
-| 17 | DOCUMENT | Last document        |
-| 18 | PROJECT  | Another project      |
-| 19 | PROJECT  | That project         |
-
-If we want to express that "David" created "My super project" (and its children)
-and wants everyone in "C.N.R.S." to be able to view it, but not access it,
-`permissions` should contain:
-
-| person_id | node_id | permission |
-|-----------|---------|------------|
-| 1         | 12      | OWNER      |
-| 2         | 12      | READ       |
-
-If "David" also wanted "Alexandre" (and no one else) to view and modify "The
-corpus" (and its children), we would have:
-
-| person_id | node_id | permission |
-|-----------|---------|------------|
-| 1         | 12      | OWNER      |
-| 2         | 12      | READ       |
-| 3         | 13      | WRITE      |
-
-If "Alexandre" created "That project" and wants "Bidule" (and no one else) to be
-able to view and modify it (and its children), the table should then have:
-
-| person_id | node_id | permission |
-|-----------|---------|------------|
-| 3         | 19      | OWNER      |
-| 6         | 19      | WRITE      |

docs/tools/demo/tuto.md

-#User guide
-
-1. Login
-
-run the gargantex box following the install procedure
-open a webrowser at http://127.0.0.1:8000/
-click on Test Gargantext
-login with:
-```
-Login : gargantua
-Password : autnagrag
-```
-
-2. Create a project
-
-3. Import an existing corpus
-
-4. Create corpus from search
-
-5. Explore stats
-
-6. Explore graphs
-
-7. Query
-
-8. Refine
-* Time periods
-* Nodes
-9. Export

docs/tools/discover.md

-#Architecture Overview
-#Database Schema
-#Website

docs/tools/index.md

-
-
-Gargantext is a web plateform to explore your corpora using text-mining[...](about.md)
-
-## Getting started
-
-* [Install](install.md) the Gargantext box
-
-* [Take a tour](demo.md) of the different features offered by Gargantext
-
-##Need some help?
-
-Ask the community at:
-
-* [http://gargantext.org/about](http://gargantext.org/about)
-* IRC Chat: (OFTC/FreeNode) #gargantex
-
-##Want to contribute?
-
-* take a look at the [architecture overview](overview.md)
-* read the [contribution guide](contribution-guide.md)
-
-
-## News
-
-## Credits and acknowledgments

docs/tools/install.md

-#Install Instructions for Gargamelle:
-
-Gargamelle is the gargantext plateforme toolbox it is a full plateform system
-with minimal modules
-
-First you need to get the source code to install it
-The folder will be /srv/gargantext:
-* docs containes all informations on gargantext
-    /srv/gargantext/docs/
-* install contains all the installation files
-    /srv/gargantext/install/
-
-Help needed ?
-See [http://gargantext.org/about](http://gargantext.org/about) and [tools](./contribution_guide.md) for the community
-
-## Get the source code
-
-by cloning gargantext into /srv/gargantext
-
-``` bash
-git clone ssh://gitolite@delanoe.org:1979/gargantext /srv/gargantext \
-        && cd /srv/gargantext \
-        && git fetch origin stable \
-        && git checkout stable \
-```
-
-
-## Install
- ```bash
- # go into the directory
- user@computer: cd /srv/gargantext/
- #git inside installation folder
- user@computer: cd /install
- #execute the installation
- user@computer: ./install
- ```
-The installation requires to create a user for gargantext,  it will be asked:
-
-```bash
-Username (leave blank to use 'gargantua'):
-#email is not mandatory
-Email address:
-Password:
-Password (again):
-```
-If successfully done this step you should see:
-```bash
-Superuser created successfully.
-[ ok ] Stopping PostgreSQL 9.5 database server: main.
-```
-
-
-## Run
-Once you proceed to installation Gargantext plateforme will be available at localhost:8000
-to start gargantext plateform:
- ``` bash
- # go into the directory
- user@computer: cd /srv/gargantext/
- #git inside installation folder
- user@computer: ./start
- #type ctrl+d to exit or simply type exit in terminal;
- ```
-
-Then open up a chromium browser and go to localhost:8000
-Click on "Enter Gargantext"
-Login in with you created username and pasword
-
-Enjoy! ;)

docs/tools/manual_install.md

-* Create user gargantua
-Main user of Gargantext is Gargantua (role of Pantagruel soon)!
-``` bash
-sudo adduser --disabled-password --gecos "" gargantua
-```
-
-* Create the directories you need
-
-here for the example gargantext package will be installed in /srv/
-``` bash
-for dir in "/srv/gargantext"
-           "/srv/gargantext_lib"
-           "/srv/gargantext_static"
-           "/srv/gargantext_media"
-           "/srv/env_3-5"; do
-    sudo mkdir -p $dir ;
-    sudo chown gargantua:gargantua $dir ;
-done
-```
-
-You should see:
-
-```bash
-$tree /srv
-/srv
-├── gargantext
-├── gargantext_lib
-├── gargantext_media
-│   └── srv
-│       └── env_3-5
-└── gargantext_static
-```
-* Get the main libraries
-
-Download uncompress and make main user access to it.
-PLease, Be patient due to the size of the packages libraries (27GO)
-this step can be long....
-
-``` bash
-wget http://dl.gargantext.org/gargantext_lib.tar.bz2 \
-&& tar xvjf gargantext_lib.tar.bz2 -o /srv/gargantext_lib \
-&& sudo chown -R gargantua:gargantua /srv/gargantext_lib \
-&& echo "Libs installed"
-```
-
-* Get the source code of Gargantext
-
-by cloning the repository of gargantext
-``` bash
-git clone ssh://gitolite@delanoe.org:1979/gargantext /srv/gargantext \
-        && cd /srv/gargantext \
-        && git fetch origin refactoring \
-        && git checkout refactoring \
-```
-
-    TODO(soon): git clone https://gogs.iscpif.fr/gargantext.git
-
-
-See the [next steps of installation procedure](install.md#Install)

docs/tools/overview.md

-#Architecture Overview
-#Database Schema
-#Website

docs/tools/overview/parser.md

-# HOW TO: Reference a new webscrapper/API + parser
-
-## Global scope
-Three main mooves to do:
-- develop and index parser
-in gargantext.util.parsers
-- developp and index a scrapper
-in gargantext.moissonneurs
-
-- adapt forms for a new source
-in templates and views
-
-## Reference parser into gargantext website
-gargantext website is stored in gargantext/gargantext
-
-### reference your new parser into contants.py
-* import your parser l.125
-```
-from gargantext.util.parsers import \
-    EuropressParser, RISParser, PubmedParser, ISIParser, CSVParser, ISTexParser, CernParser
-```
-The parser corresponds to the name of the parser referenced in gargantext/util/parser
-here  name is CernParser
-
-
-* index your RESOURCETYPE
-int RESOURCETYPES (l.145) **at the end of the list**
-```
-# type 10
-   {    "name": 'SCOAP (XML MARC21 Format)',
-        "parser": CernParser,
-        "default_language": "en",
-        'accepted_formats':["zip","xml"],
-   },
-```
-    A noter le nom ici est composé de l'API_name(SCOAP) + (GENERICFILETYPE FORMAT_XML Format)
-    La complexité du nommage correspond à trois choses:
-        * le nom de l'API (different de l'organisme de production)
-        * le type de format: XML
-        * la norme XML de ce format : MARC21 (cf. CernParser in gargantext/util/parser/Cern.py )
-
-The default_langage corresponds to the default accepted lang that **should load** the default corresponding tagger
-
-
-```
-from gargantext.util.taggers import NltkTagger
-```
-    TO DO: charger à la demander les types de taggers en fonction des langues et de l'install
-    TO DO: proposer un module pour télécharger des parsers supplémentaires
-    TO DO: provide install tagger module scripts inside lib
-
-Les formats correspondent aux types de fichiers acceptées lors de l'envoi du fichier dans le formulaire de
-parsing disponible dans `gargantext/view/pages/projects.py` et
-exposé dans `/templates/pages/projects/project.html`
-
-## reference your parser script
-
-## add your parser script into folder gargantext/util/parser/
-here my filename was Cern.py
-
-##declare it into gargantext/util/parser/__init__.py
-from .Cern  import CernParser
-
-At this step, you will be able to see your parser and add a file with the form
-but nothing will occur
-
-## the good way to write the scrapper script
-
-Three main and only requirements:
-* your parser class should inherit from the base class _Parser()
-`gargantext/gargantext/util/parser/_Parser`
-* your parser class must have a parse method that take a **file buffer** as input
-* you parser must structure and store data into **hyperdata_list** variable name
-to be properly indexed by toolchain
-! Be careful of date format: provide a publication_date in  a string format YYYY-mm-dd HH:MM:SS
-
-# Adding a scrapper API to offer search option:
-En cours
-* Add pop up question Do you have a corpus
-option search in /templates/pages/projects/project.html line 181
-
-
-## Reference a scrapper (moissonneur) into gargantext
-
-* adding accepted_formats in constants
-* adding check_file routine in Form check ==> but should inherit from utils/files.py
-that also have implmented the size upload limit check
-
-# Suggestion 4 next steps:
-* XML parser MARC21 UNIMARC ...
-* A project type is qualified by the first element add i.e:
-the first element determine the type of corpus of all the corpora within the project
-

docs/tools/resource.md

-#resources
-
-Adding a new source into Gargantext requires a previous declaration
-of the source inside constants.py
-
-```python
-RESOURCETYPES= [
-{    "type":9, #give a unique type int
-      "name": 'SCOAP [XML]', #resource name as proposed into the add corpus FORM [generic format]
-      "parser": "CernParser", #name of the new parser class inside a CERN.py file (set to None if not implemented)
-      "format": 'MARC21', #specific format
-      'file_formats':["zip","xml"],# accepted file format
-      "crawler": "CernCrawler", #name of the new crawler class inside a CERN.py file (set to None if no Crawler implemented)
-      'default_languages': ['en', 'fr'], #supported defaut languages of the source
- },
- ...
- ]
-```
-## adding a new parser
-
-Once you declared your new parser inside constants.py
-
-add your new crawler file into /srv/gargantext/utils/parsers/
-following this naming convention:
-
-* Filename must be in uppercase without the Crawler mention.
-  eg. MailParser => MAIL.py
-* Inside this file the Parser must be called following the exact typo declared as parser in constants.py
-* Your new crawler shall inherit from baseclasse Parser and provide a parse(filebuffer) method
-
-```python
-  #!/usr/bin/python3 env
-  #filename:/srv/gargantext/util/parser/MAIL.py:
-  from ._Parser import Parser
-  class MailParser(Parser):
-      def parse(self, file):
-          ...
-```
-## adding a new crawler
-
-Once you declared your new parser inside constants.py
-add your new crawler file into /srv/gargantext/utils/parsers/
-following this naming convention:
-
-* Filename must be in uppercase without the Crawler mention.
-  eg. MailCrawler => MAIL.py
-* Inside this file the Crawler must be called following the exact typo declared as crawler in constants.py
-* Your new crawler shall inherit from baseclasse Crawler and provide three method:
-  * scan_results => ids
-  * sample = > yes/no
-  * fetch
-
-```python
-  #!/usr/bin/python3 env
-  #filename:/srv/gargantext/util/crawler/MAIL.py:
-  from ._Crawler import Crawler
-  class MailCrawler(Crawler):
-      def scan_results(self, query):
-        ...
-        self.ids = set()
-      def sample(self, results_nb):
-        ...
-      def fetch(self, ids):
-        
-```

docs/tools/schemas/ngram_parsing_flow.dot

-// dot ngram_parsing_flow.dot -Tpng -o ngram_parsing_flow.png
-digraph ngramflow {
-
-    edge [fontsize=10] ;
-    label=<<B><U>gargantext.util.toolchain</U></B><BR/>(ngram extraction flow)>;
-    labelloc="t" ;
-
-    "extracted_ngrams" -> "grouplist" ;
-    "extracted_ngrams" -> "occs+tfidfs" ;
-    "main_user_stoplist" -> "stoplist" ;
-    "stoplist" -> "mainlist" ;
-    "occs+tfidfs" -> "mainlist" [label="  TFIDF_LIMIT"];
-    "mainlist" -> "coocs" [label="  COOCS_THRESHOLD"] ;
-    "coocs" -> "specificity" ;
-
-    "specificity" -> "maplist" [label="MAPLIST_LIMIT\nMONOGRAM_PART"];
-
-    "maplist" -> "explore" ;
-    "grouplist" -> "maplist" ;
-}

gargantext/__init__.py

+from .celery import app as celery_app

gargantext/celery.py

+"""
+Setup the Celery instance (see also gargantext/__init__.py) that will be
+used by all shared_task.
+
+This is the recommended way:
+http://docs.celeryproject.org/en/3.1/django/first-steps-with-django.html
+"""
+
+import os
+from celery import Celery
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'gargantext.settings')
+
+from django.conf import settings #noqa
+
+app = Celery('gargantext')
+app.config_from_object('django.conf:settings')
+app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

gargantext/constants.py

@@ -36,7 +36,7 @@ import os
 import re
 import importlib
 from gargantext.util.lists import *
-from gargantext.util.tools import datetime, convert_to_date
+from gargantext.util import datetime, convert_to_datetime
 from .settings import BASE_DIR
 
 # types & models (nodes, lists, hyperdata, resource) ---------------------------------------------
@@ -59,25 +59,25 @@ LISTTYPES = {
 NODETYPES = [
     # TODO separate id not array index, read by models.node
     None,                    # 0
-    # documents hierarchy
+    # node/file hierarchy
     'USER',                  # 1
     'PROJECT',               # 2
     #RESOURCE should be here but last
     'CORPUS',                # 3
     'DOCUMENT',              # 4
-    # lists
+    # lists of ngrams
     'STOPLIST',              # 5
     'GROUPLIST',             # 6
     'MAINLIST',              # 7
     'MAPLIST',               # 8
     'COOCCURRENCES',         # 9
-    # scores
+    # scores for ngrams
     'OCCURRENCES',           # 10
     'SPECCLUSION',           # 11
     'CVALUE',                # 12
     'TFIDF-CORPUS',          # 13
     'TFIDF-GLOBAL',          # 14
-    # docs subset
+    # node subset
     'FAVORITES',             # 15
     # more scores (sorry!)
 
@@ -108,9 +108,9 @@ INDEXED_HYPERDATA = {
 
     'publication_date':
         { 'id'             : 2
-        , 'type'           : datetime.datetime
-        , 'convert_to_db'  : convert_to_date
-        , 'convert_from_db': datetime.datetime.fromtimestamp
+        , 'type'           : datetime
+        , 'convert_to_db'  : convert_to_datetime
+        , 'convert_from_db': convert_to_datetime
         },
 
     'title':

gargantext/management/commands/nodes.py

+from django.core.management.base import BaseCommand, CommandError
+from gargantext.util.show_nodes import tree_show, nodes
+import colorama
+
+
+class Command(BaseCommand):
+    help = 'Nodes'
+
+    def add_arguments(self, parser):
+        parser.add_argument(dest='action', default='show')
+
+    def handle(self, *args, **options):
+        action = options.get('action')
+
+        if action == 'show':
+            colorama.init(strip=False)
+            for root in nodes():
+                tree_show(root)

gargantext/models/__init__.py

+from .base import Base
 from .nodes import *
 from .hyperdata import *
 from .users import *

gargantext/models/base.py

+from sqlalchemy.schema import Column, ForeignKey, UniqueConstraint, Index
+from sqlalchemy.orm import relationship, validates
+from sqlalchemy.types import TypeDecorator, \
+                             Integer, Float, Boolean, DateTime, String, Text
+from sqlalchemy.dialects.postgresql import JSONB, DOUBLE_PRECISION as Double
+from sqlalchemy.ext.mutable import MutableDict, MutableList
+from sqlalchemy.ext.declarative import declarative_base
+
+__all__ = ["Column", "ForeignKey", "UniqueConstraint", "relationship",
+           "validates", "ValidatorMixin",
+           "Integer", "Float", "Boolean", "DateTime", "String", "Text",
+           "TypeDecorator",
+           "JSONB", "Double",
+           "MutableDict", "MutableList",
+           "Base", "DjangoBase"]
+
+
+# All the models should derive from this base class, so Base.metadata keeps
+# all tables handled by Alembic migration scripts.
+Base = declarative_base()
+
+
+# To be used by tables already handled by Django ORM, such as User model. We
+# separate them in order to keep those out of Alembic sight.
+DjangoBase = declarative_base()
+
+
+class ValidatorMixin(object):
+    def enforce_length(self, key, value):
+        """Truncate a string according to its column length
+
+        Usage example:
+
+        .. code-block:: python
+
+            @validates('some_column')
+            def validate_some_column(self, key, value):
+                self.enforce_length(key, value)
+        """
+        max_len = getattr(self.__class__, key).prop.columns[0].type.length
+        if value and len(value) > max_len:
+            return value[:max_len]
+        return value

gargantext/models/hyperdata.py

-from gargantext.util.db import *
 from gargantext.constants import INDEXED_HYPERDATA
 
+from .base import Base, Column, ForeignKey, TypeDecorator, Index, \
+                  Integer, Double, DateTime, String, Text
 from .nodes import Node
 import datetime
 
@@ -64,6 +65,14 @@ class NodeHyperdata(Base):
         )
     """
     __tablename__ = 'nodes_hyperdata'
+    __table_args__ = (
+            Index('nodes_hyperdata_node_id_value_utc_idx', 'node_id', 'value_utc'),
+            Index('nodes_hyperdata_node_id_key_value_utc_idx', 'node_id', 'key', 'value_utc'),
+            Index('nodes_hyperdata_node_id_key_value_str_idx', 'node_id', 'key', 'value_str'),
+            Index('nodes_hyperdata_node_id_key_value_int_idx', 'node_id', 'key', 'value_int'),
+            Index('nodes_hyperdata_node_id_key_value_flt_idx', 'node_id', 'key', 'value_flt'),
+            Index('nodes_hyperdata_node_id_key_idx', 'node_id', 'key'))
+
     id        = Column( Integer, primary_key=True )
     node_id   = Column( Integer, ForeignKey(Node.id, ondelete='CASCADE'))
     key       = Column( HyperdataKey )

gargantext/models/migration.py

-from gargantext.util.db import *
-from gargantext.util.files import upload
-from gargantext.constants import *
-from gargantext.util.toolchain.main import parse_extract_indexhyperdata
-
-from datetime import datetime
-
-from .users import User
-from .nodes import Node
-
-#__all__ = ['Node', 'NodeType', 'Language']
-
-
-class NodeType_v2(Base):
-    __table_args__ = {'extend_existing': True}
-    __tablename__  = 'node_nodetype'
-    id             = Column(Integer, primary_key=True)
-    name           = Column(String(255))
-
-class Language_v2(Base):
-    __table_args__ = {'extend_existing': True}
-    __tablename__  = 'node_language'
-    id             = Column(Integer, primary_key=True)
-    iso2           = Column(String(2))
-    iso3           = Column(String(3))
-    fullname       = Column(String(255))
-    implemented    = Column(Boolean)
-
-
-class Node_v2(Base):
-    __table_args__ = {'extend_existing': True}
-    __tablename__  = 'node_node'
-    id             = Column(Integer, primary_key=True)
-    parent_id      = Column(Integer, ForeignKey('node_node.id'))
-    user_id        = Column(Integer, ForeignKey(User.id))
-    type_id        = Column(ForeignKey(NodeType_v2.id))
-    name           = Column(String(255))
-    language_id    = Column(Integer, ForeignKey(Language_v2.id))
-    date           = Column(DateTime(), default=datetime.now)
-    hyperdata      = Column(JSONB, default=dict)
-
-
-class ResourceType(Base):
-    __table_args__ = {'extend_existing': True}
-    __tablename__  = 'node_resourcetype'
-    id             = Column(Integer, primary_key=True)
-    name           = Column(String(255))
-
-class NodeResource(Base):
-    __table_args__ = {'extend_existing': True}
-    __tablename__  = 'node_node_resource'
-    id             = Column(Integer, primary_key=True)
-    node_id        = Column(ForeignKey(Node_v2.id))
-    resource_id    = Column(ForeignKey(ResourceType.id))
-    parsed         = Column(Boolean)
-
-
-
-def nodes_list(user_id, nodetype, parent_id=None, count=False):
-    """
-    nodes_list :: Int -> String -> Maybe Int -> Maybe Bool -> [(Int, String)]
-    """
-    nodes = ( session.query(Node_v2.id, Node_v2.name)
-                     .join(NodeType_v2, NodeType_v2.id == Node_v2.type_id)
-                     .filter(NodeType_v2.name == nodetype)
-               )
-    if parent_id is not None:
-        nodes = nodes.filter(Node_v2.parent_id == parent_id)
-
-    if count is True:
-        return nodes.count()
-    else:
-        return nodes.all()
-
-def nodes_tree(user_id):
-    """
-    nodes_tree :: Int -> Tree Nodes
-    """
-    for project_id, project_name in nodes_list(user_id, 'Project'):
-        print("* Project (%d, %s)" % (project_id, project_name))
-        for corpus_id, corpus_name in nodes_list(user_id, 'Corpus', parent_id=project_id):
-
-            count =  nodes_list( user_id
-                               , 'Document'
-                               , parent_id=corpus_id
-                               , count=True
-                               )
-
-            if count > 1:
-                print("|__ %d %s" % ( corpus_id, corpus_name ))
-                print("   |___ %s docs" % count)
-
-
-def copy_nodes(node_id, to_parent_id=None, enabled=['PROJECT', 'CORPUS', 'DOCUMENT']):
-    node = session.query(Node_v2).filter(Node_v2.id==node_id).first()
-    
-    nodetype = session.query(NodeType_v2).filter(NodeType_v2.id == node.type_id).first()
-    
-    resource = (session.query(ResourceType)
-                           .join(NodeResource, NodeResource.resource_id == ResourceType.id)
-                           .filter(NodeResource.node_id == node.id)
-                           .first()
-                           )
-    
-    nodetype_proj_id = session.query(NodeType_v2.id).filter(NodeType_v2.name == 'Project' ).first()
-    nodetype_corp_id = session.query(NodeType_v2.id).filter(NodeType_v2.name == 'Corpus'  ).first()
-    nodetype_docu_id = session.query(NodeType_v2.id).filter(NodeType_v2.name == 'Document').first()
-
-    typename = nodetype.name.upper()
-
-# Import a project:
-#         new_project = Node(
-#             user_id = user.id,
-#             typename = 'PROJECT',
-#             name = name,
-#         )
-#         session.add(new_project)
-#         session.commit()
-
-    if typename in enabled:
-        parent_node = session.query(Node).filter(Node.id==to_parent_id).first()
-        if parent_node is not None:
-            corpus = parent_node.add_child(
-                name = node.name,
-                typename = typename
-                )
-            
-            corpus.hyperdata['languages'] = {'fr' : 100}
-            try:
-                corpus.add_resource(
-                    type = resourcetype(resource.name)
-                    )
-            except:
-                corpus.add_resource(
-                    type = resourcetype('Europress (French)')
-                    )
-            session.add(corpus)
-            session.commit()
-            print("%s copied" % corpus.name)
-
-
-            nodes = (session.query(Node_v2)
-                             .filter(Node_v2.parent_id == node.id)
-                             .filter(Node_v2.type_id == nodetype_docu_id)
-                    .all()
-                    )
-
-            for n in nodes:
-                print(n.name)
-                doc = corpus.add_child( name      = n.name
-                                      , typename  = "DOCUMENT"
-                                      , hyperdata = n.hyperdata
-                                      )
-                session.add(doc)
-                session.commit()
-            
-
-#        else:
-#            print("%d is None" % parent_id)
-
-    else:
-        print('%s is not enabled' % typename)
-
-
-
-

gargantext/models/ngrams.py

-from gargantext.util.db import *
-
+from .base import Base, Column, ForeignKey, relationship, Index, \
+                  Integer, Float, String
 from .nodes import Node
 
 __all__ = ['Ngram', 'NodeNgram', 'NodeNodeNgram', 'NodeNgramNgram']
@@ -7,17 +7,39 @@ __all__ = ['Ngram', 'NodeNgram', 'NodeNodeNgram', 'NodeNgramNgram']
 
 class Ngram(Base):
     __tablename__ = 'ngrams'
+    __table_args__ = (
+            Index('ngrams_id_n_idx', 'id', 'n'),
+            Index('ngrams_n_idx', 'n'))
+
     id = Column(Integer, primary_key=True)
     terms = Column(String(255), unique=True)
     n = Column(Integer)
 
+    def __str__(self):
+        return '<{0.terms}>#{0.n}'.format(self)
+
+    def __repr__(self):
+        return '<Ngram(id={0.id}, terms={0.terms!r}, n={0.n})>'.format(self)
+
 
 class NodeNgram(Base):
     __tablename__ = 'nodes_ngrams'
+    __table_args__ = (
+            Index('nodes_ngrams_node_id_ngram_id_idx', 'node_id', 'ngram_id'),
+            Index('nodes_ngrams_node_id_idx', 'node_id'),
+            Index('nodes_ngrams_ngram_id_idx', 'ngram_id'))
+
     node_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
     ngram_id = Column(Integer, ForeignKey(Ngram.id, ondelete='CASCADE'), primary_key=True)
     weight = Column(Float)
 
+    node = relationship(Node)
+    ngram = relationship(Ngram)
+
+    def __repr__(self):
+        return '<NodeNgram(node_id={0.node_id}, ngram={0.ngram}, weight={0.weight})>'.format(self)
+
+
 class NodeNodeNgram(Base):
     """ for instance for TFIDF
     (
@@ -28,6 +50,10 @@ class NodeNodeNgram(Base):
     )
     """
     __tablename__ = 'nodes_nodes_ngrams'
+    __table_args__ = (
+            Index('nodes_nodes_ngrams_node2_id_idx', 'node2_id'),
+            Index('nodes_nodes_ngrams_node1_id_idx', 'node1_id'))
+
     node1_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
     node2_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
     ngram_id = Column(Integer, ForeignKey(Ngram.id, ondelete='CASCADE'), primary_key=True)
@@ -36,6 +62,14 @@ class NodeNodeNgram(Base):
     # sinon par défaut on aurait un type sql "double_precision" (soit 15 chiffres)
     # (cf. www.postgresql.org/docs/9.4/static/datatype-numeric.html#DATATYPE-FLOAT)
 
+    node1 = relationship(Node, foreign_keys=[node1_id])
+    node2 = relationship(Node, foreign_keys=[node2_id])
+    ngram = relationship(Ngram)
+
+    def __repr__(self):
+        return '<NodeNodeNgram(node1_id={0.node1_id}, node2_id={0.node2_id}, ngram={0.ngram}, score={0.score})>'.format(self)
+
+
 class NodeNgramNgram(Base):
     """ for instance for COOCCURRENCES and GROUPLIST
     (
@@ -46,7 +80,20 @@ class NodeNgramNgram(Base):
     )
     """
     __tablename__ = 'nodes_ngrams_ngrams'
+    __table_args__ = (
+            Index('nodes_ngrams_ngrams_node_id_ngram1_id_ngram2_id_idx', 'node_id', 'ngram1_id', 'ngram2_id'),
+            Index('nodes_ngrams_ngrams_node_id_idx', 'node_id'),
+            Index('nodes_ngrams_ngrams_ngram1_id_idx', 'ngram1_id'),
+            Index('nodes_ngrams_ngrams_ngram2_id_idx', 'ngram2_id'))
+
     node_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
     ngram1_id = Column(Integer, ForeignKey(Ngram.id, ondelete='CASCADE'), primary_key=True)
     ngram2_id = Column(Integer, ForeignKey(Ngram.id, ondelete='CASCADE'), primary_key=True)
     weight = Column(Float(precision=24))  # see comment for NodeNodeNgram.score
+
+    node = relationship(Node)
+    ngram1 = relationship(Ngram, foreign_keys=[ngram1_id])
+    ngram2 = relationship(Ngram, foreign_keys=[ngram2_id])
+
+    def __repr__(self):
+        return '<NodeNgramNgram(node_id={0.node_id}, ngram1={0.ngram1}, ngram2={0.ngram2}, weight={0.weight})>'.format(self)

gargantext/models/nodes.py

-from gargantext.util.db import *
+from gargantext.util.db import session
 from gargantext.util.files import upload
 from gargantext.constants import *
 
+# Uncomment to make column full text searchable
+#from sqlalchemy_utils.types import TSVectorType
+
 from datetime import datetime
 
+from .base import Base, Column, ForeignKey, relationship, TypeDecorator, Index, \
+                  Integer, Float, String, DateTime, JSONB, \
+                  MutableList, MutableDict, validates, ValidatorMixin
 from .users import User
 
-__all__ = ['Node', 'NodeNode']
+__all__ = ['Node', 'NodeNode', 'CorpusNode']
 
 class NodeType(TypeDecorator):
     """Define a new type of column to describe a Node's type.
@@ -19,23 +25,69 @@ class NodeType(TypeDecorator):
     def process_result_value(self, typeindex, dialect):
         return NODETYPES[typeindex]
 
-class Node(Base):
-    """This model can fit many purposes.
+
+class Node(ValidatorMixin, Base):
+    """This model can fit many purposes:
+
+    myFirstCorpus = session.query(CorpusNode).first()
+
     It intends to provide a generic model, allowing hierarchical structure
     and NoSQL-like data structuring.
     The possible types are defined in `gargantext.constants.NODETYPES`.
+
+    Thanks to __new__ overriding and SQLAlchemy's polymorphism, every Node
+    instance is automagically casted to its sub-class, assuming a typename
+    is specified.
+
+    >>> Node(name='without-type')
+    <Node(id=None, typename=None, user_id=None, parent_id=None, name='without-type', date=None)>
+    >>> Node(typename='CORPUS')
+    <CorpusNode(id=None, typename='CORPUS', user_id=None, parent_id=None, name=None, date=None)>
+    >>> from gargantext.util.db import session
+    >>> session.query(Node).filter_by(typename='USER').first() # doctest: +ELLIPSIS
+    <UserNode(...)>
+
+    But beware, there are some caveats with bulk queries. In this case typename
+    MUST be specified manually.
+
+    >>> session.query(UserNode).delete() # doctest: +SKIP
+    # Wrong: all nodes are deleted!
+    >>> session.query(UserNode).filter_by(typename='USER').delete() # doctest: +SKIP
+    # Right: only user nodes are deleted.
     """
     __tablename__ = 'nodes'
+    __table_args__ = (
+            Index('nodes_user_id_typename_parent_id_idx', 'user_id', 'typename', 'parent_id'),
+            Index('nodes_hyperdata_idx', 'hyperdata', postgresql_using='gin'))
+
+    # TODO
+    # create INDEX full_text_idx on nodes using gin(to_tsvector('english', hyperdata ->> 'abstract' || 'title'));
+
     id = Column(Integer, primary_key=True)
+
     typename = Column(NodeType, index=True)
+    __mapper_args__ = { 'polymorphic_on': typename }
+
     # foreign keys
-    user_id = Column(Integer, ForeignKey(User.id, ondelete='CASCADE'))
-    parent_id = Column(Integer, ForeignKey('nodes.id', ondelete='CASCADE'))
-    # main data
+    user_id       = Column(Integer, ForeignKey(User.id, ondelete='CASCADE'))
+    user          = relationship(User)
+
+    parent_id     = Column(Integer, ForeignKey('nodes.id', ondelete='CASCADE'))
+    parent        = relationship('Node', remote_side=[id])
+
     name = Column(String(255))
-    date  = Column(DateTime(), default=datetime.now)
+    date  = Column(DateTime(timezone=True), default=datetime.now)
+
+    hyperdata     = Column(JSONB, default=dict)
     # metadata (see https://bashelton.com/2014/03/updating-postgresql-json-fields-via-sqlalchemy/)
-    hyperdata = Column(JSONB, default=dict)
+    # To make search possible uncomment the line below
+    #search_vector = Column(TSVectorType('hyperdata'))
+
+    def __new__(cls, *args, **kwargs):
+        if cls is Node and kwargs.get('typename'):
+            typename = kwargs.pop('typename')
+            return _NODE_MODELS[typename](*args, **kwargs)
+        return super(Node, cls).__new__(cls)
 
     def __init__(self, **kwargs):
         """Node's constructor.
@@ -55,6 +107,15 @@ class Node(Base):
         """
         self.hyperdata[key] = value
 
+    def __repr__(self):
+        return '<{0.__class__.__name__}(id={0.id}, typename={0.typename!r}, ' \
+               'user_id={0.user_id}, parent_id={0.parent_id}, ' \
+               'name={0.name!r}, date={0.date})>'.format(self)
+
+    @validates('name')
+    def validate_name(self, key, value):
+        return self.enforce_length(key, value)
+
     @property
     def ngrams(self):
         """Pseudo-attribute allowing to retrieve a node's ngrams.
@@ -120,36 +181,6 @@ class Node(Base):
             **kwargs
         )
 
-    def resources(self):
-        """Return all the resources attached to a given node.
-        Mainly used for corpora.
-
-        example:
-        [{'extracted': True,
-          'path': '/home/me/gargantext/uploads/corpora/0c/0c5b/0c5b50/0c5b50ad8ebdeb2ae33d8e54141a52ee_Corpus_Europresse-Français-2015-12-11.zip',
-          'type': 1,
-          'url': None}]
-        """
-        if 'resources' not in self.hyperdata:
-            self['resources'] = MutableList()
-        return self['resources']
-
-    def add_resource(self, type, path=None, url=None):
-        """Attach a resource to a given node.
-        Mainly used for corpora.
-
-        this just adds metadata to the CORPUS node (NOT for adding documents)
-
-        example:
-        {'extracted': True,
-          'path': '/home/me/gargantext/uploads/corpora/0c/0c5b/0c5b50/0c5b50ad8ebdeb2ae33d8e54141a52ee_Corpus_Europresse-Français-2015-12-11.zip',
-          'type': 1,
-          'url': None}
-        """
-        self.resources().append(MutableDict(
-            {'type': type, 'path':path, 'url':url, 'extracted': False}
-        ))
-
     def status(self, action=None, progress=0, complete=False, error=None):
         """Get or update the status of the given action.
         If no action is given, the status of the first uncomplete or last item
@@ -187,8 +218,86 @@ class Node(Base):
         ))
         return self['statuses'][-1]
 
+
+class CorpusNode(Node):
+    __mapper_args__ = {
+        'polymorphic_identity': 'CORPUS'
+    }
+
+    def resources(self):
+        """Return all the resources attached to a given node.
+
+        example:
+        [{'extracted': True,
+          'path': '/home/me/gargantext/uploads/corpora/0c/0c5b/0c5b50/0c5b50ad8ebdeb2ae33d8e54141a52ee_Corpus_Europresse-Français-2015-12-11.zip',
+          'type': 1,
+          'url': None}]
+        """
+        if 'resources' not in self.hyperdata:
+            self['resources'] = MutableList()
+        return self['resources']
+
+    def add_resource(self, type, path=None, url=None):
+        """Attach a resource to a given node.
+
+        this just adds metadata to the CORPUS node (NOT for adding documents)
+
+        example:
+        {'extracted': True,
+          'path': '/home/me/gargantext/uploads/corpora/0c/0c5b/0c5b50/0c5b50ad8ebdeb2ae33d8e54141a52ee_Corpus_Europresse-Français-2015-12-11.zip',
+          'type': 1,
+          'url': None}
+        """
+        self.resources().append(MutableDict(
+            {'type': type, 'path':path, 'url':url, 'extracted': False}
+        ))
+
+
 class NodeNode(Base):
     __tablename__ = 'nodes_nodes'
+    __table_args__ = (
+            Index('nodes_nodes_node1_id_node2_id_idx', 'node1_id', 'node2_id'),)
+
     node1_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
     node2_id = Column(Integer, ForeignKey(Node.id, ondelete='CASCADE'), primary_key=True)
     score    = Column(Float(precision=24))
+
+    node1 = relationship(Node, foreign_keys=[node1_id])
+    node2 = relationship(Node, foreign_keys=[node2_id])
+
+    def __repr__(self):
+        return '<NodeNode(node1_id={0.node1_id}, node2_id={0.node2_id}, score={0.score})>'.format(self)
+
+
+# --8<-- Begin hack ------
+
+# XXX Hack to automatically defines subclasses of Node for every NODETYPES,
+#     in order to avoid SQLAlchemy complaints -- and subsequent exceptions.
+#
+#     We could manually write a class for every NodeType, or find a way to
+#     tell SQLAlchemy that it should stick to instantiate a Node when a
+#     class is not defined for the wanted typename.
+
+_ALREADY_IMPLEMENTED_NODE_TYPES = \
+    set(cls.__mapper_args__.get('polymorphic_identity') for cls in Node.__subclasses__())
+
+for nodetype in NODETYPES:
+    if nodetype and nodetype not in _ALREADY_IMPLEMENTED_NODE_TYPES:
+        # Convert nodetype to a CamelCase class name, assuming it's possible...
+        class_name = ''.join(nodetype.title().split("-")) + 'Node'
+        # Create new class and add it to global scope
+        globals()[class_name] = type(class_name, (Node,), {
+            "__mapper_args__": {
+                "polymorphic_identity": nodetype
+            }
+        })
+        # Add class to exports
+        __all__.append(class_name)
+
+# ------ End of hack ------
+
+_NODE_MODELS = {
+    mapper.polymorphic_identity: mapper.class_
+    for mapper in Node.__mapper__.self_and_descendants
+    if mapper.class_ is not Node
+}

gargantext/models/users.py

 from django.contrib.auth import models
-from gargantext.util.db import *
+from gargantext.util.db import session, aliased
 
 from datetime import datetime
 
+from .base import DjangoBase, Base, Column, ForeignKey, UniqueConstraint, \
+                  Integer, Boolean, DateTime, String
+
 __all__ = ['User', 'Contact']
 
-class User(Base):
+class User(DjangoBase):
     # The properties below are a reflection of Django's auth module's models.
     __tablename__ = models.User._meta.db_table
     id           = Column(Integer, primary_key=True)
@@ -60,7 +63,7 @@ class User(Base):
         """check if a given node is owned by the user"""
         return (node.user_id == self.id) or \
                 node.id in (contact.id for contact in self.contacts())
-    
+
     def get_params(self, username=None):
         print(self.__dict__.items())
         return self.hyperdata

gargantext/util/__init__.py

+from .dates import datetime, convert_to_datetime, MINYEAR

gargantext/util/alembic.py

+"""Define ReplaceableObject and related operations
+
+Implements operations to create/drop SQL objects such as views, stored
+procedures and triggers that can't be "altered" but can be replaced -- hence
+the name of "ReplaceableObject" class.
+
+This recipe is directly borrowed from Alembic documentation, see
+http://alembic.zzzcomputing.com/en/latest/cookbook.html#replaceable-objects
+
+"""
+
+from alembic.operations import Operations, MigrateOperation
+
+
+__all__ = ['ReplaceableObject']
+
+
+class ReplaceableObject(object):
+    def __init__(self, name, sqltext):
+        self.name = name
+        self.sqltext = sqltext
+
+
+class ReversibleOp(MigrateOperation):
+    def __init__(self, target):
+        self.target = target
+
+    @classmethod
+    def invoke_for_target(cls, operations, target):
+        op = cls(target)
+        return operations.invoke(op)
+
+    def reverse(self):
+        raise NotImplementedError()
+
+    @classmethod
+    def _get_object_from_version(cls, operations, ident):
+        version, objname = ident.split(".")
+
+        module = operations.get_context().script.get_revision(version).module
+        obj = getattr(module, objname)
+        return obj
+
+    @classmethod
+    def replace(cls, operations, target, replaces=None, replace_with=None):
+
+        if replaces:
+            old_obj = cls._get_object_from_version(operations, replaces)
+            drop_old = cls(old_obj).reverse()
+            create_new = cls(target)
+        elif replace_with:
+            old_obj = cls._get_object_from_version(operations, replace_with)
+            drop_old = cls(target).reverse()
+            create_new = cls(old_obj)
+        else:
+            raise TypeError("replaces or replace_with is required")
+
+        operations.invoke(drop_old)
+        operations.invoke(create_new)
+
+
+@Operations.register_operation("create_view", "invoke_for_target")
+@Operations.register_operation("replace_view", "replace")
+class CreateViewOp(ReversibleOp):
+    def reverse(self):
+        return DropViewOp(self.target)
+
+
+@Operations.register_operation("drop_view", "invoke_for_target")
+class DropViewOp(ReversibleOp):
+    def reverse(self):
+        return CreateViewOp(self.view)
+
+
+@Operations.register_operation("create_sp", "invoke_for_target")
+@Operations.register_operation("replace_sp", "replace")
+class CreateSPOp(ReversibleOp):
+    def reverse(self):
+        return DropSPOp(self.target)
+
+
+@Operations.register_operation("drop_sp", "invoke_for_target")
+class DropSPOp(ReversibleOp):
+    def reverse(self):
+        return CreateSPOp(self.target)
+
+
+@Operations.implementation_for(CreateViewOp)
+def create_view(operations, operation):
+    operations.execute("CREATE VIEW %s AS %s" % (
+        operation.target.name,
+        operation.target.sqltext
+    ))
+
+
+@Operations.implementation_for(DropViewOp)
+def drop_view(operations, operation):
+    operations.execute("DROP VIEW %s" % operation.target.name)
+
+
+@Operations.implementation_for(CreateSPOp)
+def create_sp(operations, operation):
+    operations.execute(
+        "CREATE FUNCTION %s %s" % (
+            operation.target.name, operation.target.sqltext
+        )
+    )
+
+
+@Operations.implementation_for(DropSPOp)
+def drop_sp(operations, operation):
+    operations.execute("DROP FUNCTION %s" % operation.target.name)

gargantext/util/crawlers/HAL.py

@@ -45,6 +45,7 @@ class HalCrawler(Crawler):
                , uri_s
                , isbn_s
                , issue_s
+               , docType_s
                , journalPublisher_s
              """
                #, authUrl_s
@@ -112,7 +113,7 @@ class HalCrawler(Crawler):
             msg = "Invalid sample size N = %i (max = %i)" % ( self.query_max
                                                             , QUERY_SIZE_N_MAX
                                                             )
-            print("ERROR (scrap: Multivac d/l ): " , msg)
+            print("ERROR (scrap: HAL d/l ): " , msg)
             self.query_max = QUERY_SIZE_N_MAX
         
         #for page in range(1, trunc(self.query_max / 100) + 2):

gargantext/util/dates.py

+import os
+from gargantext.settings import MEDIA_ROOT
+
+from datetime import MINYEAR
+from django.utils.dateparse import parse_datetime
+from django.utils.timezone import datetime as _datetime, utc as UTC, now as utcnow
+
+__all__ = ['convert_to_datetime', 'datetime', 'MINYEAR']
+
+
+class datetime(_datetime):
+    @staticmethod
+    def now():
+        return utcnow()
+
+    @staticmethod
+    def utcfromtimestamp(ts):
+        return _datetime.utcfromtimestamp(ts).replace(tzinfo=UTC)
+
+    @staticmethod
+    def parse(s):
+        dt = parse_datetime(s)
+        return dt.astimezone(UTC) if dt.tzinfo else dt.replace(tzinfo=UTC)
+
+
+def convert_to_datetime(dt):
+    if isinstance(dt, (int, float)):
+        return datetime.utcfromtimestamp(dt)
+
+    elif isinstance(dt, str):
+        return datetime.parse(dt)
+
+    elif isinstance(dt, _datetime):
+        args = (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
+        return datetime(*args, tzinfo=dt.tzinfo or UTC).astimezone(UTC)
+
+    else:
+        raise ValueError("Can't convert to datetime: %r" % dt)

gargantext/util/db.py

@@ -5,16 +5,18 @@ from gargantext.util.json import json_dumps
 ########################################################################
 # get engine, session, etc.
 ########################################################################
+import sqlalchemy as sa
 from sqlalchemy.orm import sessionmaker, scoped_session
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import delete
 
+# To make Full Text search possible, uncomment lines below
+# (and install it with pip before)
+#from sqlalchemy_searchable import make_searchable
+
 def get_engine():
     from sqlalchemy import create_engine
-    url = 'postgresql+psycopg2://{USER}:{PASSWORD}@{HOST}:{PORT}/{NAME}'.format(
-        **settings.DATABASES['default']
-    )
-    return create_engine( url
+    return create_engine( settings.DATABASES['default']['URL']
                         , use_native_hstore = True
                         , json_serializer = json_dumps
                         , pool_size=20, max_overflow=0
@@ -22,20 +24,16 @@ def get_engine():
 
 engine = get_engine()
 
+# To make Full Text search possible, uncomment lines below
+# https://sqlalchemy-searchable.readthedocs.io/
+#sa.orm.configure_mappers()
 Base = declarative_base()
+#Base.metadata.create_all(engine)
+#make_searchable()
 
 session = scoped_session(sessionmaker(bind=engine))
 
 
-########################################################################
-# tools to build models
-########################################################################
-from sqlalchemy.types import *
-from sqlalchemy.schema import Column, ForeignKey, UniqueConstraint
-from sqlalchemy.dialects.postgresql import JSONB, DOUBLE_PRECISION
-from sqlalchemy.ext.mutable import MutableDict, MutableList
-Double = DOUBLE_PRECISION
-
 ########################################################################
 # useful for queries
 ########################################################################

gargantext/util/db_cache.py

@@ -29,6 +29,7 @@ class ModelCache(dict):
                 continue
         if formatted_key in self:
             self[key] = self[formatted_key]
+            element = self[key]
         else:
             element = session.query(self._model).filter(or_(*conditions)).first()
             if element is None:

gargantext/util/http.py

@@ -73,7 +73,8 @@ from rest_framework.views import APIView
 from gargantext.util.json import json_encoder
 def JsonHttpResponse(data, status=200):
     return HttpResponse(
-        content      = json_encoder.encode(data),
+        content      = data.encode('utf-8') if isinstance(data, str) else \
+                       json_encoder.encode(data),
         content_type = 'application/json; charset=utf-8',
         status       = status
     )

gargantext/util/json.py

@@ -10,7 +10,7 @@ __all__ = ['json_encoder', 'json_dumps']
 
 class JSONEncoder(json.JSONEncoder):
     def default(self, obj):
-        from gargantext.util.db import Base
+        from gargantext.models import Base
         if isinstance(obj, Base):
             return {
                 key: value

gargantext/util/ngramlists_tools.py

@@ -472,6 +472,7 @@ def import_ngramlists(the_file, delimiter=DEFAULT_CSV_DELIM,
                     raise ValueError('Wrong header "%s" on line %i (only possible headers are "label", "forms" and "status")' % (colname, n_read_lines))
             if 'label' not in columns:
                 raise ValueError('CSV must contain at least one column with the header "label"')
+            continue
 
         if not len(csv_row):
             continue
@@ -578,7 +579,8 @@ def import_ngramlists(the_file, delimiter=DEFAULT_CSV_DELIM,
          'map':  UnweightedList(),
          'main': UnweightedList(),
          'stop': UnweightedList(),
-         'groupings' : Translations()
+         'groupings' : Translations(),
+         'new_ngram_count': n_added_ng,
          }
 
     for list_type in imported_nodes_ngrams:
@@ -674,12 +676,13 @@ def merge_ngramlists(new_lists={}, onto_corpus=None, del_originals=[]):
             for ng_id in new_lists[list_type].items:
                 collect(ng_id)
 
-    from gargantext.util.toolchain.main import t
-    print("MERGE DEBUG: starting index_new_ngrams", t())
-    n_added = index_new_ngrams(all_possibly_new_ngram_ids, onto_corpus)
-    print("MERGE DEBUG: finished index_new_ngrams", t())
+    if new_lists.get('new_ngram_count', 0) > 0:
+        from gargantext.util.toolchain.main import t
+        print("MERGE DEBUG: starting index_new_ngrams", t())
+        n_added = index_new_ngrams(all_possibly_new_ngram_ids, onto_corpus)
+        print("MERGE DEBUG: finished index_new_ngrams", t())
 
-    my_log.append("MERGE: added %i new ngram occurrences in docs" % n_added)
+        my_log.append("MERGE: added %i new ngram occurrences in docs" % n_added)
 
     # ======== Get the old lists =========
     old_lists = {}
@@ -838,7 +841,7 @@ def merge_ngramlists(new_lists={}, onto_corpus=None, del_originals=[]):
 
 
 @shared_task
-def import_and_merge_ngramlists(file_contents, onto_corpus_id):
+def import_and_merge_ngramlists(file_contents, onto_corpus_id, overwrite=False):
     """
     A single function to run import_ngramlists and merge_ngramlists together
     """
@@ -848,6 +851,7 @@ def import_and_merge_ngramlists(file_contents, onto_corpus_id):
     corpus_node = session.query(Node).filter(Node.id == onto_corpus_id).first()
 
     # merge the new_lists onto those of the target corpus
-    log_msg = merge_ngramlists(new_lists, onto_corpus=corpus_node)
+    del_originals = ['stop', 'main', 'map'] if overwrite else []
+    log_msg = merge_ngramlists(new_lists, onto_corpus=corpus_node, del_originals=del_originals)
 
     return log_msg

gargantext/util/parsers/HAL.py

@@ -11,17 +11,8 @@ from datetime import datetime
 import json
 
 class HalParser(Parser):
-
-    def parse(self, filebuf):
-        '''
-        parse :: FileBuff -> [Hyperdata]
-        '''
-        contents = filebuf.read().decode("UTF-8")
-        data = json.loads(contents)
+    def _parse(self, json_docs):
         
-        filebuf.close()
-        
-        json_docs = data
         hyperdata_list = []
         
         hyperdata_path = { "id"       : "isbn_s"
@@ -73,3 +64,13 @@ class HalParser(Parser):
                 hyperdata_list.append(hyperdata)
         
         return hyperdata_list
+
+    def parse(self, filebuf):
+        '''
+        parse :: FileBuff -> [Hyperdata]
+        '''
+        contents = filebuf.read().decode("UTF-8")
+        data = json.loads(contents)
+
+        return self._parse(data)
+

gargantext/util/parsers/MULTIVAC.py

@@ -18,30 +18,30 @@ class MultivacParser(Parser):
         '''
         contents = filebuf.read().decode("UTF-8")
         data = json.loads(contents)
-        
+
         filebuf.close()
-        
+
         json_docs = data
         hyperdata_list = []
-        
+
         hyperdata_path = { "id"       : "id"
                          , "title"    : "title"
                          , "abstract" : "abstract"
                          , "type"     : "type"
                          }
-        
+
         for json_doc in json_docs:
 
             hyperdata = {}
-            
+
             doc = json_doc["_source"]
 
             for key, path in hyperdata_path.items():
                     hyperdata[key] = doc.get(path, "")
-            
+
             hyperdata["source"] = doc.get("serial"      , {})\
                                      .get("journaltitle", "REPEC Database")
-            
+
             try:
                 hyperdata["url"]    = doc.get("file", {})\
                                          .get("url" , "")
@@ -51,15 +51,15 @@ class MultivacParser(Parser):
             hyperdata["authors"] = ", ".join(
                                              [ p.get("person", {})
                                                 .get("name"  , "")
-                          
+
                                                for p in doc.get("hasauthor", [])
                                              ]
                                             )
-            
+
 
             year = doc.get("serial"  , {})\
                       .get("issuedate", None)
-            
+
             if year == "Invalide date":
                 year = doc.get("issuedate"  , None)
 
@@ -73,10 +73,7 @@ class MultivacParser(Parser):
                     date = datetime.now()
 
             hyperdata["publication_date"] = date
-            hyperdata["publication_year"]  = str(date.year)
-            hyperdata["publication_month"] = str(date.month)
-            hyperdata["publication_day"]   = str(date.day)
-            
+
             hyperdata_list.append(hyperdata)
-        
+
         return hyperdata_list

gargantext/util/parsers/_Parser.py

-import datetime
 import dateutil.parser
 import zipfile
 import re
 import dateparser as date_parser
 from gargantext.util.languages import languages
+from gargantext.util import datetime, convert_to_datetime, MINYEAR
 
 
-DEFAULT_DATE = datetime.datetime(datetime.MINYEAR, 1, 1)
+DEFAULT_DATE = datetime(MINYEAR, 1, 1)
 
 
 class Parser:
@@ -34,29 +34,29 @@ class Parser:
     def format_hyperdata_dates(self, hyperdata):
         """Format the dates found in the hyperdata.
         Examples:
-            {"publication_date": "2014-10-23 09:57:42"}
-            -> {"publication_date": "2014-10-23 09:57:42", "publication_year": "2014", ...}
+            {"publication_date": "2014-10-23 09:57:42+00:00"}
+            -> {"publication_date": "2014-10-23 09:57:42+00:00", "publication_year": "2014", ...}
             {"publication_year": "2014"}
-            -> {"publication_date": "2014-01-01 00:00:00", "publication_year": "2014", ...}
+            -> {"publication_date": "2014-01-01 00:00:00+00:00", "publication_year": "2014", ...}
         """
 
         # First, check the split dates...
         # This part mainly deal with Zotero data but can be usefull for others
         # parts
-        date_string = hyperdata.get('publication_date_to_parse', None)
+        date_string = hyperdata.get('publication_date_to_parse')
         if date_string is not None:
             date_string = re.sub(r'\/\/+(\w*|\d*)', '', date_string)
             try:
-                hyperdata['publication' + "_date"] = dateutil.parser.parse(
+                hyperdata['publication_date'] = dateutil.parser.parse(
                     date_string,
                     default=DEFAULT_DATE
-                ).strftime("%Y-%m-%d %H:%M:%S")
+                )
             except Exception as error:
                 print(error, 'Date not parsed for:', date_string)
-                hyperdata['publication_date'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                hyperdata['publication_date'] = datetime.now()
 
 
-        elif hyperdata.get('publication_year', None) is not None:
+        elif hyperdata.get('publication_year') is not None:
 
             prefixes = [key[:-5] for key in hyperdata.keys() if key[-5:] == "_year"]
             # eg prefixes : ['publication']
@@ -64,56 +64,45 @@ class Parser:
             for prefix in prefixes:
                 date_string = hyperdata[prefix + "_year"]
 
-                # FIXME: except for year is it necessary to test that key exists
-                #        when we have a default value in .get(key, "01") ??
-                key = prefix + "_month"
-                if key in hyperdata:
-                    date_string += " " + hyperdata.get(key, "01")
-                    key = prefix + "_day"
-                    if key in hyperdata:
-                        date_string += " " + hyperdata.get(key, "01")
-                        key = prefix + "_hour"
-                        if key in hyperdata:
-                            date_string += " " + hyperdata.get(key, "01")
-                            key = prefix + "_minute"
-                            if key in hyperdata:
-                                date_string += ":" + hyperdata.get(key, "01")
-                                key = prefix + "_second"
-                                if key in hyperdata:
-                                    date_string += ":" + hyperdata.get(key, "01")
+                for part in ('month', 'day', 'hour', 'minute', 'second'):
+                    key = prefix + '_' + part
+                    if key not in hyperdata:
+                        break
+
+                    sep = ":" if key in ('minute', 'second') else " "
+                    date_string += sep + hyperdata.get(key, '01')
+
                 try:
-                    hyperdata[prefix + "_date"] = dateutil.parser.parse(date_string).strftime("%Y-%m-%d %H:%M:%S")
+                    hyperdata[prefix + "_date"] = dateutil.parser.parse(date_string)
                 except Exception as error:
                     try:
                         print("_Parser: error in full date parse", error, date_string)
                         # Date format:  1994 NOV-DEC
-                        hyperdata[prefix + "_date"] = date_parser.parse(str(date_string)[:8]).strftime("%Y-%m-%d %H:%M:%S")
+                        hyperdata[prefix + "_date"] = date_parser.parse(str(date_string)[:8])
 
                     except Exception as error:
                         try:
                             print("_Parser: error in short date parse", error)
                             # FIXME Date format:  1994 SPR
                             # By default, we take the year only
-                            hyperdata[prefix + "_date"] = date_parser.parse(str(date_string)[:4]).strftime("%Y-%m-%d %H:%M:%S")
+                            hyperdata[prefix + "_date"] = date_parser.parse(str(date_string)[:4])
 
                         except Exception as error:
                             print("_Parser:", error)
         else:
             print("WARNING: Date unknown at _Parser level, using now()")
-            hyperdata['publication_date'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+            hyperdata['publication_date'] = datetime.now()
 
         # ...then parse all the "date" fields, to parse it into separate elements
         prefixes = [key[:-5] for key in hyperdata.keys() if key[-5:] == "_date"]
         for prefix in prefixes:
-            date = dateutil.parser.parse(hyperdata[prefix + "_date"])
-            #print(date)
-
-            hyperdata[prefix + "_year"]      = date.strftime("%Y")
-            hyperdata[prefix + "_month"]     = date.strftime("%m")
-            hyperdata[prefix + "_day"]       = date.strftime("%d")
-            hyperdata[prefix + "_hour"]      = date.strftime("%H")
-            hyperdata[prefix + "_minute"]    = date.strftime("%M")
-            hyperdata[prefix + "_second"]    = date.strftime("%S")
+            name = prefix + "_date"
+            date = hyperdata[name]
+            hyperdata[name] = str(convert_to_datetime(date))
+
+            for part in ('year', 'month', 'day', 'hour', 'minute', 'second'):
+                hyperdata[prefix + '_' + part] = getattr(date, part)
+
         # print("line 116", hyperdata['publication_date'])
         # finally, return the transformed result!
         return hyperdata

gargantext/util/show_nodes.py

+# Make this a standalone script...
+# Can be called this way: python3 gargantext/util/show_nodes.py
+
+import os
+import django
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'gargantext.settings')
+django.setup()
+
+# ...End of jiberish.
+
+import itertools
+import colorama
+from colorama import Fore
+from sqlalchemy.sql.expression import literal_column
+
+from gargantext.util.db import session, func, aliased
+from gargantext.models import Node
+
+
+NODE_BULLET = '‣'
+
+# https://en.wikipedia.org/wiki/Box-drawing_character
+TREE_ROOT = '╾'
+TREE_VERT = '│'
+TREE_HORI = '─'
+TREE_FORK = '├'
+TREE_CORN = '└'
+
+FIRST  = 0x01
+LAST   = 0x02
+
+
+def nodes(parent=None, group_by='typename', order_by='typename', has_child='check'):
+    if group_by or has_child is not None:
+        select = [func.min(Node.id).label('id'),
+                  func.min(Node.name).label('name'),
+                  func.min(Node.typename).label('typename'),
+                  func.count(Node.id).label('cnt')]
+    else:
+        select = [Node.id.label('id'),
+                  Node.name.label('name'),
+                  Node.typename.label('typename'),
+                  literal_column('1').label('cnt')]
+
+    if has_child is not None:
+        N = aliased(Node)
+        select.append(func.count(N.id).label('children'))
+    else:
+        select.append(literal_column('NULL').label('children'))
+
+    parent_id = getattr(parent, 'id', parent)
+    q = session.query(*select).filter_by(parent_id=parent_id) \
+               .group_by(getattr(Node, group_by if group_by else 'id'))
+
+    if has_child is not None:
+        q = q.outerjoin(N, N.parent_id == Node.id).group_by(N.parent_id)
+
+    return q.order_by(order_by)
+
+
+def node_show(node, prefix='', maxlen=60):
+    if node.children > 0 or node.cnt == 1:
+        name = node.name[:maxlen] + '..' if len(node.name) > maxlen else node.name
+        label = Fore.CYAN + name + Fore.RESET
+    else:
+        label = Fore.MAGENTA + str(node.cnt) + Fore.RESET
+
+    print(prefix, '%s%s %s' % (Fore.GREEN, node.typename, label), sep='')
+
+
+def tree_show(node, pos=FIRST|LAST, level=0, prefix='', maxlen=60, compact=True):
+    #print('%02d %x' % (level, pos), end='')
+
+    branch = TREE_ROOT if pos&FIRST and level == 0 else TREE_FORK if not pos&LAST else TREE_CORN
+    node_prefix = prefix + branch + 2*TREE_HORI + ' '
+    node_show(node, node_prefix, maxlen)
+
+    childs = iter(nodes(parent=node, group_by=compact and 'typename'))
+
+    try:
+        node = next(childs)
+    except StopIteration:
+        return
+
+    prefix = prefix + (' ' if pos&LAST else TREE_VERT) + '   '
+
+    for i, next_node in enumerate(itertools.chain(childs, [None])):
+        pos = (FIRST if i == 0 else 0) | (LAST if next_node is None else 0)
+        tree_show(node, pos, level + 1, prefix, maxlen, compact)
+        node = next_node
+
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) == 1:
+        compact = True
+    elif len(sys.argv) == 2 and sys.argv[1] in ('-a', '--all'):
+        compact = False
+    else:
+        print("Usage: %s [-a|--all]" % sys.argv[0], file=sys.stderr)
+        sys.exit(1)
+
+    colorama.init(strip=False)
+    for root in nodes():
+        tree_show(root, compact=compact)

gargantext/util/toolchain/hyperdata_indexing.py

@@ -43,8 +43,7 @@ def _nodes_hyperdata_generator(corpus):
                             key['id'],
                             None,
                             None,
-                            value.strftime("%Y-%m-%d %H:%M:%S"), 
-                            # FIXME check timestamp +%Z
+                            str(value),
                             None,
                             None,
                         )

gargantext/util/toolchain/list_map.py

@@ -12,15 +12,19 @@ from gargantext.constants     import DEFAULT_MAPLIST_MAX,\
                                      DEFAULT_MAPLIST_GENCLUSION_RATIO,\
                                      DEFAULT_MAPLIST_MONOGRAMS_RATIO
 
+def do_maplist_query():
+    return None
+
+
 def do_maplist(corpus,
-               overwrite_id = None,
-               mainlist_id  = None,
-               specclusion_id = None,
-               genclusion_id = None,
-               grouplist_id = None,
-               limit=DEFAULT_MAPLIST_MAX,
-               genclusion_part=DEFAULT_MAPLIST_GENCLUSION_RATIO,
-               monograms_part=DEFAULT_MAPLIST_MONOGRAMS_RATIO
+               overwrite_id    = None,
+               mainlist_id     = None,
+               specclusion_id  = None,
+               genclusion_id   = None,
+               grouplist_id    = None,
+               limit           = DEFAULT_MAPLIST_MAX,
+               genclusion_part = DEFAULT_MAPLIST_GENCLUSION_RATIO,
+               monograms_part  = DEFAULT_MAPLIST_MONOGRAMS_RATIO
                ):
     '''
     According to Genericity/Specificity and mainlist
@@ -28,9 +32,9 @@ def do_maplist(corpus,
     Parameters:
       - mainlist_id (starting point, already cleaned of stoplist terms)
       - specclusion_id (ngram inclusion by cooc specificity -- ranking factor)
-      - genclusion_id (ngram inclusion by cooc genericity -- ranking factor)
+      - genclusion_id  (ngram inclusion by cooc genericity  -- ranking factor)
       - grouplist_id (filtering grouped ones)
-      - overwrite_id: optional if preexisting MAPLIST node to overwrite
+      - overwrite_id: optional. Overwrite if preexisting MAPLIST node
 
       + 3 params to modulate the terms choice
         - limit for the amount of picked terms
@@ -77,6 +81,7 @@ def do_maplist(corpus,
                         )
                 .join(Ngram, Ngram.id == ScoreSpec.ngram_id)
                 .join(ScoreGen, ScoreGen.ngram_id == ScoreSpec.ngram_id)
+                
                 .filter(ScoreSpec.node_id == specclusion_id)
                 .filter(ScoreGen.node_id == genclusion_id)
 
@@ -155,10 +160,10 @@ def do_maplist(corpus,
         # at the end of the first loop we just need to sort all by the second ranker (gen)
         scored_ngrams = sorted(scored_ngrams, key=lambda ng_infos: ng_infos[2], reverse=True)
 
-    obtained_spec_mono = len(chosen_ngrams['topspec']['monograms'])
+    obtained_spec_mono  = len(chosen_ngrams['topspec']['monograms'])
     obtained_spec_multi = len(chosen_ngrams['topspec']['multigrams'])
-    obtained_gen_mono = len(chosen_ngrams['topgen']['monograms'])
-    obtained_gen_multi = len(chosen_ngrams['topgen']['multigrams'])
+    obtained_gen_mono   = len(chosen_ngrams['topgen']['monograms'])
+    obtained_gen_multi  = len(chosen_ngrams['topgen']['multigrams'])
     obtained_total = obtained_spec_mono   \
                     + obtained_spec_multi \
                     + obtained_gen_mono   \

gargantext/util/toolchain/ngram_coocs.py

@@ -9,7 +9,6 @@ from gargantext.util.db        import get_engine
 from gargantext.util.db_cache  import cache
 from gargantext.constants      import DEFAULT_COOC_THRESHOLD, NODETYPES
 from gargantext.constants      import INDEXED_HYPERDATA
-from gargantext.util.tools     import datetime, convert_to_date
 
 def compute_coocs(  corpus,
                     overwrite_id    = None,
@@ -95,7 +94,7 @@ def compute_coocs(  corpus,
 
     # 2b) stating the filters
     cooc_filter_sql = """
-        WHERE 
+        WHERE
             n.typename  = {nodetype_id}
         AND n.parent_id = {corpus_id}
         GROUP BY 1,2
@@ -105,7 +104,7 @@ def compute_coocs(  corpus,
         """.format( nodetype_id = NODETYPES.index('DOCUMENT')
                   , corpus_id=corpus.id
                   )
-    
+
     # 3) taking the cooccurrences of ngram x2
     ngram_filter_A_sql += """
         -- STEP 1: X axis of the matrix
@@ -162,25 +161,25 @@ def compute_coocs(  corpus,
     # 4) prepare the synonyms
     if groupings_id:
         ngram_filter_A_sql += """
-        LEFT JOIN  nodes_ngrams_ngrams 
-               AS grA  ON wlA.ngram_id = grA.ngram1_id 
+        LEFT JOIN  nodes_ngrams_ngrams
+               AS grA  ON wlA.ngram_id = grA.ngram1_id
                       AND grA.node_id  = {groupings_id}
         -- \--> adding (joining) ngrams that are grouped
         LEFT JOIN  nodes_ngrams
                AS wlAA ON grA.ngram2_id = wlAA.ngram_id
-                      AND wlAA.node_id  = wlA.node_id 
+                      AND wlAA.node_id  = wlA.node_id
         -- \--> adding (joining) ngrams that are not grouped
         --LEFT JOIN  ngrams        AS wlAA ON grA.ngram2_id = wlAA.id
         -- \--> for joining all synonyms even if they are not in the main list (white list)
 
         """.format(groupings_id = groupings_id)
-        
+
         ngram_filter_B_sql += """
         LEFT JOIN  nodes_ngrams_ngrams
-               AS grB  ON wlB.ngram_id = grB.ngram1_id 
+               AS grB  ON wlB.ngram_id = grB.ngram1_id
                       AND grB.node_id  = {groupings_id}
         -- \--> adding (joining) ngrams that are grouped
-        LEFT JOIN  nodes_ngrams 
+        LEFT JOIN  nodes_ngrams
                AS wlBB ON grB.ngram2_id = wlBB.ngram_id
                       AND wlBB.node_id   = wlB.node_id
         -- \--> adding (joining) ngrams that are not grouped

gargantext/util/tools.py

-import os
-from gargantext.settings import MEDIA_ROOT
-
-import datetime
-import dateutil
-
-def convert_to_date(date):
-    if isinstance(date, (int, float)):
-        return datetime.datetime.timestamp(date)
-    else:
-        return dateutil.parser.parse(date)
-
-
-def ensure_dir(user):
-    '''
-    If user is new, folder does not exist yet, create it then
-    '''
-    dirpath = '%s/corpora/%s' % (MEDIA_ROOT, user.username)
-    if not os.path.exists(dirpath):
-        print("Creating folder %s" % dirpath)
-        os.makedirs(dirpath)

gargantext/views/api/ngramlists.py

@@ -90,10 +90,11 @@ class CSVLists(APIView):
         # import the csv
         # try:
         log_msg = "Async generation"
-        
+
         corpus_node_id = corpus_node.id
-        scheduled(import_and_merge_ngramlists)(csv_contents, corpus_node_id)
-        
+        scheduled(import_and_merge_ngramlists)(csv_contents, corpus_node_id,
+                                               overwrite=bool(params.get('overwrite')))
+
         return JsonHttpResponse({
             'log': log_msg,
             }, 200)
@@ -153,7 +154,8 @@ class CSVLists(APIView):
         # attempt to merge and send response
         try:
             # merge the source_lists onto those of the target corpus
-            log_msg = merge_ngramlists(source_lists, onto_corpus=corpus_node)
+            delete = todo_lists if bool(params.get('overwrite')) else []
+            log_msg = merge_ngramlists(source_lists, onto_corpus=corpus_node, del_originals=delete)
             return JsonHttpResponse({
                 'log': log_msg,
                 }, 200)

gargantext_notebook.py

+install/notebook/gargantext_notebook.py
\ No newline at end of file

install/docker.install

+#!/bin/bash
+
+sudo apt-get update
+sudo apt-get install \
+     apt-transport-https \
+     ca-certificates \
+     curl \
+     gnupg2 \
+     software-properties-common
+
+curl -fsSL https://download.docker.com/linux/debian/gpg | sudo apt-key add -
+sudo apt-key fingerprint 0EBFCD88
+echo "Should be: Key fingerprint = 9DC8 5822 9FC7 DD38 854A  E2D8 8D81 803C 0EBF CD88"
+
+sudo add-apt-repository \
+   "deb [arch=amd64] https://download.docker.com/linux/debian \
+   $(lsb_release -cs) \
+   stable"
+
+sudo apt-get update
+sudo apt-get install docker-ce
+sudo docker run hello-world

install/gargamelle/Debian.sh

@@ -54,6 +54,7 @@ update-locale LC_ALL=fr_FR.UTF-8
  cd /srv/
  pip3 install virtualenv
  virtualenv /srv/env_3-5 -p /usr/bin/python3.5
+ echo '/srv/gargantext' > /srv/env_3-5/lib/python3.5/site-packages/gargantext.pth
  echo 'alias venv="source /srv/env_3-5/bin/activate"' >> ~/.bashrc
  # CONFIG FILES
 

install/gargamelle/Dockerfile

@@ -15,9 +15,9 @@ RUN apt-get update && \
     apt-utils ca-certificates locales \
     sudo aptitude gcc g++ wget git vim \
     build-essential make \
-    postgresql-9.5 postgresql-client-9.5 postgresql-contrib-9.5 \
-    postgresql-server-dev-9.5 libpq-dev libxml2 \
-    postgresql-9.5 postgresql-client-9.5 postgresql-contrib-9.5
+    postgresql-9.6 postgresql-client-9.6 postgresql-contrib-9.6 \
+    postgresql-server-dev-9.6 libpq-dev libxml2 \
+    postgresql-9.6 postgresql-client-9.6 postgresql-contrib-9.6
 
 
 ### Configure timezone and locale
@@ -37,7 +37,7 @@ ENV LC_ALL fr_FR.UTF-8
 ### Install main dependencies and python packages based on Debian distrib
 RUN echo "############# PYTHON DEPENDENCIES ###############"
 RUN apt-get update && apt-get install -y \
-        libxml2-dev xml-core libgfortran-5-dev \
+        libxml2-dev xml-core libgfortran-6-dev \
         libpq-dev \
         python3.5 \
         python3-dev \
@@ -47,8 +47,8 @@ RUN apt-get update && apt-get install -y \
         # python dependencies
         python3-pip \
         # for lxml
-        libxml2-dev libxslt-dev
-        #libxslt1-dev zlib1g-dev
+        libxml2-dev libxslt-dev \
+        libxslt1-dev zlib1g-dev
 
 # UPDATE AND CLEAN
 RUN apt-get update && apt-get autoclean &&\

install/gargamelle/nginx.config

@@ -51,12 +51,12 @@ server {
         # Add index.php to the list if you are using PHP
         #index index.html index.htm index.nginx-debian.html;
 
-        server_name _ stable.gargantext.org gargantext.org ;
+        server_name _ localhost ;
 
             # Django media
 
         location /media  {
-                alias /var/www/gargantext/media;  # your Django project's media files - amend as required
+                alias /srv/gargantext_media;  # your Django project's media files - amend as required
         }
 
         location /static {
@@ -72,23 +72,3 @@ server {
         access_log /var/log/nginx/access.log;
         error_log /var/log/nginx/error.log;
 }
-
-
-server {
-    listen 80 ;
-    listen [::]:80;
-    server_name dl.gargantext.org ;
-    error_page 404 /index.html;
-
-
-    location / {
-        root /var/www/dl ;
-        proxy_set_header Host $host;
-        proxy_buffering off;
-
-}
-    access_log /var/log/nginx/dl.gargantext.org-access.log;
-    error_log /var/log/nginx/dl.gargantext.org-error.log;
-}
-
-

install/gargamelle/psqlIndexes.sql

---                       ____ 
---                      / ___|
---                     | |  _ 
---                     | |_| |
---                      \____|arganTexT
-----------------------------------------------------------------------
---                  Gargantext optimization of Database             --
-----------------------------------------------------------------------
-
---> Manual optimization with indexes according to usages
--- Weakness and Strengths of indexes:
-    --> it can slow down the insertion(s)
-    --> it can speed up  the selection(s)
-
---> Conventions for this document:
-    --> indexes commented already have been created 
-    --> indexes not commented have not been created yet
-
-----------------------------------------------------------------------
--- Retrieve Nodes
-----------------------------------------------------------------------
-create INDEX on nodes (user_id, typename, parent_id) ;
-create INDEX on nodes_hyperdata (node_id, key);
-create INDEX on ngrams (id, n) ;
-create INDEX on ngrams (n) ;
-create INDEX on nodes_ngrams (node_id, ngram_id) ;
-create INDEX on nodes_ngrams (node_id) ;
-create INDEX on nodes_ngrams (ngram_id) ;
-create INDEX on nodes_ngrams_ngrams (node_id, ngram1_id, ngram2_id) ;
-
-create INDEX on nodes_ngrams_ngrams (node_id) ;
-create INDEX on nodes_ngrams_ngrams (ngram1_id) ;
-create INDEX on nodes_ngrams_ngrams (ngram2_id) ;
-
-----------------------------------------------------------------------
--- DELETE optimization of Nodes -- todo on dev
-create INDEX on nodes_nodes_ngrams (node1_id);
-create INDEX on nodes_nodes_ngrams (node2_id);
-
-create INDEX on nodes_nodes (node1_id, node2_id);
-
--- Maybe needed soon:
--- create INDEX on nodes_nodes_ngrams (node1_id, node2_id);
-----------------------------------------------------------------------
--- Analytics
-
-create INDEX on nodes_hyperdata (node_id,value_utc);  -- remove ?
-create INDEX on nodes_hyperdata (node_id,key,value_utc);
-create INDEX on nodes_hyperdata (node_id,key,value_int);
-create INDEX on nodes_hyperdata (node_id,key,value_flt);
-create INDEX on nodes_hyperdata (node_id,key,value_str);
-
-----------------------------------------------------------------------
-----------------------------------------------------------------------
-
-create index on nodes using GIN (hyperdata);
-----------------------------------------------------------------------

install/gargamelle/requirements.txt

@@ -17,7 +17,7 @@ jdatetime==1.7.2
 kombu==3.0.37                  # messaging
 langdetect==1.0.6              #detectinglanguage
 nltk==3.1
-numpy==1.10.4
+numpy==1.13.1
 psycopg2==2.6.2
 pycountry==1.20
 python-dateutil==2.4.2
@@ -33,3 +33,5 @@ lxml==3.5.0
 requests-futures==0.9.7
 bs4==0.0.1
 requests==2.10.0
+alembic>=0.9.2
+# SQLAlchemy-Searchable==0.10.4

install/notebook.install

+#!/bin/bash
+
+sudo adduser --disabled-password --gecos "" notebooks
+sudo docker rm $(sudo docker ps -a | grep sh | awk '{print $1}')
+sudo docker build -t garg-notebook:latest ./notebook

install/notebook.run

+#!/bin/bash
+
+        #-v /srv/gargandata:/srv/gargandata \
+        #-v /srv/gargantext_lib:/srv/gargantext_lib \
+
+sudo docker rm $(sudo docker ps -a | grep notebook | grep sh | awk '{print $1}')
+
+#HOSTIP=$(ip route show 0.0.0.0/0 | awk '{print $3}')
+#--add-host=localhost:${HOSTIP} \
+
+
+sudo docker run \
+        --name=garg-notebook \
+        --net=host \
+        -p 8899:8899 \
+        --env POSTGRES_HOST=localhost \
+        -v /srv/gargantext:/srv/gargantext \
+        -it garg-notebook:latest \
+        /bin/bash -c "/bin/su notebooks -c 'source /env_3-5/bin/activate && cd /home/notebooks && jupyter notebook --port=8899 --ip=0.0.0.0 --no-browser'"
+#        #&& jupyter nbextension enable --py widgetsnbextension --sys-prefix 
+        #/bin/bash -c "/bin/su notebooks -c 'source /env_3-5/bin/activate && cd /srv/gargantext/ && jupyter notebook --port=8899 --ip=0.0.0.0 --no-browser --notebook-dir=/home/notebooks/'"
+
+

install/notebook/Dockerfile

+###########################################################
+# Gargamelle WEB
+###########################################################
+#Build an image starting with debian:stretch image
+# wich contains all the source code of the app
+FROM debian:stretch
+MAINTAINER ISCPIF <gargantext@iscpif.fr>
+
+USER root
+
+
+### Update and install base dependencies
+RUN echo "############ DEBIAN LIBS ###############"
+RUN apt-get update &&                                     \
+    apt-get install -y                                     \
+    apt-utils ca-certificates locales                       \
+    sudo aptitude gcc g++ wget git vim                       \
+    build-essential make                                      \
+    curl                                                       
+#    postgresql-9.6 postgresql-client-9.6 postgresql-contrib-9.6 \
+#    postgresql-server-dev-9.6 libpq-dev libxml2                  \
+#    postgresql-9.6 postgresql-client-9.6 postgresql-contrib-9.6
+
+# Install Stack
+
+### Configure timezone and locale
+RUN echo "###########  LOCALES & TZ #################"
+RUN echo "Europe/Paris" > /etc/timezone
+ENV TZ "Europe/Paris"
+
+RUN sed -i -e 's/# en_GB.UTF-8 UTF-8/en_GB.UTF-8 UTF-8/' /etc/locale.gen && \
+    sed -i -e 's/# fr_FR.UTF-8 UTF-8/fr_FR.UTF-8 UTF-8/' /etc/locale.gen &&  \
+    dpkg-reconfigure --frontend=noninteractive locales                   &&   \
+    echo 'LANG="fr_FR.UTF-8"' > /etc/default/locale
+ENV LANG fr_FR.UTF-8
+ENV LANGUAGE fr_FR.UTF-8
+ENV LC_ALL fr_FR.UTF-8
+
+
+### Install main dependencies and python packages based on Debian distrib
+RUN echo "############# PYTHON DEPENDENCIES ###############"
+RUN apt-get update && apt-get install -y       \
+        libxml2-dev xml-core libgfortran-6-dev  \
+        libpq-dev                                \
+        python3.5                                 \
+        python3-dev                                \
+        # for numpy, pandas and numpyperf           \
+        python3-six python3-numpy python3-setuptools \
+        python3-numexpr                               \
+        # python dependencies                          \
+        python3-pip                                     \
+        # for lxml
+        libxml2-dev libxslt-dev libxslt1-dev zlib1g-dev
+
+
+# UPDATE AND CLEAN
+RUN apt-get update && apt-get autoclean \ 
+    && rm -rf /var/lib/apt/lists/*
+#NB: removing /var/lib will avoid to significantly fill up your /var/ folder on your native system
+
+########################################################################
+### PYTHON ENVIRONNEMENT (as ROOT)
+########################################################################
+
+RUN adduser --disabled-password --gecos "" notebooks
+
+RUN pip3 install virtualenv
+RUN virtualenv /env_3-5
+RUN echo 'alias venv="source /env_3-5/bin/activate"' >> ~/.bashrc
+# CONFIG FILES
+ADD requirements.txt /
+#ADD psql_configure.sh /
+ADD django_configure.sh /
+
+RUN . /env_3-5/bin/activate && pip3 install -r requirements.txt                         && \
+    pip3  install git+https://github.com/zzzeek/sqlalchemy.git@rel_1_1                  && \
+    python3 -m nltk.downloader averaged_perceptron_tagger -d /usr/local/share/nltk_data
+
+#RUN ./psql_configure.sh
+#RUN ./django_configure.sh
+RUN chown notebooks:notebooks -R /env_3-5
+
+########################################################################
+### POSTGRESQL DATA (as ROOT)
+########################################################################
+
+#RUN sed -iP "s%^data_directory.*%data_directory = \'\/srv\/gargandata\'%" /etc/postgresql/9.5/main/postgresql.conf
+#RUN echo "host all  all    0.0.0.0/0  md5" >> /etc/postgresql/9.5/main/pg_hba.conf
+#RUN echo "listen_addresses='*'" >> /etc/postgresql/9.5/main/postgresql.conf
+
+EXPOSE 5432 8899
+
+VOLUME ["/srv/","/home/notebooks/"]
+
+
+########################################################################
+### Notebook IHaskell and IPYTHON ENVIRONNEMENT
+########################################################################
+
+RUN apt-get update && apt-get install -y \
+        libtinfo-dev                      \
+        libzmq3-dev                        \
+        libcairo2-dev                       \
+        libpango1.0-dev                      \
+        libmagic-dev                          \
+        libblas-dev                            \
+        liblapack-dev
+
+USER notebooks
+
+RUN cd  /home/notebooks                              \
+    &&  curl -sSL https://get.haskellstack.org/ | sh  \
+    &&  stack setup                                    \
+    &&  git clone https://github.com/gibiansky/IHaskell \
+    &&  . /env_3-5/bin/activate                          \
+    &&  cd IHaskell                                       \
+    &&  stack install gtk2hs-buildtools                    \
+    &&  stack install --fast                                \
+    &&  /root/.local/bin/ihaskell install --stack
+
+
+

install/notebook/django_configure.sh

+#!/bin/bash
+##################################################
+#             __| |(_) __ _ _ __   __ _  ___
+#            / _` || |/ _` | '_ \ / _` |/ _ \
+#           | (_| || | (_| | | | | (_| | (_) |
+#            \__,_|/ |\__,_|_| |_|\__, |\___/
+#                 |__/             |___/
+##################################################
+#configure django migrations
+##################################################
+echo "::::: DJANGO :::::"
+#echo "Starting Postgres"
+#/usr/sbin/service postgresql start
+
+
+
+su gargantua -c 'source /srv/env_3-5/bin/activate &&\
+    echo "Activated env" &&\
+    /srv/gargantext/manage.py makemigrations &&\
+    /srv/gargantext/manage.py migrate && \
+    echo "migrations ok" &&\
+    /srv/gargantext/dbmigrate.py && \
+    /srv/gargantext/dbmigrate.py && \
+    /srv/gargantext/dbmigrate.py && \
+    /srv/gargantext/manage.py createsuperuser'
+
+service postgresql stop

install/notebook/gargantext_notebook.py

+#!/usr/bin/env python
+"""
+   Gargantext Software Copyright (c) 2016-2017 CNRS ISC-PIF -
+http://iscpif.fr
+    Licence (see :
+http://gitlab.iscpif.fr/humanities/gargantext/blob/stable/LICENSE )
+    - In France : a CECILL variant affero compliant
+    - GNU aGPLV3 for all other countries
+"""
+
+import os
+import django
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'gargantext.settings')
+django.setup()
+
+from gargantext.constants import QUERY_SIZE_N_MAX, get_resource, get_resource_by_name
+from gargantext.models import ProjectNode, DocumentNode, UserNode, User
+from gargantext.util.db import session, get_engine
+from collections import Counter
+import importlib
+from django.http import Http404
+
+
+class NotebookError(Exception):
+    pass
+
+
+def documents(corpus_id):
+    return (session.query(DocumentNode).filter_by(parent_id=corpus_id)
+                  #.order_by(Node.hyperdata['publication_date'])
+                   .all())
+
+
+#import seaborn as sns
+import pandas as pd
+
+def chart(docs, field):
+    year_publis = list(Counter([doc.hyperdata[field] for doc in docs]).items())
+    frame0 = pd.DataFrame(year_publis, columns=['Date', 'DateValue'])
+    frame1 = pd.DataFrame(year_publis, columns=['Date', 'DateValue'], index=frame0.Date)
+    return frame1
+
+
+from gargantext.util.crawlers.HAL import HalCrawler
+
+def scan_hal(request):
+    hal = HalCrawler()
+    return hal.scan_results(request)
+
+
+def scan_gargantext(corpus_id, lang, request):
+    connection = get_engine().connect()
+    # TODO add some sugar the request (ideally request should be the same for hal and garg)
+    query = """select count(n.id) from nodes n
+                  where to_tsvector('%s', hyperdata ->> 'abstract' || 'title')
+                  @@ to_tsquery('%s')
+                  AND n.parent_id = %s;""" % (lang, request, corpus_id)
+    return [i for i in connection.execute(query)][0][0]
+    connection.close()
+
+
+def myProject_fromUrl(url):
+    """
+    myProject :: String -> Project
+    """
+    project_id = url.split("/")[4]
+    project = session.query(ProjectNode).get(project_id)
+    return project
+
+
+def newCorpus(project, source, name=None, query=None):
+    error = False
+
+    if name is None:
+        name = query
+
+    if not isinstance(project, ProjectNode):
+        error = "a valid project"
+    if not isinstance(source, int) and not isinstance(source, str):
+        error = "a valid source identifier: id or name"
+    elif not isinstance(query, str):
+        error = "a valid query"
+    elif not isinstance(name, str):
+        error = "a valid name"
+
+    if error:
+        raise NotebookError("Please provide %s." % error)
+
+    resource = get_resource(source) if isinstance(source, int) else \
+               get_resource_by_name(source)
+
+    moissonneur_name = get_moissonneur_name(resource) if resource else \
+                       source.lower()
+
+    try:
+        moissonneur = get_moissonneur(moissonneur_name)
+    except ImportError:
+        raise NotebookError("Invalid source identifier: %r" % source)
+
+    return run_moissonneur(moissonneur, project, name, query)
+
+
+def get_moissonneur_name(ident):
+    """ Return moissonneur module name from RESOURCETYPE or crawler name """
+
+    # Does it quacks like a RESOURCETYPE ?
+    if hasattr(ident, 'get'):
+        ident = ident.get('crawler')
+
+    # Extract name from crawler class name, otherwise assume ident is already
+    # a moissonneur name.
+    if isinstance(ident, str) and ident.endswith('Crawler'):
+        return ident[:-len('Crawler')].lower()
+
+
+def get_moissonneur(name):
+    """ Return moissonneur module from its name """
+    if not isinstance(name, str) or not name.islower():
+        raise NotebookError("Invalid moissonneur name: %r" % name)
+
+    module = importlib.import_module('moissonneurs.%s' % name)
+    module.name = name
+
+    return module
+
+
+def run_moissonneur(moissonneur, project, name, query):
+    """ Run moissonneur and return resulting corpus """
+
+    # XXX Uber-kludge with gory details. Spaghetti rulezzzzz!
+    class Dummy(object):
+        pass
+
+    request = Dummy()
+    request.method = 'POST'
+    request.path = 'nowhere'
+    request.META = {}
+    # XXX 'string' only have effect on moissonneurs.pubmed; its value is added
+    #     when processing request client-side, take a deep breath and see
+    #     templates/projects/project.html for more details.
+    request.POST = {'string': name,
+                    'query': query,
+                    'N': QUERY_SIZE_N_MAX}
+    request.user = Dummy()
+    request.user.id = project.user_id
+    request.user.is_authenticated = lambda: True
+
+    if moissonneur.name == 'istex':
+        # Replace ALL spaces by plus signs
+        request.POST['query'] = '+'.join(filter(None, query.split(' ')))
+
+    try:
+        import json
+
+        r = moissonneur.query(request)
+        raw_json = r.content.decode('utf-8')
+        data = json.loads(raw_json)
+
+        if moissonneur.name == 'pubmed':
+            count = sum(x['count'] for x in data)
+            request.POST['query'] = raw_json
+        elif moissonneur.name == 'istex':
+            count = data.get('total', 0)
+        else:
+            count = data.get('results_nb', 0)
+
+        if count > 0:
+            corpus = moissonneur.save(request, project.id, return_corpus=True)
+        else:
+            return None
+
+    except (ValueError, Http404) as e:
+        raise e
+
+    # Sometimes strange things happens...
+    if corpus.name != name:
+        corpus.name = name
+        session.commit()
+
+    return corpus

install/notebook/psql_configure.sh

+#!/bin/bash
+
+#######################################################################
+##    ____           _
+##   |  _ \ ___  ___| |_ __ _ _ __ ___  ___
+##   | |_) / _ \/ __| __/ _` | '__/ _ \/ __|
+##   |  __/ (_) \__ \ || (_| | | |  __/\__ \
+##   |_|   \___/|___/\__\__, |_|  \___||___/
+##                      |___/
+#######################################################################
+echo "::::: POSTGRESQL :::::"
+su postgres -c 'pg_dropcluster 9.4 main --stop'
+#done in docker but redoing it
+rm -rf /srv/gargandata && mkdir /srv/gargandata && chown postgres:postgres /srv/gargandata
+su postgres -c '/usr/lib/postgresql/9.6/bin/initdb -D /srv/gargandata/'
+su postgres -c '/usr/lib/postgresql/9.6/bin/pg_ctl -D /srv/gargandata/ -l /srv/gargandata/journal_applicatif start'
+
+su postgres -c 'pg_createcluster -D /srv/gargandata 9.6 main '
+su postgres -c 'pg_ctlcluster -D /srv/gargandata 9.6 main start '
+su postgres -c 'pg_ctlcluster 9.6 main start'
+
+service postgresql start
+
+su postgres -c "psql -c \"CREATE user gargantua WITH PASSWORD 'C8kdcUrAQy66U'\""
+su postgres -c "createdb -O gargantua gargandb"
+
+echo "Postgres configured"
+#service postgresql stop

install/notebook/requirements.txt

+# try bottleneck
+eventlet==0.20.1
+amqp==1.4.9
+anyjson==0.3.3
+billiard==3.3.0.23
+celery==3.1.25
+chardet==2.3.0
+dateparser==0.3.5
+Django==1.10.5
+django-celery==3.2.1
+django-pgfields==1.4.4
+django-pgjsonb==0.0.23
+djangorestframework==3.5.3
+html5lib==0.9999999
+#python-igraph>=0.7.1
+jdatetime==1.7.2
+kombu==3.0.37                  # messaging
+langdetect==1.0.6              #detectinglanguage
+nltk==3.1
+numpy==1.13.1
+psycopg2==2.6.2
+pycountry==1.20
+python-dateutil==2.4.2
+pytz==2016.10                   # timezones
+PyYAML==3.11
+RandomWords==0.1.12
+ujson==1.35
+umalqurra==0.2                 # arabic calendars (?? why use ??)
+networkx==1.11
+pandas==0.18.0
+six==1.10.0
+lxml==3.5.0
+requests-futures==0.9.7
+bs4==0.0.1
+requests==2.10.0
+djangorestframework-jwt==1.9.0
+jupyter==1.0.0
+jupyter-client==5.0.0
+jupyter-console==5.1.0
+jupyter-core==4.3.0
+ipython==5.2.0
+ipython-genutils==0.1.0
+ipywidgets
+matplotlib==2.0.2

moissonneurs/cern.py

@@ -30,7 +30,7 @@ def query( request):
             #ids = crawlerbot.get_ids(query)
             return JsonHttpResponse({"results_nb":crawlerbot.results_nb})
 
-def save(request, project_id):
+def save(request, project_id, return_corpus=False):
     '''save'''
     if request.method == "POST":
 
@@ -101,6 +101,9 @@ def save(request, project_id):
             session.rollback()
             # --------------------------------------------
 
+        if return_corpus:
+            return corpus
+
         return render(
             template_name = 'pages/projects/wait.html',
             request = request,

moissonneurs/hal.py

@@ -33,7 +33,7 @@ def query( request):
             print(results)
             return JsonHttpResponse({"results_nb":crawlerbot.results_nb})
 
-def save(request, project_id):
+def save(request, project_id, return_corpus=False):
     '''save'''
     if request.method == "POST":
 
@@ -103,6 +103,9 @@ def save(request, project_id):
             session.rollback()
             # --------------------------------------------
 
+        if return_corpus:
+            return corpus
+
         return render(
             template_name = 'pages/projects/wait.html',
             request = request,

moissonneurs/isidore.py

@@ -29,7 +29,7 @@ def query( request):
             #ids = crawlerbot.get_ids(query)
             return JsonHttpResponse({"results_nb":crawlerbot.results_nb})
 
-def save(request, project_id):
+def save(request, project_id, return_corpus=False):
     '''save'''
     if request.method == "POST":
 
@@ -100,6 +100,9 @@ def save(request, project_id):
             session.rollback()
             # --------------------------------------------
 
+        if return_corpus:
+            return corpus
+
         return render(
             template_name = 'pages/projects/wait.html',
             request = request,

moissonneurs/istex.py

@@ -52,7 +52,7 @@ def query( request ):
 
 
 
-def save(request , project_id):
+def save(request , project_id, return_corpus=False):
     print("testISTEX:")
     print(request.method)
     alist = ["bar","foo"]
@@ -171,6 +171,9 @@ def save(request , project_id):
             session.rollback()
             # --------------------------------------------
 
+        if return_corpus:
+            return corpus
+
         return render(
             template_name = 'pages/projects/wait.html',
             request = request,

moissonneurs/multivac.py

@@ -33,7 +33,7 @@ def query( request):
             print(results)
             return JsonHttpResponse({"results_nb":crawlerbot.results_nb})
 
-def save(request, project_id):
+def save(request, project_id, return_corpus=False):
     '''save'''
     if request.method == "POST":
 
@@ -104,6 +104,9 @@ def save(request, project_id):
             session.rollback()
             # --------------------------------------------
 
+        if return_corpus:
+            return corpus
+
         return render(
             template_name = 'pages/projects/wait.html',
             request = request,

moissonneurs/pubmed.py

@@ -69,7 +69,7 @@ def query( request ):
     return JsonHttpResponse(data)
 
 
-def save( request , project_id ) :
+def save( request , project_id, return_corpus=False ) :
     # implicit global session
     # do we have a valid project id?
     try:
@@ -164,6 +164,10 @@ def save( request , project_id ) :
             session.rollback()
             # --------------------------------------------
         sleep(1)
+
+        if return_corpus:
+            return corpus
+
         return HttpResponseRedirect('/projects/' + str(project_id))
 
     data = alist

templates/pages/corpora/terms.html

@@ -250,6 +250,23 @@ em {
 
             <br/>
 
+            <div class="checkbox">
+                <label>
+                    <input type="checkbox" id="importoverwrite"> Overwrite old lists
+                    <script>
+                        function updateSubmitLabel() {
+                            $('#importsubmit').val($(this).is(':checked') ? 'Overwrite current table' : 'Import and merge with current table');
+                        }
+                        $(function() {
+                            updateSubmitLabel.call($('#importoverwrite'));
+                            $('#importoverwrite').change(updateSubmitLabel);
+                        });
+                    </script>
+                </label>
+            </div>
+
+            <br/>
+
             <input type="submit" class="btn btn-xs btn-info" id="importsubmit" value="Import and merge with current table" />
         </form>
     </div>
@@ -372,6 +389,8 @@ function listmergeUpdate(aFormData){
     // all params are added in the url like a GET
     theUrl += "&from_corpus="+sourceCorpusId
     theUrl += "&todo="+todoLists.join(',')
+    if ($('#importoverwrite').is(':checked'))
+        theUrl += "&overwrite=1"
 
     // result url looks like this : /api/ngramlists/import?onto_corpus=2&from=13308&todo=map,stop
     // console.log(theUrl)
@@ -424,7 +443,7 @@ function listmergeCsvPost(theFile){
 
         //postCorpusFile
         $.ajax({
-             url: "{{importroute | safe}}",
+             url: "{{importroute | safe}}" + ($('#importoverwrite').is(':checked') ? '&overwrite=1' : ''),
              type: 'POST',
              async: true,
              contentType: false,
@@ -436,11 +455,11 @@ function listmergeCsvPost(theFile){
              success: function(response) {
                  my_html  = '<h3 style="color:green">File upload, you will receive a notification email</h3>'
                  my_html += "<p class='note'>" + response['log'].replace(/\n/g, '<br/>') + "</p>"
-                 my_html += "<p'>(this page will reload in 3s)</p>"
+                 my_html += "<p'>(this page will reload in 30s)</p>"
                  $('#formanswer').html(my_html);
                  console.log(response) ;
                  // reload after 3s
-                 setTimeout("location.reload(true)", 3000);
+                 setTimeout("location.reload(true)", 30000);
                  },
               error: function(result, t) {
                   if (t != 'timeout') {