[ADMIN] Merge into prod-dev with table renames.

analysis/diachronic_specificity.py

@@ -13,8 +13,8 @@ NodeNgram = models.Node_Ngram.sa
 NodeNodeNgram = models.NodeNodeNgram.sa
 NodeNgramNgram = models.NodeNgramNgram.sa
 Ngram = models.Ngram.sa
-Node_Metadata = models.Node_Metadata.sa
-Metadata = models.Metadata.sa
+Node_Hyperdata = models.Node_Hyperdata.sa
+Hyperdata = models.Hyperdata.sa
 Node = models.Node.sa
 Corpus = models.Corpus.sa
 
@@ -50,18 +50,18 @@ def diachronic_specificity(corpus_id, terms, order=True):
     Nowadays, the measure is rather simple: distance of frequency of period from mean of frequency of all corpus.
     '''
     ngram_frequency_query = (session
-        .query(Node.metadata['publication_year'], func.count('*'))
+        .query(Node.hyperdata['publication_year'], func.count('*'))
         .join(NodeNgram, Node.id == NodeNgram.node_id)
         .join(Ngram, Ngram.id == NodeNgram.ngram_id)
         .filter(Ngram.terms == terms)
         .filter(Node.parent_id == corpus_id)
-        .group_by(Node.metadata['publication_year'])
+        .group_by(Node.hyperdata['publication_year'])
     )
 
     document_year_sum_query = (session
-        .query(Node.metadata['publication_year'], func.count('*'))
+        .query(Node.hyperdata['publication_year'], func.count('*'))
         .filter(Node.parent_id == corpus_id)
-        .group_by(Node.metadata['publication_year'])
+        .group_by(Node.hyperdata['publication_year'])
     )
             
             

analysis/functions.py

@@ -21,7 +21,7 @@ def create_synonymes(user, corpus):
 
 size = 1000 
 
-def create_whitelist(user, corpus_id, size=size):
+def create_whitelist(user, corpus_id, size=size, count_min=2):
     cursor = connection.cursor()
     
     whitelist_type_id = cache.NodeType['WhiteList'].id
@@ -66,13 +66,13 @@ def create_whitelist(user, corpus_id, size=size):
         GROUP BY
             ngX.id
         Having
-            COUNT(*) >= 3
+            COUNT(*) >= %d
         ORDER BY
             occurrences DESC
         LIMIT
             %d
         ;
-    """  % (white_list.id, int(corpus_id), int(type_document_id), size)
+    """  % (white_list.id, int(corpus_id), int(type_document_id), count_min, size)
     # print("PRINTING QYERY OF WHITELIST:")
     # print(query_whitelist)
     cursor.execute(query_whitelist)
@@ -192,8 +192,7 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size=
 
         x = pd.DataFrame(matrix).fillna(0)
         y = pd.DataFrame(matrix).fillna(0)
-#    x = copy(df.values)
-#    y = copy(df.values)
+        
         #xo = diag_null(x)
         #y = diag_null(y)
         
@@ -205,23 +204,30 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size=
         ys = x.sum(axis=0) - x
     
         # top inclus ou exclus
-        #n = ( xs + ys) / (2 * (x.shape[0] -1))
+        n = ( xs + ys) / (2 * (x.shape[0] - 1))
         # top generic or specific
-        m = ( xs - ys) / (2 * (x.shape[0] -1))
-        #m = pd.DataFrame.abs(m)
+        m = ( xs - ys) / (2 * (x.shape[0] - 1))
         
-        #n = n.sort(inplace=False)
+        n = n.sort(inplace=False)
         m = m.sort(inplace=False)
         
-        matrix_size = int(round(size/5,0))
-        # TODO user the generic score for the node size
-        #n_index = pd.Index.intersection(x.index, n.index[-matrix_size:])
+        print(n)
+        print(m)
+        
+        nodes_included = 300 #int(round(size/20,0))
+        #nodes_excluded = int(round(size/10,0))
+        
+        nodes_specific = 300 #int(round(size/10,0))
+        #nodes_generic = int(round(size/10,0))
+        
+        # TODO user the included score for the node size
+        n_index = pd.Index.intersection(x.index, n.index[:nodes_included])
         # Generic: 
-        #m_index = pd.Index.intersection(x.index, m.index[:matrix_size])
+        #m_index = pd.Index.intersection(x.index, m.index[:nodes_generic])
         # Specific: 
-        m_index = pd.Index.intersection(x.index, m.index[-matrix_size:])
+        m_index = pd.Index.intersection(x.index, m.index[-nodes_specific:])
         
-        x_index = m_index# pd.Index.union(n_index, m_index)
+        x_index = pd.Index.union(n_index, m_index)
         xx = x[list(x_index)].T[list(x_index)]
 
         # import pprint
@@ -241,11 +247,10 @@ def get_cooc(request=None, corpus_id=None, cooc_id=None, type='node_link', size=
         G = nx.relabel_nodes(G, dict(enumerate([ labels[label] for label in list(xx.columns)])))
         
         #print(G)
-        #G = nx.relabel_nodes(G, dict(enumerate(df.columns)))
         # Removing too connected nodes (find automatic way to do it)
-        #    outdeg = G.degree()
-        #    to_remove = [n for n in outdeg if outdeg[n] >= 10]
-        #    G.remove_nodes_from(to_remove)
+        degree = G.degree()
+        to_remove = [n for n in degree if degree[n] <= 1]
+        G.remove_nodes_from(to_remove)
 
         partition = best_partition(G)
     except:

gargantext_web/celery.py

 # -*- coding: utf-8 -*-
 
-
-#import os
-#import djcelery
-#
-#from celery import Celery
-#
-#from django.conf import settings
-#
-## set the default Django settings module for the 'celery' program.
-#os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'gargantext_web.settings')
-#
-#app = Celery('gargantext_web')
-#
-#
-#app.conf.update(
-#    CELERY_RESULT_BACKEND='djcelery.backends.database:DatabaseBackend',
-#)
-#
-#
-#app.conf.update(
-#    CELERY_RESULT_BACKEND='djcelery.backends.cache:CacheBackend',
-#)
-#
-## Using a string here means the worker will not have to
-## pickle the object when using Windows.
-##app.config_from_object('django.conf:settings')
-#app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
-#
-
-from admin.utils import PrintException
-
 from celery import shared_task
 from node import models
 
@@ -49,40 +18,30 @@ def apply_sum(x, y):
 
 from parsing.corpustools import add_resource, parse_resources, extract_ngrams, compute_tfidf
 
+from admin.utils import PrintException
+
+def update_processing(corpus, step=0):
+    try:
+        corpus.hyperdata.update({'Processing' : step})
+        session.query(Node).filter(Node.id==corpus.id).update({'hyperdata' : corpus.hyperdata})
+        session.commit()
+    except :
+        PrintException()
 
 @shared_task
 def apply_workflow(corpus_id):
     corpus = session.query(Node).filter(Node.id==corpus_id).first()
 
+    update_processing(corpus, 1)
     parse_resources(corpus)
     
-    try:
-        print("-" *60)
-        
-        # With Django ORM 
-        corpus_django = models.Node.objects.get(id=corpus_id)
-        corpus_django.metadata['Processing'] = "2"
-        corpus_django.save()
-        print("-" *60)
-        
-        #TODO With SLA ORM (KO why?)
-#        corpus.metadata['Processing'] = 0
-#        session.add(corpus)
-#        session.flush()
-
-    except :
-        PrintException()
-
-       
-    #extract_ngrams(corpus, ['title',])
+    update_processing(corpus, 2)
     extract_ngrams(corpus, ['title', 'abstract'])
+    
+    update_processing(corpus, 3)
     compute_tfidf(corpus)
     
-    try:
-        corpus_django.metadata['Processing'] = 0
-        corpus_django.save()
-    except :
-        PrintException()
+    update_processing(corpus, 0)
 
 
 

gargantext_web/db.py

@@ -2,7 +2,6 @@ from gargantext_web import settings
 from node import models
 
 
-
 __all__ = ['literalquery', 'session', 'cache', 'Session', 'bulk_insert', 'engine', 'get_cursor']
 
 
@@ -45,7 +44,7 @@ def model_repr(modelname):
 # map the Django models found in node.models to SQLAlchemy models
 
 for model_name, model in models.__dict__.items():
-    if hasattr(model, '_meta') :
+    if hasattr(model, '_meta'):
         table_name = model._meta.db_table
         if hasattr(Base.classes, table_name):
             sqla_model = getattr(Base.classes, table_name)
@@ -64,15 +63,19 @@ from sqlalchemy.schema import Column, ForeignKey
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import relationship, aliased
 
-# class Node(Base):
+#class Node(Base):
 #     __tablename__ = 'node_node'
+#     __table_args__ = {'auto_load':True, 'extend_existing':True}
 #     id = Column(Integer, primary_key=True)
 #     user_id = Column(Integer, ForeignKey('auth_user.id', ondelete='CASCADE'), index=True, nullable=False)
 #     type_id = Column(Integer, ForeignKey('node_nodetype.id', ondelete='CASCADE'), index=True, nullable=False)
 #     name = Column(String(255))
 #     language_id = Column(Integer, ForeignKey('node_language.id', ondelete='CASCADE'), index=True, nullable=False)
 #     date = Column(DateTime(), default=datetime.utcnow, nullable=True)
-#     metadata = Column(JSONB, default={}, nullable=False)
+#     hyperdata = Column(JSONB, default={}, nullable=False)
+#
+#     def __repr__(self):
+#        return '<Id %r>' % self.id
 
 
 # debugging tool, to translate SQLAlchemy queries to string

gargantext_web/settings.py

@@ -52,7 +52,7 @@ SECRET_KEY = 'bt)3n9v&a02cu7^^=+u_t2tmn8ex5fvx8$x4r*j*pb1yawd+rz'
 DEBUG = False
 MAINTENANCE = False
 
-TEMPLATE_DEBUG = True
+TEMPLATE_DEBUG = False
 
 
 TEMPLATE_DIRS = (
@@ -70,17 +70,17 @@ TEMPLATE_DIRS = (
 
 
 #ALLOWED_HOSTS = ['*',]
-ALLOWED_HOSTS = ['localhost', 
-                'gargantext.org', 
-                'stable.gargantext.org', 
-                'dev.gargantext.org', 
-                'iscpif.gargantext.org', 
-                'gargantext.iscpif.fr', 
-                'mines.gargantext.org', 
-                'pasteur.gargantext.org', 
-                'beta.gargantext.org', 
-                'garg-dev.iscpif.fr', 
-                'garg-stable.iscpif.fr', 
+ALLOWED_HOSTS = ['localhost',
+                'gargantext.org',
+                'stable.gargantext.org',
+                'dev.gargantext.org',
+                'iscpif.gargantext.org',
+                'gargantext.iscpif.fr',
+                'mines.gargantext.org',
+                'pasteur.gargantext.org',
+                'beta.gargantext.org',
+                'garg-dev.iscpif.fr',
+                'garg-stable.iscpif.fr',
                 ]
 
 

gargantext_web/urls.py

@@ -39,8 +39,6 @@ urlpatterns = patterns('',
     url(r'^project/(\d+)/corpus/(\d+)/$', views.corpus),
     url(r'^project/(\d+)/corpus/(\d+)/corpus.csv$', views.corpus_csv),
     url(r'^project/(\d+)/corpus/(tests_mvc_listdocuments+)/corpus.tests_mvc_listdocuments$', views.corpus_csv),
-    
-    url(r'^project/(\d+)/corpus/(\d+)/timerange/(\d+)/(\d+)$', views.subcorpus),
 
     # Visualizations
     url(r'^project/(\d+)/corpus/(\d+)/chart$', views.chart),
@@ -61,13 +59,11 @@ urlpatterns = patterns('',
     url(r'^api/nodes$', gargantext_web.api.NodesList.as_view()),
     url(r'^api/nodes/(\d+)$', gargantext_web.api.Nodes.as_view()),
     url(r'^api/nodes/(\d+)/children/ngrams$', gargantext_web.api.NodesChildrenNgrams.as_view()),  # => repeated children ?
-    url(r'^api/nodes/(\d+)/children/metadata$', gargantext_web.api.NodesChildrenMetatadata.as_view()),
+    url(r'^api/nodes/(\d+)/children/hyperdata$', gargantext_web.api.NodesChildrenMetatadata.as_view()),
     url(r'^api/nodes/(\d+)/children/queries$', gargantext_web.api.NodesChildrenQueries.as_view()),
     url(r'^api/nodes/(\d+)/children/duplicates$', gargantext_web.api.NodesChildrenDuplicates.as_view()),
     # url(r'^api/nodes/(\d+)/children/duplicates/delete$', gargantext_web.api.NodesChildrenDuplicates.delete ),
 
-    url(r'^api/project/(\d+)/corpus/(\d+)/timerange/(\d+)/(\d+)$', views.subcorpusJSON),
-
     url(r'^api/nodes/(\d+)/ngrams$', gargantext_web.api.CorpusController.ngrams),
 
     # Provisory tests
@@ -79,8 +75,9 @@ urlpatterns = patterns('',
     url(r'^tests/istextquery$', pubmedscrapper.getGlobalStatsISTEXT), # api/query?type=istext ?
     url(r'^tests/pubmedquery$', pubmedscrapper.getGlobalStats),
     url(r'^tests/project/(\d+)/pubmedquery/go$', pubmedscrapper.doTheQuery),
-    url(r'^tests/project/(\d+)/ISTEXquery/go$', pubmedscrapper.testISTEX)
-
+    url(r'^tests/project/(\d+)/ISTEXquery/go$', pubmedscrapper.testISTEX),
+    url(r'^tests/paginator/corpus/(\d+)/$', views.newpaginatorJSON),
+    url(r'^tests/move2trash/$' , views.move_to_trash_multiple )
 )
 
 

gargantext_web/urls.py~

-from django.conf.urls import patterns, include, url
-
-from django.contrib import admin
-from django.contrib.auth.views import login
-
-from gargantext_web import views
-
-import gargantext_web.api
-
-
-admin.autodiscover()
-
-urlpatterns = patterns('',
-    
-    # Admin views
-    url(r'^admin/', include(admin.site.urls)),
-    url(r'^login/', include(admin.site.urls)),
-    url(r'^grappelli/', include('grappelli.urls')),
-    
-    # User views
-    url(r'^$', views.home),
-    
-    url(r'^projects/$', views.projects),
-    url(r'^project/(\d+)/delete/$', views.delete_project),
-    
-    url(r'^project/(\d+)/$', views.project),
-    
-    url(r'^project/(\d+)/corpus/(\d+)/$', views.corpus),
-    url(r'^project/(\d+)/corpus/(\d+)/delete/$', views.delete_corpus),
-    
-    # Visualizations
-    url(r'^corpus/(\d+)/explorer$', views.explorer_graph),
-    url(r'^corpus/(\d+)/matrix$', views.explorer_matrix),
-    
-    # Getting data
-    url(r'^chart/corpus/(\d+)/data.csv$', views.send_csv),
-    url(r'^corpus/(\d+)/node_link.json$', views.node_link),
-    url(r'^corpus/(\d+)/adjacency.json$', views.adjacency),
-
-    url(r'^api$', gargantext_web.api.Root),
-    url(r'^api/nodes/(\d+)/children/metadata$', gargantext_web.api.NodesChildrenMetatadata.as_view()),
-    url(r'^api/nodes/(\d+)/children/queries$', gargantext_web.api.NodesChildrenQueries.as_view()),
-    
-    #url(r'^api/nodes$', gargantext_web.api.NodesController.get),
-    url(r'^api/nodes/(\d+)/ngrams$', gargantext_web.api.CorpusController.ngrams),
-    url(r'^api/nodes/(\d+)/data$', gargantext_web.api.CorpusController.data),
-
-    url(r'^graph-it$', views.graph_it),
-    url(r'^ngrams$', views.ngrams),
-)
-
-
-from django.conf import settings
-if settings.DEBUG:
-    urlpatterns += patterns('',
-        url(r'^media/(?P<path>.*)$', 'django.views.static.serve', {
-            'document_root': settings.MEDIA_ROOT,
-        }),
-        url(r'^static/(?P<path>.*)$', 'django.views.static.serve', {
-            'document_root': settings.STATIC_ROOT,
-        }),
-)
-

gargantext_web/views_optimized.py

@@ -59,7 +59,7 @@ def project(request, project_id):
     #  ... sqlalchemy.func by Resource.type_id is the guilty
     # ISSUE L51
     corpus_query = (session
-        .query(Node.id, Node.name, func.count(ChildrenNode.id), Node.metadata['Processing'])
+        .query(Node.id, Node.name, func.count(ChildrenNode.id), Node.hyperdata['Processing'])
         #.query(Node.id, Node.name, Resource.type_id, func.count(ChildrenNode.id))
         #.join(Node_Resource, Node_Resource.node_id == Node.id)
         #.join(Resource, Resource.id == Node_Resource.resource_id)
@@ -137,7 +137,7 @@ def project(request, project_id):
                 parent_id   = project_id,
                 type_id     = cache.NodeType['Corpus'].id,
                 language_id = language_id,
-                metadata    = {'Processing' : 1,}
+                hyperdata    = {'Processing' : 1,}
             )
             session.add(corpus)
             session.commit()
@@ -210,16 +210,21 @@ def tfidf(request, corpus_id, ngram_ids):
         .order_by(func.sum(NodeNodeNgram.score).desc())
         .limit(limit)
     )
+    # print("\n")
+    # print("in TFIDF:")
+    # print("\tcorpus_id:",corpus_id)
     # convert query result to a list of dicts
     for node, score in nodes_query:
+        print("\t corpus:",corpus_id,"\t",node.name)
         node_dict = {
             'id': node.id,
             'score': score,
         }
         for key in ('title', 'publication_date', 'journal', 'authors', 'fields'):
-            if key in node.metadata:
-                node_dict[key] = node.metadata[key]
+            if key in node.hyperdata:
+                node_dict[key] = node.hyperdata[key]
         nodes_list.append(node_dict)
 
+    # print("= = = = = = = = \n")
     data = json.dumps(nodes_list) 
     return JsonHttpResponse(data)

init/init_accounts.py

@@ -19,8 +19,10 @@ def notify_user(username, email, password):
 
     ''' % (username, password)
 
-    #send_mail('[Gargantext] Votre compte', message, 'alexandre.delanoe@mines-paristech.fr', [email], fail_silently=False )
-    send_mail('[Gargantext] Votre compte', message, 'alexandre.delanoe@mines-paristech.fr', [email], ['alexandre@delanoe.org'] )
+    send_mail('[Gargantext] Création de votre compte', message, 'alexandre.delanoe@mines-paristech.fr', [email], fail_silently=False )
+    #send_mail('[Gargantext] Votre compte', message, 'alexandre.delanoe@mines-paristech.fr', [email], ['alexandre@delanoe.org'] )
+
+
     # add option for mass sending email
 
 def create_user(username, email, password=None, active=False, notify=True):
@@ -56,8 +58,8 @@ def mines_account_creation(fichier=None):
         fichier = "/home/alexandre/projets/forccast/Tutorat/2014-2015/comptes_gargantext.csv"
     accounts = open(fichier, "r")
     for line in accounts.readlines():
-        username, email, password = line.split(',')
-        create_user(username, email, password=password, notify=True)
+        username, email, password, fin = line.split(',')
+        create_user(username, email, password=password, notify=False)
         #delete_user(username)
     accounts.close()
 

init/init_gargantext.py

@@ -16,8 +16,8 @@ from gargantext_web.db import *
 #
 #tables_to_empty = [
 #    Node,
-#    Node_Metadata,
-#    Metadata,
+#    Node_Hyperdata,
+#    Hyperdata,
 #    NodeType,
 #    ResourceType,
 #    Resource,
@@ -27,10 +27,10 @@ from gargantext_web.db import *
 #    table.objects.all().delete()
 
 
-# Integration: metadata types
+# Integration: hyperdata types
 
-print('Initialize metadata...')
-metadata = {
+print('Initialize hyperdata...')
+hyperdata = {
     'publication_date': 'datetime',
     'authors': 'string',
     'language_fullname': 'string',
@@ -43,8 +43,8 @@ metadata = {
     'doi': 'string',
     'journal': 'string',
 }
-for name, type in metadata.items():
-    models.Metadata(name=name, type=type).save()
+for name, type in hyperdata.items():
+    models.Hyperdata(name=name, type=type).save()
 
 
 # Integration: languages

init/sql/drop_db.sh

+
+export PGPASSWORD=C8kdcUrAQy66U 
+
+psql -U gargantua -d gargandb -f drop_db.sql

init/sql/rename_metadata.sql

+
+ALTER TABLE node_node RENAME metadata TO hyperdata ;
+
+ALTER TABLE node_metadata RENAME TO node_hyperdata ;
+
+ALTER TABLE node_node_metadata 
+	RENAME TO node_node_hyperdata 
+	RENAME metadata_id TO  hyperdata_id 
+	;
+

init/sql/terminate_processus.sql

+
+SELECT pg_terminate_backend(pg_stat_activity.pid)
+FROM pg_stat_activity
+WHERE pg_stat_activity.datname = 'TARGET_DB' AND pid <> pg_backend_pid();

node/admin.py

@@ -62,7 +62,7 @@ class NodeAdmin(admin.ModelAdmin):
                                                 parent  = nodeParent,\
                                                 user    = request.user,\
                                                 name    = obj.name,\
-                                                metadata= obj.metadata,\
+                                                hyperdata= obj.hyperdata,\
                                                 )
             
             #nodeParent.save()
@@ -85,7 +85,7 @@ from django.db.models.query import EmptyQuerySet
 class ProjectForm(ModelForm):
     class Meta:
         model = Project
-        exclude = ['ngrams', 'metadata', 'parent', 'user', 'type', 'language', 'date']
+        exclude = ['ngrams', 'hyperdata', 'parent', 'user', 'type', 'language', 'date']
 
 class ResourceForm(ModelForm):
     class Meta:
@@ -139,7 +139,7 @@ class CorpusForm(ModelForm):
     
     class Meta:
         model   = Corpus
-        exclude = ['parent', 'user', 'language', 'type', 'ngrams', 'metadata', 'date']
+        exclude = ['parent', 'user', 'language', 'type', 'ngrams', 'hyperdata', 'date']
 
 class CorpusAdmin(NodeAdmin):
     _parent_nodetype_name = 'Project'

parsing/FileParsers/FileParser.py

@@ -19,8 +19,8 @@ class FileParser:
         return encoding.get('encoding', 'UTF-8')
     
     
-    def format_metadata_dates(self, metadata):
-        """Format the dates found in the metadata.
+    def format_hyperdata_dates(self, hyperdata):
+        """Format the dates found in the hyperdata.
         Examples:
             {"publication_date": "2014-10-23 09:57:42"}
             -> {"publication_date": "2014-10-23 09:57:42", "publication_year": "2014", ...}
@@ -29,64 +29,64 @@ class FileParser:
         """
         
         # First, check the split dates...
-        prefixes = [key[:-5] for key in metadata.keys() if key[-5:] == "_year"]
+        prefixes = [key[:-5] for key in hyperdata.keys() if key[-5:] == "_year"]
         for prefix in prefixes:
-            date_string = metadata[prefix + "_year"]
+            date_string = hyperdata[prefix + "_year"]
             key = prefix + "_month"
-            if key in metadata:
-                date_string += " " + metadata[key]
+            if key in hyperdata:
+                date_string += " " + hyperdata[key]
                 key = prefix + "_day"
-                if key in metadata:
-                    date_string += " " + metadata[key]
+                if key in hyperdata:
+                    date_string += " " + hyperdata[key]
                     key = prefix + "_hour"
-                    if key in metadata:
-                        date_string += " " + metadata[key]
+                    if key in hyperdata:
+                        date_string += " " + hyperdata[key]
                         key = prefix + "_minute"
-                        if key in metadata:
-                            date_string += ":" + metadata[key]
+                        if key in hyperdata:
+                            date_string += ":" + hyperdata[key]
                             key = prefix + "_second"
-                            if key in metadata:
-                                date_string += ":" + metadata[key]
+                            if key in hyperdata:
+                                date_string += ":" + hyperdata[key]
             try:
-                metadata[prefix + "_date"] = dateutil.parser.parse(date_string).strftime("%Y-%m-%d %H:%M:%S")
+                hyperdata[prefix + "_date"] = dateutil.parser.parse(date_string).strftime("%Y-%m-%d %H:%M:%S")
             except:
                 pass
         
         # ...then parse all the "date" fields, to parse it into separate elements
-        prefixes = [key[:-5] for key in metadata.keys() if key[-5:] == "_date"]
+        prefixes = [key[:-5] for key in hyperdata.keys() if key[-5:] == "_date"]
         for prefix in prefixes:
-            date = dateutil.parser.parse(metadata[prefix + "_date"])
-            metadata[prefix + "_year"]      = date.strftime("%Y")
-            metadata[prefix + "_month"]     = date.strftime("%m")
-            metadata[prefix + "_day"]       = date.strftime("%d")
-            metadata[prefix + "_hour"]      = date.strftime("%H")
-            metadata[prefix + "_minute"]    = date.strftime("%M")
-            metadata[prefix + "_second"]    = date.strftime("%S")
+            date = dateutil.parser.parse(hyperdata[prefix + "_date"])
+            hyperdata[prefix + "_year"]      = date.strftime("%Y")
+            hyperdata[prefix + "_month"]     = date.strftime("%m")
+            hyperdata[prefix + "_day"]       = date.strftime("%d")
+            hyperdata[prefix + "_hour"]      = date.strftime("%H")
+            hyperdata[prefix + "_minute"]    = date.strftime("%M")
+            hyperdata[prefix + "_second"]    = date.strftime("%S")
                 
         # finally, return the transformed result!
-        return metadata
+        return hyperdata
         
-    def format_metadata_languages(self, metadata):
-        """format the languages found in the metadata."""
+    def format_hyperdata_languages(self, hyperdata):
+        """format the languages found in the hyperdata."""
         language = None
         for key in ["fullname", "iso3", "iso2"]:
             language_key = "language_" + key
-            if language_key in metadata:
-                language_symbol = metadata[language_key]
+            if language_key in hyperdata:
+                language_symbol = hyperdata[language_key]
                 language = self._languages_cache[language_symbol]
                 if language:
                     break
         if language:
-            metadata["language_iso2"]       = language.iso2
-            metadata["language_iso3"]       = language.iso3
-            metadata["language_fullname"]   = language.fullname
-        return metadata
+            hyperdata["language_iso2"]       = language.iso2
+            hyperdata["language_iso3"]       = language.iso3
+            hyperdata["language_fullname"]   = language.fullname
+        return hyperdata
         
-    def format_metadata(self, metadata):
-        """Format the metadata."""
-        metadata = self.format_metadata_dates(metadata)
-        metadata = self.format_metadata_languages(metadata)
-        return metadata
+    def format_hyperdata(self, hyperdata):
+        """Format the hyperdata."""
+        hyperdata = self.format_hyperdata_dates(hyperdata)
+        hyperdata = self.format_hyperdata_languages(hyperdata)
+        return hyperdata
     
     
     def _parse(self, file):
@@ -96,28 +96,28 @@ class FileParser:
     def parse(self, file):
         """Parse the file, and its children files found in the file.
         """
-        # initialize the list of metadata
-        metadata_list = []
+        # initialize the list of hyperdata
+        hyperdata_list = []
         # is the file is a ZIP archive, recurse on each of its files...
         if zipfile.is_zipfile(file):
             zipArchive = zipfile.ZipFile(file)
             for filename in zipArchive.namelist():
                 try:
                     f = zipArchive.open(filename, 'r')
-                    metadata_list += self.parse(f)
+                    hyperdata_list += self.parse(f)
                     f.close()
                 except Exception as error:
                     print(error)
         # ...otherwise, let's parse it directly!
         else:
             try:
-                for metadata in self._parse(file):
-                    metadata_list.append(self.format_metadata(metadata))
+                for hyperdata in self._parse(file):
+                    hyperdata_list.append(self.format_hyperdata(hyperdata))
                 if hasattr(file, 'close'):
                     file.close()
             except Exception as error:
                 print(error)
-        # return the list of formatted metadata
-        return metadata_list
+        # return the list of formatted hyperdata
+        return hyperdata_list
 
 

parsing/FileParsers/ISText.py

@@ -13,8 +13,8 @@ class ISText(FileParser):
         data = json.load(json_data)
         json_data.close()
         json_docs = data["hits"]
-        metadata_list = []
-        metadata_path = {
+        hyperdata_list = []
+        hyperdata_path = {
             "id"                : "id",
             "source"           : 'corpusName',
             "title"             : 'title',
@@ -27,65 +27,65 @@ class ISText(FileParser):
             "authorsRAW"        : 'author',
             "keywords"          : "keywords"
         }
-        metadata = {}
+        hyperdata = {}
         import pprint
         import datetime
         for json_doc in json_docs:
-            for key, path in metadata_path.items():
+            for key, path in hyperdata_path.items():
                 try:
                     # print(path," ==> ",len(json_doc[path]))
-                    metadata[key] = json_doc[path]
+                    hyperdata[key] = json_doc[path]
                 except: pass
 
-            # print("|",metadata["publication_date"])
+            # print("|",hyperdata["publication_date"])
 
-            if "doi" in metadata: metadata["doi"] = metadata["doi"][0]
+            if "doi" in hyperdata: hyperdata["doi"] = hyperdata["doi"][0]
             
             keywords = []
-            if "keywords" in metadata:
-                for keyw in metadata["keywords"]:
+            if "keywords" in hyperdata:
+                for keyw in hyperdata["keywords"]:
                     keywords.append(keyw["value"] )
-                metadata["keywords"] = ", ".join( keywords )
+                hyperdata["keywords"] = ", ".join( keywords )
 
             moredate=False
             moresource=False
-            if "host" in metadata:
+            if "host" in hyperdata:
 
-                if "genre" in metadata["host"] and len(metadata["host"]["genre"])>0:
-                    if "genre" in metadata and len(metadata["genre"])==0:
-                        metadata["genre"] = metadata["host"]["genre"]
+                if "genre" in hyperdata["host"] and len(hyperdata["host"]["genre"])>0:
+                    if "genre" in hyperdata and len(hyperdata["genre"])==0:
+                        hyperdata["genre"] = hyperdata["host"]["genre"]
 
-                # print(metadata["host"])
-                if "pubdate" in metadata["host"]:
-                    onebuffer = metadata["publication_date"]
-                    metadata["publication_date"] = []
-                    metadata["publication_date"].append(onebuffer)
-                    metadata["publication_date"].append( metadata["host"]["pubdate"] )
+                # print(hyperdata["host"])
+                if "pubdate" in hyperdata["host"]:
+                    onebuffer = hyperdata["publication_date"]
+                    hyperdata["publication_date"] = []
+                    hyperdata["publication_date"].append(onebuffer)
+                    hyperdata["publication_date"].append( hyperdata["host"]["pubdate"] )
 
-                if "title" in metadata["host"]:
-                    metadata["journal"] = metadata["host"]["title"]
+                if "title" in hyperdata["host"]:
+                    hyperdata["journal"] = hyperdata["host"]["title"]
 
             authors=False
-            if "authorsRAW" in metadata:
+            if "authorsRAW" in hyperdata:
                 names = []
-                for author in metadata["authorsRAW"]: 
+                for author in hyperdata["authorsRAW"]: 
                     names.append(author["name"])
-                metadata["authors"] = ", ".join(names)
+                hyperdata["authors"] = ", ".join(names)
 
-            if "host" in metadata: metadata.pop("host")
-            if "genre" in metadata:
-                if len(metadata["genre"])==0:
-                    metadata.pop("genre")
+            if "host" in hyperdata: hyperdata.pop("host")
+            if "genre" in hyperdata:
+                if len(hyperdata["genre"])==0:
+                    hyperdata.pop("genre")
             
-            if "publication_date" in metadata and isinstance(metadata["publication_date"], list):
-                if len(metadata["publication_date"])>1:
-                    d1 = metadata["publication_date"][0]
-                    d2 = metadata["publication_date"][1]
+            if "publication_date" in hyperdata and isinstance(hyperdata["publication_date"], list):
+                if len(hyperdata["publication_date"])>1:
+                    d1 = hyperdata["publication_date"][0]
+                    d2 = hyperdata["publication_date"][1]
                     # print("date1:",d1)
                     # print("date2:",d2)
                     if len(d1)==len(d2):
-                        metadata["publication_date"] = d2
-                        # if int(d1)>int(d2): metadata["publication_date"] = d2
+                        hyperdata["publication_date"] = d2
+                        # if int(d1)>int(d2): hyperdata["publication_date"] = d2
                     else:
                         fulldate = ""
                         year = d2[:4]
@@ -96,17 +96,17 @@ class ISText(FileParser):
                             if len(d2)>6:
                                 day = d2[6:8]
                                 fulldate+="-"+day
-                        metadata["publication_date"] = fulldate
+                        hyperdata["publication_date"] = fulldate
                 else:
                     if "copyrightdate" in json_doc: 
-                        metadata["publication_date"] = json_doc["copyrightdate"]
+                        hyperdata["publication_date"] = json_doc["copyrightdate"]
             else:
                 if "copyrightdate" in json_doc:
-                    metadata["publication_date"] = json_doc["copyrightdate"]
+                    hyperdata["publication_date"] = json_doc["copyrightdate"]
             
-            print("||",metadata["title"])
-            metadata_list.append(metadata)
+            print("||",hyperdata["title"])
+            hyperdata_list.append(hyperdata)
             print("=============================")
 
-        print("\nlen list:",len(metadata_list))
-        return metadata_list
+        print("\nlen list:",len(hyperdata_list))
+        return hyperdata_list

parsing/FileParsers/IsiFileParser.py

@@ -12,13 +12,13 @@ class IsiFileParser(RisFileParser):
         
         self._parameters = {
             b"ER":  {"type": "delimiter"},
-            b"TI":  {"type": "metadata", "key": "title", "separator": " "},
-            b"AU":  {"type": "metadata", "key": "authors", "separator": ", "},
-            b"DI":  {"type": "metadata", "key": "doi"},
-            b"PY":  {"type": "metadata", "key": "publication_year"},
-            b"PD":  {"type": "metadata", "key": "publication_month"},
-            b"LA":  {"type": "metadata", "key": "language_fullname"},
-            b"AB":  {"type": "metadata", "key": "abstract", "separator": " "},
-            b"WC":  {"type": "metadata", "key": "fields"},
+            b"TI":  {"type": "hyperdata", "key": "title", "separator": " "},
+            b"AU":  {"type": "hyperdata", "key": "authors", "separator": ", "},
+            b"DI":  {"type": "hyperdata", "key": "doi"},
+            b"PY":  {"type": "hyperdata", "key": "publication_year"},
+            b"PD":  {"type": "hyperdata", "key": "publication_month"},
+            b"LA":  {"type": "hyperdata", "key": "language_fullname"},
+            b"AB":  {"type": "hyperdata", "key": "abstract", "separator": " "},
+            b"WC":  {"type": "hyperdata", "key": "fields"},
         }
 

parsing/FileParsers/JstorFileParser.py

@@ -11,13 +11,13 @@ class JstorFileParser(RisFileParser):
 
         self._parameters = {
             b"ER":  {"type": "delimiter"},
-            b"TI":  {"type": "metadata", "key": "title", "separator": " "},
-            b"AU":  {"type": "metadata", "key": "authors", "separator": ", "},
-            b"UR":  {"type": "metadata", "key": "doi"},
-            b"Y1":  {"type": "metadata", "key": "publication_year"},
-            b"PD":  {"type": "metadata", "key": "publication_month"},
-            b"LA":  {"type": "metadata", "key": "language_iso2"},
-            b"AB":  {"type": "metadata", "key": "abstract", "separator": " "},
-            b"WC":  {"type": "metadata", "key": "fields"},
+            b"TI":  {"type": "hyperdata", "key": "title", "separator": " "},
+            b"AU":  {"type": "hyperdata", "key": "authors", "separator": ", "},
+            b"UR":  {"type": "hyperdata", "key": "doi"},
+            b"Y1":  {"type": "hyperdata", "key": "publication_year"},
+            b"PD":  {"type": "hyperdata", "key": "publication_month"},
+            b"LA":  {"type": "hyperdata", "key": "language_iso2"},
+            b"AB":  {"type": "hyperdata", "key": "abstract", "separator": " "},
+            b"WC":  {"type": "hyperdata", "key": "fields"},
         }
 

parsing/FileParsers/PubmedFileParser.py

@@ -16,13 +16,13 @@ class PubmedFileParser(FileParser):
         else: xml = etree.parse(file, parser=xml_parser)
 
         xml_articles = xml.findall('PubmedArticle')
-        # initialize the list of metadata
-        metadata_list = []
+        # initialize the list of hyperdata
+        hyperdata_list = []
         # parse all the articles, one by one
         for xml_article in xml_articles:
             # extract data from the document
-            metadata = {}
-            metadata_path = {
+            hyperdata = {}
+            hyperdata_path = {
                 "journal"           : 'MedlineCitation/Article/Journal/Title',
                 "title"             : 'MedlineCitation/Article/ArticleTitle',
                 "abstract"          : 'MedlineCitation/Article/Abstract/AbstractText',
@@ -38,44 +38,44 @@ class PubmedFileParser(FileParser):
                 "publication_day"   : 'MedlineCitation/DateCreated/Day',
                 "authors"           : 'MedlineCitation/Article/AuthorList',
             }
-            for key, path in metadata_path.items():
+            for key, path in hyperdata_path.items():
                 try:
                     xml_node = xml_article.find(path)
                     # Authors tag
                     if key == 'authors':
-                        metadata[key] = ', '.join([
+                        hyperdata[key] = ', '.join([
                             xml_author.find('ForeName').text + ' ' + xml_author.find('LastName').text
                             for xml_author in xml_node
                         ])
                     else:
-                        metadata[key] = xml_node.text
+                        hyperdata[key] = xml_node.text
 
                 except:
                     pass
 
             #Title-Decision
             Title=""
-            if not metadata["title"] or metadata["title"]=="":
-                if "title2" in metadata:
-                    metadata["title"] = metadata["title2"]
-                else: metadata["title"] = ""
+            if not hyperdata["title"] or hyperdata["title"]=="":
+                if "title2" in hyperdata:
+                    hyperdata["title"] = hyperdata["title2"]
+                else: hyperdata["title"] = ""
 
             # Date-Decision
             # forge.iscpif.fr/issues/1418
             RealDate = ""
-            if "realdate_full_" in metadata:
-                RealDate = metadata["realdate_full_"]
+            if "realdate_full_" in hyperdata:
+                RealDate = hyperdata["realdate_full_"]
             else:
-                if "realdate_year_" in metadata: RealDate+=metadata["realdate_year_"]
-                if "realdate_month_" in metadata: RealDate+=" "+metadata["realdate_month_"]
-                if "realdate_day_" in metadata: RealDate+=" "+metadata["realdate_day_"]
-            metadata["realdate_full_"] = RealDate
+                if "realdate_year_" in hyperdata: RealDate+=hyperdata["realdate_year_"]
+                if "realdate_month_" in hyperdata: RealDate+=" "+hyperdata["realdate_month_"]
+                if "realdate_day_" in hyperdata: RealDate+=" "+hyperdata["realdate_day_"]
+            hyperdata["realdate_full_"] = RealDate
             RealDate = RealDate.split("-")[0]
 
             PubmedDate = ""
-            if "publication_year" in metadata: PubmedDate+=metadata["publication_year"]
-            if "publication_month" in metadata: PubmedDate+=" "+metadata["publication_month"]
-            if "publication_day" in metadata: PubmedDate+=" "+metadata["publication_day"]
+            if "publication_year" in hyperdata: PubmedDate+=hyperdata["publication_year"]
+            if "publication_month" in hyperdata: PubmedDate+=" "+hyperdata["publication_month"]
+            if "publication_day" in hyperdata: PubmedDate+=" "+hyperdata["publication_day"]
 
             Decision=""
             if len(RealDate)>4:
@@ -94,14 +94,14 @@ class PubmedFileParser(FileParser):
                 except: Decision=False
 
             if Decision!=False:
-                if "publication_year" in metadata: metadata["publication_year"] = str(Decision.year)
-                if "publication_month" in metadata: metadata["publication_month"] = str(Decision.month)
-                if "publication_day" in metadata: metadata["publication_day"] = str(Decision.day)
-                if "realdate_year_" in metadata: metadata.pop("realdate_year_")
-                if "realdate_month_" in metadata: metadata.pop("realdate_month_")
-                if "realdate_day_" in metadata: metadata.pop("realdate_day_")
-                if "title2" in metadata: metadata.pop("title2")
+                if "publication_year" in hyperdata: hyperdata["publication_year"] = str(Decision.year)
+                if "publication_month" in hyperdata: hyperdata["publication_month"] = str(Decision.month)
+                if "publication_day" in hyperdata: hyperdata["publication_day"] = str(Decision.day)
+                if "realdate_year_" in hyperdata: hyperdata.pop("realdate_year_")
+                if "realdate_month_" in hyperdata: hyperdata.pop("realdate_month_")
+                if "realdate_day_" in hyperdata: hyperdata.pop("realdate_day_")
+                if "title2" in hyperdata: hyperdata.pop("title2")
                 
-                metadata_list.append(metadata)
-        # return the list of metadata
-        return metadata_list
+                hyperdata_list.append(hyperdata)
+        # return the list of hyperdata
+        return hyperdata_list

parsing/FileParsers/PubmedFileParser_wYield.py

+from django.db import transaction
+from lxml import etree
+from .FileParser import FileParser
+from ..NgramsExtractors import *
+
+class PubmedFileParser(FileParser):
+    
+    def _parse(self, file):
+        # open the file as XML
+        xml_parser = etree.XMLParser(resolve_entities=False, recover=True)
+        xml = etree.parse(file, parser=xml_parser)
+        xml_articles = xml.findall('PubmedArticle')
+        # parse all the articles, one by one
+        for xml_article in xml_articles:
+            # extract data from the document
+            hyperdata = {}
+            hyperdata_path = {
+                "journal"           : 'MedlineCitation/Article/Journal/Title',
+                "title"             : 'MedlineCitation/Article/ArticleTitle',
+                "language_iso3"     : 'MedlineCitation/Article/Language',
+                "doi"               : 'PubmedData/ArticleIdList/ArticleId[@type=doi]',
+                "abstract"          : 'MedlineCitation/Article/Abstract/AbstractText',
+                "publication_year"  : 'MedlineCitation/DateCreated/Year',
+                "publication_month" : 'MedlineCitation/DateCreated/Month',
+                "publication_day"   : 'MedlineCitation/DateCreated/Day',
+                "authors"           : 'MedlineCitation/Article/AuthorList',
+            }
+            for key, path in hyperdata_path.items():
+                try:
+                    xml_node = xml_article.find(path)
+                    if key == 'authors':
+                        hyperdata[key] = ', '.join([
+                            xml_author.find('ForeName').text + ' ' + xml_author.find('LastName').text
+                            for xml_author in xml_node
+                        ])
+                    else:
+                        hyperdata[key] = xml_node.text
+                except:
+                    pass
+            yield hyperdata
\ No newline at end of file

parsing/FileParsers/RisFileParser.py

@@ -14,20 +14,20 @@ class RisFileParser(FileParser):
         
         self._parameters = {
             b"ER":  {"type": "delimiter"},
-            b"TI":  {"type": "metadata", "key": "title", "separator": " "},
-            b"ST":  {"type": "metadata", "key": "subtitle", "separator": " "},
-            b"AU":  {"type": "metadata", "key": "authors", "separator": ", "},
-            b"UR":  {"type": "metadata", "key": "doi"},
-            b"PY":  {"type": "metadata", "key": "publication_year"},
-            b"PD":  {"type": "metadata", "key": "publication_month"},
-            b"LA":  {"type": "metadata", "key": "language_iso2"},
-            b"AB":  {"type": "metadata", "key": "abstract", "separator": " "},
-            b"WC":  {"type": "metadata", "key": "fields"},
+            b"TI":  {"type": "hyperdata", "key": "title", "separator": " "},
+            b"ST":  {"type": "hyperdata", "key": "subtitle", "separator": " "},
+            b"AU":  {"type": "hyperdata", "key": "authors", "separator": ", "},
+            b"UR":  {"type": "hyperdata", "key": "doi"},
+            b"PY":  {"type": "hyperdata", "key": "publication_year"},
+            b"PD":  {"type": "hyperdata", "key": "publication_month"},
+            b"LA":  {"type": "hyperdata", "key": "language_iso2"},
+            b"AB":  {"type": "hyperdata", "key": "abstract", "separator": " "},
+            b"WC":  {"type": "hyperdata", "key": "fields"},
         }
         
 
     def _parse(self, file):
-        metadata = {}
+        hyperdata = {}
         last_key = None
         last_values = []
         # browse every line of the file
@@ -39,23 +39,23 @@ class RisFileParser(FileParser):
                     if last_key in self._parameters:
                         # translate the parameter key
                         parameter = self._parameters[last_key]
-                        if parameter["type"] == "metadata":
+                        if parameter["type"] == "hyperdata":
                             separator = parameter["separator"] if "separator" in parameter else ""
-                            metadata[parameter["key"]] = separator.join(last_values)
+                            hyperdata[parameter["key"]] = separator.join(last_values)
                         elif parameter["type"] == "delimiter":
-                            if 'language_fullname' not in metadata.keys():
-                                if 'language_iso3' not in metadata.keys():
-                                    if 'language_iso2' not in metadata.keys():
-                                        metadata['language_iso2'] = 'en'
-                            yield metadata
-                            metadata = {}
+                            if 'language_fullname' not in hyperdata.keys():
+                                if 'language_iso3' not in hyperdata.keys():
+                                    if 'language_iso2' not in hyperdata.keys():
+                                        hyperdata['language_iso2'] = 'en'
+                            yield hyperdata
+                            hyperdata = {}
                     last_key = parameter_key
                     last_values = []
                 try:
                     last_values.append(line[self._begin:-1].decode())
                 except Exception as error:
                     print(error)
-        # if a metadata object is left in memory, yield it as well
-        if metadata:
-            #print(metadata['title'])
-            yield metadata
+        # if a hyperdata object is left in memory, yield it as well
+        if hyperdata:
+            #print(hyperdata['title'])
+            yield hyperdata

parsing/Taggers/Melt.py

+from .Tagger import Tagger
+
+import subprocess
+import threading
+import time
+
+
+# TODO: have a look at "queue" instead of "list" (cf. http://stackoverflow.com/questions/17564804/in-python-how-to-wait-until-only-the-first-thread-is-finished)
+
+class identity_dict(dict):
+    def __missing__(self, key):
+        return key
+    
+_tag_replacements = identity_dict({
+    "NOM": "NN",
+    "NAM": "NN",
+    "ADJ": "NN",
+    "VER": "JJ",
+    "PREP": "PRP",
+    "KON": "CC",
+    "DET": "DT",
+    "PRO": "DT",
+    # Do we also have to take semicolons, comas and other points into account?
+})
+
+def _readOutput(output, buffer):
+    hasStarted = False
+    while True:
+        line = output.readline()
+        if line:
+            if line == b"<block>\n":
+                hasStarted = True
+                continue
+            if line == b"<block/>\n":
+                break
+            if hasStarted:
+                token, tag = line.decode('utf8').split()[:2]
+                tag = _tag_replacements[tag.split(':')[0]]
+                buffer.append((token, tag))
+        else:
+            time.sleep(0.1)
+
+
+"""Use MElt for the tagging.
+"""
+class Melt(Tagger):
+    
+    def start(self, taggerPath = "/usr/local/bin/"):
+        binaryFile = "%s/MElt" % taggerPath
+        tagcmdlist = [
+            binaryFile,
+            "-l",
+        ]
+
+        tagcmdlist = []
+        self._popen = subprocess.Popen(
+            tagcmdlist,     # Use a list of params in place of a string.
+            bufsize=0,      # Not buffered to retrieve data asap from Tagger
+            executable=binaryFile, # As we have it, specify it
+            stdin=subprocess.PIPE,  # Get a pipe to write input data to Tagger process
+            stdout=subprocess.PIPE, # Get a pipe to read processing results from Tagger
+            stderr=subprocess.PIPE, # Get a pipe to read processing results from Tagger
+        )
+        self._input, self._output = self._popen.stdin, self._popen.stdout
+        # self._thread = threading.Thread(target=_readOutput, args=(self._output, self.buffer, )).start()
+        # self.buffer = OutputBuffer()
+        
+    def stop(self):
+        # terminates the process
+        try:
+            self._popen.kill()
+            self._popen.terminate()
+        except:
+            pass
+        
+    def tagging_start(self):
+        self.buffer = []
+        self._thread = threading.Thread(target=_readOutput, args=(self._output, self.buffer, ))
+        self._thread.start()
+        #self._input.write(b"<block>\n")
+        
+    def tagging_end(self):
+        #self._input.write(b"<block/>\n")
+        # sends some dummy tokens, then wait for the text to be treated
+        #self.tag_tokens("Les sanglots longs des violons de l ' automne bercent mon coeur d ' une langueur monotone .".split(), False)
+        self._thread.join()
+        
+        
+    def tag_tokens(self, tokens, single=True):
+        if single:
+            self.tagging_start()
+        for token in tokens:
+            self._input.write(bytes(token + "\n", "utf8"))
+        if single:
+            self.tagging_end()
+            return self.buffer
+            
+    def tag_text(self, text):
+        self.tagging_start()
+        for line in text.split('\n'):
+            tokens = self._re_sentence.findall(line)
+            self.tag_tokens(tokens, False)
+        self.tagging_end()
+        return self.buffer
+

parsing/corpustools.py

@@ -45,7 +45,7 @@ parsers = Parsers()
 
 
 
-# resources managment
+# resources management
 
 def add_resource(corpus, **kwargs):
     # only for tests
@@ -106,23 +106,23 @@ def parse_resources(corpus, user=None, user_id=None):
     nodes = list()
     for resource, resourcetype in resources_query:
         parser = parsers[resourcetype.name]
-        for metadata_dict in parser.parse(resource.file):
-            # retrieve language ID from metadata
-            if 'language_iso2' in metadata_dict:
+        for hyperdata_dict in parser.parse(resource.file):
+            # retrieve language ID from hyperdata
+            if 'language_iso2' in hyperdata_dict:
                 try:
-                    language_id = cache.Language[metadata_dict['language_iso2']].id
+                    language_id = cache.Language[hyperdata_dict['language_iso2']].id
                 except KeyError:
                     language_id = None
             else:
                 language_id = None
             # create new node
             node = Node(
-                name = metadata_dict.get('title', '')[:200],
+                name = hyperdata_dict.get('title', '')[:200],
                 parent_id = corpus_id,
                 user_id = user_id,
                 type_id = type_id,
                 language_id = language_id,
-                metadata = metadata_dict,
+                hyperdata = hyperdata_dict,
                 date = datetime.utcnow(),
             )
             nodes.append(node)
@@ -132,30 +132,30 @@ def parse_resources(corpus, user=None, user_id=None):
     dbg.show('insert %d documents' % len(nodes))
     session.add_all(nodes)
     session.commit()
-    # now, index the metadata
-    dbg.show('insert metadata')
-    node_metadata_lists = defaultdict(list)
-    metadata_types = {
-        metadata.name: metadata
-        for metadata in session.query(Metadata)
+    # now, index the hyperdata
+    dbg.show('insert hyperdata')
+    node_hyperdata_lists = defaultdict(list)
+    hyperdata_types = {
+        hyperdata.name: hyperdata
+        for hyperdata in session.query(Hyperdata)
     }
     for node in nodes:
         node_id = node.id
-        for metadata_key, metadata_value in node.metadata.items():
+        for hyperdata_key, hyperdata_value in node.hyperdata.items():
             try:
-                metadata = metadata_types[metadata_key]
+                hyperdata = hyperdata_types[hyperdata_key]
             except KeyError:
                 # Why silent continue here ?
                 continue
-            if metadata.type == 'string':
-                metadata_value = metadata_value[:255]
-            node_metadata_lists[metadata.type].append((
+            if hyperdata.type == 'string':
+                hyperdata_value = hyperdata_value[:255]
+            node_hyperdata_lists[hyperdata.type].append((
                 node_id,
-                metadata.id,
-                metadata_value,
+                hyperdata.id,
+                hyperdata_value,
             ))
-    for key, values in node_metadata_lists.items():
-        bulk_insert(Node_Metadata, ['node_id', 'metadata_id', 'value_'+key], values)
+    for key, values in node_hyperdata_lists.items():
+        bulk_insert(Node_Hyperdata, ['node_id', 'hyperdata_id', 'value_'+key], values)
     # mark the corpus as parsed
     corpus.parsed = True
 
@@ -192,9 +192,9 @@ ngramsextractors = NgramsExtractors()
 def extract_ngrams(corpus, keys):
     dbg = DebugTime('Corpus #%d - ngrams' % corpus.id)
     default_language_iso2 = None if corpus.language_id is None else cache.Language[corpus.language_id].iso2
-    # query the metadata associated with the given keys
-    columns = [Node.id, Node.language_id] + [Node.metadata[key] for key in keys]
-    metadata_query = (session
+    # query the hyperdata associated with the given keys
+    columns = [Node.id, Node.language_id] + [Node.hyperdata[key] for key in keys]
+    hyperdata_query = (session
         .query(*columns)
         .filter(Node.parent_id == corpus.id)
         .filter(Node.type_id == cache.NodeType['Document'].id)
@@ -211,7 +211,7 @@ def extract_ngrams(corpus, keys):
     ngrams_tag_data = set()
 
     node_ngram_list = defaultdict(lambda: defaultdict(int))
-    for nodeinfo in metadata_query:
+    for nodeinfo in hyperdata_query:
         node_id = nodeinfo[0]
         language_id = nodeinfo[1]
 

scrappers/scrap_pubmed/MedlineFetcherDavid2015.py

@@ -177,14 +177,14 @@ class MedlineFetcher:
             # globalresults = self.medlineEsearch(pubmedquery)
             if globalresults["count"]>0:
                 N+=globalresults["count"]
-                querymetadata = { 
+                queryhyperdata = { 
                     "string": globalresults["query"] , 
                     "count": globalresults["count"] , 
                     "queryKey":globalresults["queryKey"] , 
                     "webEnv":globalresults["webEnv"] , 
                     "retmax":0 
                 }
-                thequeries.append ( querymetadata )
+                thequeries.append ( queryhyperdata )
 
         print("Total Number:", N,"publications")
         print("And i want just:",globalLimit,"publications")

scrappers/scrap_pubmed/views.py

@@ -49,7 +49,6 @@ def getGlobalStats(request ):
 		print ("LOG::TIME:_ "+datetime.datetime.now().isoformat()+" query =", query )
 		print ("LOG::TIME:_ "+datetime.datetime.now().isoformat()+" N =", N )
 		instancia = MedlineFetcher()
-		# alist = instancia.serialFetcher( 5, query , int(request.POST["N"]) )
 		alist = instancia.serialFetcher( 5, query , N )
 
 	data = alist
@@ -130,7 +129,7 @@ def doTheQuery(request , project_id):
 			parent_id = project_id,
 			type_id = cache.NodeType['Corpus'].id,
 			language_id = None,
-                        metadata    = {'Processing' : 1,}
+                        hyperdata    = {'Processing' : 1,}
 		)
 		session.add(corpus)
 		session.commit()

static/css/jquery.dynatable.css

+/*
+ * jQuery Dynatable plugin 0.3.1
+ *
+ * Copyright (c) 2014 Steve Schwartz (JangoSteve)
+ *
+ * Dual licensed under the AGPL and Proprietary licenses:
+ *   http://www.dynatable.com/license/
+ *
+ * Date: Tue Jan 02 2014
+ */
+th {
+  background: #bd2525;
+}
+th a {
+  color: #fff;
+}
+th a:hover {
+  color: #fff;
+  text-decoration: underline;
+}
+
+.dynatable-search {
+  float: left;
+  margin-bottom: 10px;
+}
+
+.dynatable-pagination-links {
+  float: right;
+}
+
+.dynatable-record-count {
+  display: block;
+  padding: 5px 0;
+}
+
+.dynatable-pagination-links span,
+.dynatable-pagination-links li {
+  display: inline-block;
+}
+
+.dynatable-page-link,
+.dynatable-page-break {
+  display: block;
+  padding: 5px 7px;
+}
+
+.dynatable-page-link {
+  cursor: pointer;
+}
+
+.dynatable-active-page,
+.dynatable-disabled-page {
+  cursor: text;
+}
+.dynatable-active-page:hover,
+.dynatable-disabled-page:hover {
+  text-decoration: none;
+}
+
+.dynatable-active-page {
+  background: #bd2525;
+  border-radius: 5px;
+  color: #fff;
+}
+.dynatable-active-page:hover {
+  color: #fff;
+}
+.dynatable-disabled-page,
+.dynatable-disabled-page:hover {
+  background: none;
+  color: #999;
+}
\ No newline at end of file

static/js/charts/dc.js

@@ -342,7 +342,7 @@ dc.events.trigger = function(closure, delay) {
     dc.events.current = closure;
 
     setTimeout(function() {
-        if (closure == dc.events.current)
+        if (closure == dc.events.current) 
             closure();
     }, delay);
 };

static/js/extras_explorerjs.js

-/srv/gargantext_lib/js/extras_explorerjs.js
\ No newline at end of file

static/js/gargantext.angular.js

@@ -189,7 +189,7 @@ gargantext.controller("QueryController", function($scope, $http) {
             // change view to loading mode
             $scope.loading = true;
             // query parameters: columns
-            var retrieve = {type: 'fields', list: ['id', 'name', 'metadata.publication_date']};
+            var retrieve = {type: 'fields', list: ['id', 'name', 'hyperdata.publication_date']};
             // query parameters: pagination
             var pagination = $scope.pagination;
             // query parameters: sort
@@ -268,13 +268,13 @@ gargantext.controller("DatasetController", function($scope, $http) {
     };
     // update entities depending on the selected corpus
     $scope.updateEntities = function() {
-        var url = '/api/nodes/' + $scope.corpusId + '/children/metadata';
+        var url = '/api/nodes/' + $scope.corpusId + '/children/hyperdata';
         $scope.entities = undefined;
         $scope.filters = [];
         $http.get(url, {cache: true}).success(function(response){
             $scope.entities = [
                 {
-                    key: 'metadata',
+                    key: 'hyperdata',
                     columns: response.data
                 },
                 {
@@ -520,10 +520,10 @@ gargantext.controller("GraphController", function($scope, $http, $element) {
             var query = dataset.query;
             var data = {
                 filters: query.filters,
-                sort: ['metadata.publication_date.day'],
+                sort: ['hyperdata.publication_date.day'],
                 retrieve: {
                     type: 'aggregates',
-                    list: ['metadata.publication_date.day', query.mesured]
+                    list: ['hyperdata.publication_date.day', query.mesured]
                 }
             };
             // request to the server
@@ -569,7 +569,7 @@ setTimeout(function(){
         $('div.corpus select').change();
     //     $('div.filters button').last().click();
     //     var d = $('li.dataset').last();
-    //     d.find('select').last().val('metadata').change();
+    //     d.find('select').last().val('hyperdata').change();
     //     d.find('select').last().val('publication_date').change();
     //     d.find('select').last().val('>').change();
     //     d.find('input').last().val('2010').change();
@@ -580,7 +580,7 @@ setTimeout(function(){
     //     // d.find('select').change();
     //     // // second dataset's filter
     //     // d.find('div.filters button').last().click();
-    //     // d.find('select').last().val('metadata').change();
+    //     // d.find('select').last().val('hyperdata').change();
     //     // d.find('select').last().val('abstract').change();
     //     // d.find('select').last().val('contains').change();
     //     // d.find('input').last().val('dea').change();

static/js/graph-it.js

@@ -187,7 +187,7 @@ container.prepend(', corpus ');
 var selectProject = $('<select>').prependTo(container);
 container.prepend('In the project ');
 
-var metadataCollection;
+var hyperdataCollection;
 var corpusId;
 
 // how shall we group the data?
@@ -205,7 +205,7 @@ var buttonView = $('<button>').text('Graph it!').click(function(e) {
         liDataset = $(liDataset);
         var getData = {
             mesured:      liDataset.find('*[name]').first().val(),
-            parameters:   ['metadata.publication_date'],
+            parameters:   ['hyperdata.publication_date'],
             filters:      [],
             format:       'json',
         };
@@ -225,17 +225,17 @@ var buttonView = $('<button>').text('Graph it!').click(function(e) {
 }).insertAfter(ulDatasets).hide();
 
 
-// Load metadata
+// Load hyperdata
 selectCorpus.change(function() {
     corpusId = selectCorpus.val();
     emWait.show();
     ulDatasets.empty();
-    $.get('/api/corpus/' + corpusId + '/metadata', function(collection) {
+    $.get('/api/corpus/' + corpusId + '/hyperdata', function(collection) {
         // Unleash the power of the filter!
         emWait.hide();
         buttonAddDataset.show();
         buttonView.show();
-        metadataCollection = collection;
+        hyperdataCollection = collection;
         buttonAddDataset.click();
     });
 });
@@ -289,7 +289,7 @@ buttonAddDataset.click(function() {
         .appendTo(liDataset);
     // Add a filter when asked
     var ulFilters = $('<ul>').appendTo(liDataset);
-    var addFilter = function(metadataCollection) {
+    var addFilter = function(hyperdataCollection) {
         var liFilter = $('<li>').appendTo(ulFilters);
         liFilter.append('...where the ');
         // Type of filter: ngrams
@@ -301,25 +301,25 @@ buttonAddDataset.click(function() {
             .attr('name', 'ngrams.in')
             .appendTo(spanNgrams);
         spanNgrams.append(' (comma-separated ngrams)')
-        // Type of filter: metadata
-        $('<option>').text('metadata').appendTo(selectType);
-        var spanMetadata = $('<span>').appendTo(liFilter).hide();
-        var selectMetadata = $('<select>').appendTo(spanMetadata);
-        var spanMetadataValue = $('<span>').appendTo(spanMetadata);
-        $.each(metadataCollection, function(i, metadata) {
+        // Type of filter: hyperdata
+        $('<option>').text('hyperdata').appendTo(selectType);
+        var spanHyperdata = $('<span>').appendTo(liFilter).hide();
+        var selectHyperdata = $('<select>').appendTo(spanHyperdata);
+        var spanHyperdataValue = $('<span>').appendTo(spanHyperdata);
+        $.each(hyperdataCollection, function(i, hyperdata) {
             $('<option>')
-                .appendTo(selectMetadata)
-                .text(metadata.text)
-                .data(metadata);
+                .appendTo(selectHyperdata)
+                .text(hyperdata.text)
+                .data(hyperdata);
         });
-        // How do we present the metadata?
-        selectMetadata.change(function() {
-            var metadata = selectMetadata.find(':selected').data();
-            spanMetadataValue.empty();
-            if (metadata.type == 'datetime') {
-                spanMetadataValue.append(' is between ');
-                $('<input>').appendTo(spanMetadataValue)
-                    .attr('name', 'metadata.' + metadata.key + '.gt')
+        // How do we present the hyperdata?
+        selectHyperdata.change(function() {
+            var hyperdata = selectHyperdata.find(':selected').data();
+            spanHyperdataValue.empty();
+            if (hyperdata.type == 'datetime') {
+                spanHyperdataValue.append(' is between ');
+                $('<input>').appendTo(spanHyperdataValue)
+                    .attr('name', 'hyperdata.' + hyperdata.key + '.gt')
                     .datepicker({dateFormat: 'yy-mm-dd'})
                     .blur(function() {
                         var input = $(this);
@@ -327,9 +327,9 @@ buttonAddDataset.click(function() {
                         date += '2000-01-01'.substr(date.length);
                         input.val(date);
                     });
-                spanMetadataValue.append(' and ');
-                $('<input>').appendTo(spanMetadataValue)
-                    .attr('name', 'metadata.' + metadata.key + '.lt')
+                spanHyperdataValue.append(' and ');
+                $('<input>').appendTo(spanHyperdataValue)
+                    .attr('name', 'hyperdata.' + hyperdata.key + '.lt')
                     .datepicker({dateFormat: 'yy-mm-dd'})
                     .blur(function() {
                         var input = $(this);
@@ -337,41 +337,41 @@ buttonAddDataset.click(function() {
                         date += '2000-01-01'.substr(date.length);
                         input.val(date);
                     });
-            } else if (metadata.values) {
-                $('<span>').text(' is ').appendTo(spanMetadataValue);
-                var selectMetadataValue = $('<select>')
-                    .attr('name', 'metadata.' + metadata.key + '.eq')
-                    .appendTo(spanMetadataValue);
-                $.each(metadata.values, function(i, value) {
+            } else if (hyperdata.values) {
+                $('<span>').text(' is ').appendTo(spanHyperdataValue);
+                var selectHyperdataValue = $('<select>')
+                    .attr('name', 'hyperdata.' + hyperdata.key + '.eq')
+                    .appendTo(spanHyperdataValue);
+                $.each(hyperdata.values, function(i, value) {
                     $('<option>')
                         .text(value)
-                        .appendTo(selectMetadataValue);
+                        .appendTo(selectHyperdataValue);
                 });
-                selectMetadataValue.change().focus();
+                selectHyperdataValue.change().focus();
             } else {
-                spanMetadataValue.append(' contains ');
+                spanHyperdataValue.append(' contains ');
                 $('<input>')
-                    .attr('name', 'metadata.' + metadata.key + '.contains')
-                    .appendTo(spanMetadataValue)
+                    .attr('name', 'hyperdata.' + hyperdata.key + '.contains')
+                    .appendTo(spanHyperdataValue)
                     .focus();
             }
         }).change();
-        // Ngram or metadata?
+        // Ngram or hyperdata?
         selectType.change(function() {
             var spans = liFilter.children().filter('span').hide();
             switch (selectType.val()) {
                 case 'ngrams':
                     spanNgrams.show().find('input').focus();
                     break;
-                case 'metadata':
-                    spanMetadata.show();
+                case 'hyperdata':
+                    spanHyperdata.show();
                     break;
             }
         }).change();
     };
 
     buttonFilter.click(function(e) {
-        addFilter(metadataCollection);
+        addFilter(hyperdataCollection);
     });
 });
 

static/js/settings_explorerjs.js

-/srv/gargantext_lib/js/settings_explorerjs.js
\ No newline at end of file

static/js/settings_explorerjs.js

+/*
+ * Customize as you want ;)
+ */
+
+// ============ < DEVELOPER OPTIONS > ============
+var geomap=false;
+var minimap=false;
+var getAdditionalInfo=true;//for topPapers div
+
+
+var mainfile=false;
+getUrlParam.file = $("#graphid").html();
+
+var dataFolderTree = {};
+var gexfDict={};
+var egonode = {}
+var iwantograph = "";
+
+var bridge={};
+external="";
+//external="http://tina.iscpif.fr/explorerjs/";//Just if you want to use the server-apps from tina.server
+bridge["forFilteredQuery"] = external+"php/bridgeClientServer_filter.php";
+bridge["forNormalQuery"] = external+"php/bridgeClientServer.php";
+
+var clusters = [];
+
+
+ircNick="";
+ircCHN="";
+
+var catSoc = "Document";
+var catSem = "NGram";
+
+var sizeMult = [];
+    sizeMult[catSoc] = 0.0;
+    sizeMult[catSem] = 0.0;
+
+var inactiveColor = '#666';
+var startingNodeId = "1";
+var minLengthAutoComplete = 1;
+var maxSearchResults = 10;
+var strSearchBar = "Search";
+
+var cursor_size_min= 0;
+var cursor_size= 0;
+var cursor_size_max= 100;
+
+var desirableTagCloudFont_MIN=12;
+var desirableTagCloudFont_MAX=20;
+var desirableNodeSizeMIN=1;
+var desirableNodeSizeMAX=12;
+var desirableScholarSize=6; //Remember that all scholars have the same size!
+
+/*
+ *Three states:
+ *  - true: fa2 running at start
+ *  - false: fa2 stopped at start, button exists
+ *  - "off": button doesn't exist, fa2 stopped forever 
+ **/ var fa2enabled=false;//"off";
+var stopcriteria = false;
+var iterationsFA2=1000;
+var seed=999999999;//defaultseed
+var semanticConverged=false;
+var fa2seconds = 4;
+
+
+var showLabelsIfZoom=1.0;
+var greyColor = "#9b9e9e";
+
+// ============ < SIGMA.JS PROPERTIES > ============
+
+var sigmaJsDrawingProperties = {
+    defaultLabelColor: 'black',
+    defaultLabelSize: 10,//in fact I'm using it as minLabelSize'
+    defaultLabelBGColor: '#fff',
+    defaultLabelHoverColor: '#000',
+    labelThreshold: 6,
+    defaultEdgeType: 'curve',
+
+    borderSize: 2.5,//Something other than 0
+    nodeBorderColor: "default",//exactly like this
+    defaultNodeBorderColor: "black"//,//Any color of your choice
+    //defaultBorderView: "always"
+};
+var sigmaJsGraphProperties = {
+    minEdgeSize: 2,
+    maxEdgeSize: 2
+};
+var sigmaJsMouseProperties = {
+    minRatio:0.1,
+    maxRatio: 15
+};
+// ============ < / SIGMA.JS PROPERTIES > ============
+         
+
+// ============ < / DEVELOPER OPTIONS > ============
+
+
+
+// ============ < VARIABLES.JS > ============
+//"http://webchat.freenode.net/?nick=Ademe&channels=#anoe"
+var ircUrl="http://webchat.freenode.net/?nick="+ircNick+"&channels="+ircCHN;
+var twjs="tinawebJS/";
+var categories = {};
+var categoriesIndex = [];
+var wnws_buffer = null;
+var gexf;
+//var zoom=0;
+
+var checkBox=false;
+var overNodes=false;
+var shift_key=false;
+
+var NOW="A";
+var PAST="--";
+
+var swclickActual="";
+var swclickPrev="";
+var swMacro=true;
+
+var socsemFlag=false;
+var constantNGramFilter;
+
+// var nodeFilterA_past = ""
+// var nodeFilterA_now = ""
+
+// var nodeFilterB_past = ""
+// var nodeFilterB_now = ""
+
+var lastFilter = []
+    lastFilter["#sliderBNodeWeight"] = "-"
+    lastFilter["#sliderAEdgeWeight"] = "-"
+    lastFilter["#sliderBEdgeWeight"] = "-"
+
+// var edgeFilterB_past = ""
+// var edgeFilterB_now = ""
+
+
+
+var overviewWidth = 200;
+var overviewHeight = 175;
+var overviewScale = 0.25;
+var overviewHover=false;
+var moveDelay = 80, zoomDelay = 2;
+//var Vecindad;
+var partialGraph; 
+var otherGraph;
+var Nodes = []; 
+var Edges = [];
+
+var nodeslength=0;
+
+var labels = [];   
+
+var numberOfDocs=0;
+var numberOfNGrams=0;
+
+var selections = [];
+var deselections={};
+var opossites = {};
+var opos=[];
+var oposMAX;
+
+var matches = [];
+
+var nodes1 = [];
+var nodes2 = [];
+var bipartiteD2N = [];
+var bipartiteN2D = [];
+
+var flag=0;
+var firstime=0;
+var leftright=true;
+var edgesTF=false;
+
+//This variables will be updated in sigma.parseCustom.js
+var minNodeSize=1.00;
+var maxNodeSize=5.00;
+var minEdgeWeight=5.0;
+var maxEdgeWeight=0.0;
+//---------------------------------------------------
+
+var bipartite=false;
+var gexfDictReverse={}
+for (var i in gexfDict){
+    gexfDictReverse[gexfDict[i]]=i;
+}
+
+var colorList = ["#000000", "#FFFF00", "#1CE6FF", "#FF34FF", "#FF4A46", "#008941", "#006FA6", "#A30059", "#FFDBE5", "#7A4900", "#0000A6", "#63FFAC", "#B79762", "#004D43", "#8FB0FF", "#997D87", "#5A0007", "#809693", "#FEFFE6", "#1B4400", "#4FC601", "#3B5DFF", "#4A3B53", "#FF2F80", "#61615A", "#BA0900", "#6B7900", "#00C2A0", "#FFAA92", "#FF90C9", "#B903AA", "#D16100", "#DDEFFF", "#000035", "#7B4F4B", "#A1C299", "#300018", "#0AA6D8", "#013349", "#00846F", "#372101", "#FFB500", "#C2FFED", "#A079BF", "#CC0744", "#C0B9B2", "#C2FF99", "#001E09", "#00489C", "#6F0062", "#0CBD66", "#EEC3FF", "#456D75", "#B77B68", "#7A87A1", "#788D66", "#885578", "#FAD09F", "#FF8A9A", "#D157A0", "#BEC459", "#456648", "#0086ED", "#886F4C","#34362D", "#B4A8BD", "#00A6AA", "#452C2C", "#636375", "#A3C8C9", "#FF913F", "#938A81", "#575329", "#00FECF", "#B05B6F", "#8CD0FF", "#3B9700", "#04F757", "#C8A1A1", "#1E6E00", "#7900D7", "#A77500", "#6367A9", "#A05837", "#6B002C", "#772600", "#D790FF", "#9B9700", "#549E79", "#FFF69F", "#201625", "#72418F", "#BC23FF", "#99ADC0", "#3A2465", "#922329", "#5B4534", "#FDE8DC", "#404E55", "#0089A3", "#CB7E98", "#A4E804", "#324E72", "#6A3A4C", "#83AB58", "#001C1E", "#D1F7CE", "#004B28", "#C8D0F6", "#A3A489", "#806C66", "#222800", "#BF5650", "#E83000", "#66796D", "#DA007C", "#FF1A59", "#8ADBB4", "#1E0200", "#5B4E51", "#C895C5", "#320033", "#FF6832", "#66E1D3", "#CFCDAC", "#D0AC94", "#7ED379", "#012C58"];
+
+var RVUniformC = function(seed){
+    this.a=16807;
+    this.b=0;
+    this.m=2147483647;
+    this.u;
+    this.seed=seed;
+    this.x = this.seed;
+    //    this.generar = function(n){
+    //        uniforme = [];
+    //        x = 0.0;
+    //        x = this.seed;
+    //        for(i = 1; i < n ; i++){
+    //            x = ((x*this.a)+this.b)%this.m;
+    //            uniforme[i] = x/this.m;
+    //        }
+    //        return uniforme;
+    //    };
+    this.getRandom = function(){
+        x = ((this.x*this.a)+this.b)%this.m;
+        this.x = x;
+        this.u = this.x/this.m;
+        return this.u;
+    };
+}
+//unifCont = new RVUniformC(100000000)
+

templates/add_corpus_test.html

@@ -57,9 +57,9 @@
 		</div>
 			
 		<div class="fieldWrapper">
-		<label for="id_metadata">Metadata:</label>
-		{{ form.metadata.errors }}
-		<p>{{ form.metadata }}</p>
+		<label for="id_hyperdata">Hyperdata:</label>
+		{{ form.hyperdata.errors }}
+		<p>{{ form.hyperdata }}</p>
 		</div>
 		
 		<div class="fieldWrapper">

templates/chart.html

@@ -28,13 +28,13 @@
     <script type="text/template" id="filter-template">
         <span>...where</span>
         <select name="entity">
-            <option value="metadata">metadata</option>
+            <option value="hyperdata">hyperdata</option>
             <option value="ngrams">ngrams</option>
         </select>
 
-        <span class="entity metadata">
-            <select name="key"><% _.each(metadataList, function(metadata){ %>
-                <option><%= metadata.name %></option>
+        <span class="entity hyperdata">
+            <select name="key"><% _.each(hyperdataList, function(hyperdata){ %>
+                <option><%= hyperdata.name %></option>
             <% }); %></select>
         </span>
         <span class="entity ngrams">
@@ -65,10 +65,10 @@
         <li>
             <span>...where the</span>
             <select name="entity">
-                <option>metadata</option>
+                <option>hyperdata</option>
                 <option>ngrams</option>
             </select>
-            <span class="entity metadata">
+            <span class="entity hyperdata">
                 <select name="key"></select>
             </span>
             <span class="entity ngrams">

templates/explorer.html

@@ -4,7 +4,7 @@
 <link rel="stylesheet" href="{% static "css/bootstrap.css" %}">
 <link rel="stylesheet" href="{% static "css/bootstrap-theme.min.css" %}">
 <link rel="stylesheet" href="{% static "js/libs/jquery/jquery-ui.css" %}" media="screen">
-<link rel="stylesheet" href="{% static "js/libs/bootstrap/css/bootstrap.css" %}" media="screen">
+<!-- <link rel="stylesheet" href="{% static "js/libs/bootstrap/css/bootstrap.css" %}" media="screen"> -->
 <link rel="stylesheet" href="{% static "js/libs/css2/freshslider.css" %}" media="screen">
 <link rel="stylesheet" href="{% static "js/libs/css2/custom.css" %}" media="screen">
 <link rel="stylesheet" href="{% static "js/libs/css2/sidebar.css" %}" media="screen">
@@ -260,13 +260,13 @@
 
         </div>
         
-        <div id="topPapers"></div>
-        <!--
+        <!-- <div id="topPapers"></div> -->
+        
         <div id="tab-container-top" class='tab-container'>
 
            <ul class='etabs'>
              <li id="tabmed" class='tab active'><a href="#tabs3">Medline Pubs</a></li>
-             <li id="tabgps" class='tab'><a href="#tabs4">+</a></li>
+             <li id="tabgps" class='tab'><a onclick="$('#corpuses').modal('show');">+</a></li>
            </ul>
 
            <div class='panel-container'>
@@ -278,7 +278,7 @@
               </div>
            </div>
         </div>
-        -->
+        
 
 
 
@@ -350,6 +350,70 @@
 </div> 
 
 
+
+<div id="corpuses" class="modal fade">
+  <div class="modal-dialog">
+    <div class="modal-content">
+
+
+      <div class="modal-header">
+        <button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button>
+        <h4 class="modal-title">Corpus Comparison</h4>
+      </div>
+
+
+
+      <div class="modal-body form-horizontal">
+        Chose another corpus to compare with:
+        <div class="form-group">
+          <label class="col-lg-2 control-label"> </label>
+          <div class="col-lg-10">
+
+
+              <form id="corpuses_form" role="form">
+
+                <ul>
+
+                  {% if corpusinfo  %}
+                    {% for k1, v1 in corpusinfo.items %}
+                        {% if v1.corpuses|length > 0  %}
+                            <br><li><a href="/project/{{k1}}/">{{v1.proj_name}}</a><br>
+                              <ul style="list-style-type: none;">
+                              {% for c in v1.corpuses %}
+                                <li>
+                                  <div class="radio">
+                                    <label><input type="radio" id="{{c.id}}" name="optradio">
+                                      <a href="/project/{{k1}}/corpus/{{c.id}}/">{{c.name}}</a>
+                                    </label>
+                                  </div>
+                                </li>
+                              {% endfor %}
+                              </ul>
+                            </li>
+                        {% endif %}
+                    {% endfor %}
+                  {% endif %}
+
+                </ul>
+
+              </form>
+
+
+          </div>
+        </div>
+      </div>
+
+
+      <div class="modal-footer">
+        <button id="closecorpuses" type="button" class="btn btn-default" data-dismiss="modal">Close</button>
+        <button type="button" class="btn btn-primary" onclick='printCorpuses();'>Add Tab</button>
+      </div>
+
+    </div>
+  </div>
+</div> 
+
+
 <div id="modalloader" class="modal fade">
     <div id="loader" class="loader">
         <img src="{% static "js/libs/img2/loader.gif" %}" ></img>

templates/menu-bottom.html

-<div class="container">
-		<div class="row">
-				<div class="col-md-4">
-						<div class="jumbotron">
-								<h3><a href="/graph-it">1) Documents</a></h3>
-						<ol>
-								<li>Read</li> <!-- write -->
-								<li>Count</li> <!-- compute -->
-								<li>Select</li> <!-- cut -->
-						</ol>
-					</div>
-				</div>
-
-				<div class="col-md-4">
-						<div class="jumbotron">
-								<h3><a href="/graph">2) Ngrams</a></h3>
-						<ol>
-								<li>White Lists</li>
-								<li>Black Lists</li>
-								<li>Synonyms</li>
-						</ol>
-						</div>
-				</div>
-
-				<div class="col-md-4">
-						<div class="jumbotron">
-						<h3><a href="/graph">3) Visualizations</a></h3>
-						<ol>
-								<li>Matrix</li>
-								<li>Static maps</li>
-								<li>Dynamics maps</li>
-						</ol>
-						</div>
-				</div>
-		</div>
-</div>
-
-

templates/menu.html

@@ -105,6 +105,25 @@ $(function() {
 		}
 });});</script>
 
+
+{% if debug == False %}
+<!-- Piwik -->
+<script type="text/javascript">
+  var _paq = _paq || [];
+	  _paq.push(['trackPageView']);
+	  _paq.push(['enableLinkTracking']);
+	  (function() {
+	    var u="//piwik.iscpif.fr/";
+	    _paq.push(['setTrackerUrl', u+'piwik.php']);
+	    _paq.push(['setSiteId', 2]);
+	    var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
+	    g.type='text/javascript'; g.async=true; g.defer=true; g.src=u+'piwik.js'; s.parentNode.insertBefore(g,s);
+	  })();
+</script>
+<noscript><p><img src="//piwik.iscpif.fr/piwik.php?idsite=2" style="border:0;" alt="" /></p></noscript>
+<!-- End Piwik Code -->
+{% endif %}
+
 </body>
 </html>
 

templates/node-info.html

@@ -40,7 +40,7 @@
 {% block content %}
 {% if node_id %}
 <div id="nodeid" style="visibility: hidden;" >{{node_id}}</div>
-<div id="metadata"></div>
+<div id="hyperdata"></div>
 {% endif %}
 
 
@@ -60,7 +60,7 @@
 
     var node_id = $("#nodeid").html()
 	var img = '<center><img src="'+window.location.origin+'/static/img/ajax-loader.gif" width="30%"></img></center>';
-	$("#metadata").html(img);
+	$("#hyperdata").html(img);
 
     $.ajax({
         type: 'GET',
@@ -69,7 +69,7 @@
         //dataType: 'json',
         success : function(data){
         	console.log( window.location.origin+'/api/nodes/'+node_id )
-        	jsondata = data.metadata
+        	jsondata = data.hyperdata
         	console.log(jsondata)
         	var output = ""
         	// if(jsondata.title) output += "Title:<br>"+jsondata.title+"<br>";
@@ -92,7 +92,7 @@
         	}
         	output += "</div>"
 
-            $("#metadata").html(output);
+            $("#hyperdata").html(output);
         },
         error: function(){ 
             console.log('Page Not found: getTopPapers()');

templates/project.html

@@ -280,7 +280,12 @@
 				console.log("we've in dynamic mode")
 				$("#simpleloader").html('<img width="30px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img>')
 				$("#submit_thing").prop('onclick',null);
-				doTheQuery();
+
+				var theType = $("#id_type option:selected").html();
+				if(theType=="Pubmed (xml format)") doTheQuery();
+				if(theType=="istex") {
+					testISTEX(origQuery.replace(" ","+"),1000)
+				}
 			}
 			else {
 				console.log("we dont have nothing inside results div")
@@ -328,7 +333,7 @@
 					console.log("enabling "+"#"+value.id)
 					$("#"+value.id).attr('onclick','getGlobalResults(this);');
 					// $("#submit_thing").prop('disabled' , false)
-					$("#submit_thing").html("Process a 100 sample!")
+					$("#submit_thing").html("Process a 1000 sample!")
 
 		            thequeries = data
 		            var N=0,k=0;
@@ -365,7 +370,7 @@
 					console.log("enabling "+"#"+value.id)
 					$("#"+value.id).attr('onclick','getGlobalResults(this);');
 					// $("#submit_thing").prop('disabled' , false)
-					$("#submit_thing").html("Process a 100 sample!")
+					$("#submit_thing").html("Process a 1000 sample!")
 
 		            thequeries = data
 		            var N=data.length,k=0;

templates/project.html_backup

-
-{% extends "menu.html" %}
-
-{% block css %}
-{% load staticfiles %}
-<link rel="stylesheet" href="{% static "css/bootstrap.css" %}">
-
-<link rel="stylesheet" type="text/css" href="{% static "css/morris.css" %}">
-<link rel="stylesheet" type="text/css" href="{% static "css/jquery.easy-pie-chart.css"%}">
-<script type="text/javascript" src="{% static "js/jquery/jquery.min.js" %}"></script>
-
-<script src="{% static "js/raphael-min.js"%}"></script>
-<script src="{% static "js/morris.min.js"%}"></script>
-
-
-{% endblock %}
-
-
-
-{% block content %}
-
-<div class="container theme-showcase" role="main">
-        <div class="jumbotron">
-				<div class="row">
-        <div class="col-md-6">
-            {% if project %}
-            <h1>{{ project.name }}</h1>
-						<!--<h3> {{number}} corpora </h3>-->
-            {% endif %}
-        </div>
-
-        <div class="col-md-4">
-				<p>
-				{% if donut %}
-	  		<div id="hero-donut" style="height: 200px;"></div>
-				{% endif %}
-        <center>
-            <button 
-								type="button" 
-								class="btn btn-primary btn-lg" 
-								data-container="body" 
-								data-toggle="popover" 
-								data-placement="bottom"
-								>Add a corpus</button>
-								<div id="popover-content" class="hide">
-
-					<form enctype="multipart/form-data" action="/project/{{project.id}}/" method="post">
-    {% csrf_token %}
-		{{ form.non_field_errors }}
-		{{ form.as_p}}
-		
-		{{ formResource.non_field_errors }}
-		{{ formResource.as_p}}
-		<input onclick='$("#semLoader").css("visibility", "visible"); $("#semLoader").show();' type="submit" name="submit" id="submit" class="btn" value="Add this corpus" /><div>
-    </center>
-				</p>
-
-								</div>
-		        </div>
-
-        </div>
-
-
-        </div>
-        </div>
-</div>
-<!-- Add jumbotron container for each type of corpus (presse, science etc.) -->
-
-          <div id="semLoader" style="position:absolute; top:50%; left:40%; width:80px; visibility: hidden;">
-							<img src="{% static "js/libs/img2/loading-bar.gif" %}"></img>
-          </div>
-
-<div class="container">
-				{% if list_corpora  %}
-								<h1>Resources</h1>
-								<h2>Corpora</h2>
-										<ul>
-										{% for key, corpora in list_corpora.items %}
-										<li>{{ key }}</li>
-												<ul>
-														{% for corpus in corpora %}
-														<li> {% ifnotequal corpus.count 0 %}
-																		<a href="/project/{{project.id}}/corpus/{{corpus.id}}"> 
-																			{{corpus.name}}
-																		</a>
-																		, {{ corpus.count }} Documents 
-																 {% else %}
-																 {{corpus.name}} : <img width="20px" src="{% static "js/libs/img2/loading-bar.gif" %}"></img> Processing, drink a cup of tea, and refresh the page :)
-																 {% endifnotequal %}
-																		<button type="button" class="btn btn-xs btn-default" data-container="body" data-toggle="popover" data-placement="bottom" 
-																		data-content='
-																		<ul>
-																		<li> Rename </li>
-																		<li> Add new documents </li>
-																		<li><a href="/project/{{ project.id }}/corpus/{{ corpus.id}}/delete">Delete</a></li>
-																		</ul>
-																		'>Manage</button>
-																</li>
-														{% endfor %}
-												</ul>
-										{% endfor %}
-										</ul>
-				{% endif %}
-
-
-								{% if list_corporax  %}
-										<div class="col-md-4">
-                        <h3><a href="/project/{{project.id}}/corpus/{{corpus.id}}">{{corpus.name}}</a>
-												</h3>
-												<h4>{{ corpus.count }} Documents </h4>
-												<h5>Activity:</h5>
-												<div class="chart" data-percent="73">73%</div>
-										</div>
-								{% endif %}
-				
-						{% if whitelists  %}
-						<h2>Lists of Ngrams</h2>
-								<h3>White Lists</h2>
-								{% for list in whitelists %}
-										<ul>
-                    <li> {{list.name }}
-										</ul>
-								{% endfor %}
-						{% endif %}
-						
-            {% if whitelists  %}
-						<h3>Black Lists</h2>
-								{% for list in blacklists %}
-										<ul>
-                    <li> {{list.name }}
-										</ul>
-								{% endfor %}
-						{% endif %}
-						
-
-				
-        {% if cooclists  %}
-						<h2>Results (graphs)</h2>
-						<h3>Cooccurrences Lists</h2>
-								{% for list in cooclists %}
-										<ul>
-                    <li> {{list.name }}
-										</ul>
-								{% endfor %}
-						{% endif %}
-			  
-</div>
-
-
-<script>
-        // Morris Donut Chart
-        Morris.Donut({
-            element: 'hero-donut',
-            data: [
-            {% if donut %}
-            {% for part in donut %}
-						{label: '{{ part.source }}', value: {{ part.part }} },
-            {% endfor %}
-            {% endif %}
-
-            ],
-            colors: ["@white", "@white"],
-            //colors: ["#30a1ec", "#76bdee"],
-            formatter: function (y) { return y + "%" }
-        });
-</script>
-
-
-
-
-{% endblock %}

templates/subcorpus.html

-
-
-<div class="pagination">
-    <span class="step-links">
-        {% if documents.has_previous %}
-            <a style="cursor: pointer;" onclick="updateDocuments({{ documents.previous_page_number }},true);">previous</a>
-        {% endif %}
-
-        <div id="currentpage" style="display: none;">{{ documents.number }}</div>
-        <span class="current">
-            <strong>Page {{ documents.number }}</strong> of {{ documents.paginator.num_pages }}.
-        </span>
-
-        {% if documents.has_next %}
-            <a style="cursor: pointer;" onclick="updateDocuments({{ documents.next_page_number }},true);">next</a>
-        {% endif %}
-    </span>
-</div>
-
-{% if documents %}
-
-<ul>
-{% for doc in documents %}
-    {% if doc.date %}
-    <li><div id="doc_{{doc.id}}"> <b>{{ doc.date }}</b>: <a target="_blank" href="/nodeinfo/{{doc.id}}">{{ doc.name}}</a> , @ {{ doc.metadata.source}}</div></li>
-    {% endif %}
-{% endfor %}
-
-
-          <div id="delAll" style="visibility: hidden;">
-							<center>
-              <button onclick="deleteDuplicates(theurl);">Delete all Duplicates in one click</button>
-							</center>
-          </div>
-
-
-</ul>
-
-<script>
-
-function getCookie(name) {
-    var cookieValue = null;
-    if (document.cookie && document.cookie != '') {
-        var cookies = document.cookie.split(';');
-        for (var i = 0; i < cookies.length; i++) {
-            var cookie = jQuery.trim(cookies[i]);
-            // Does this cookie string begin with the name we want?
-            if (cookie.substring(0, name.length + 1) == (name + '=')) {
-                cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
-                break;
-            }
-        }
-    }
-    return cookieValue;
-}
-
-function deleteDuplicates(url) {
-    console.log("hello world")
-    console.log(url)
-    $.ajax({
-      url: url,
-      type: 'DELETE',
-      beforeSend: function(xhr) {
-        xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
-      },
-      success: function(data) {
-        console.log("in DeleteDuplicates")
-        console.log(data)
-        location.reload();
-      },
-        error: function(result) {
-            console.log("Data not found");
-            console.log(result)
-        }
-    });
-}
-
-function deleteNode(node_id) {
-    $.ajax({
-        url: '/api/nodes/'+node_id,
-        type: 'DELETE',
-        beforeSend: function(xhr) {
-            xhr.setRequestHeader("X-CSRFToken", getCookie("csrftoken"));
-        },
-        success: function(result) {
-            console.log("deleting: ")
-            console.log(current_docs[node_id])
-            var title = current_docs[node_id]
-            console.log(result)
-            var currentpage = $("#currentpage").html()
-            console.log("vaciando div de subcorpus")
-            $("#divsubcorpus").html("")
-            console.log("before:")
-            console.log(BIS_dict[title])
-            BIS_dict[title][0]--;
-            BIS_dict[title][1]++;
-            console.log("after:")
-            console.log(BIS_dict[title])
-            // updateDocuments( currentpage , true );
-            console.log("after delete:     updateDocuments( "+currentpage+" , true )")
-            updateDocuments( currentpage , true )
-            // $.when(  updateDocuments( currentpage , true )  ).done(function(a1) {
-            //     console.log("inside the wheeeeen")
-            //     console.log(BIS_dict[title])
-            //     if(BIS_dict[title][0] == BIS_dict[title][1]) {
-
-            //         $.doTimeout( 1000, function(){
-            //             var elems = current_docs[title]
-            //             for (var i in elems ) {
-            //                 var id = elems[i]
-            //                 console.log("deleting icons for :"+"#delete_"+id)
-            //                 $("#delete_"+id).remove();  
-            //             }
-            //         });
-            //     }
-            //     // // // if(BIS_dict[title][0] == BIS_dict[title][1])
-            //     // // //     $("#delete_"+node_id).remove();         
-            //     // // current_docs = []
-            //     // {% for doc in documents %}
-            //     //     id = "doc_{{doc.id}}"
-            //     //     title = "{{doc.name}}"
-            //     //     console.log(BIS_dict[title])
-            //     //     if(BIS_dict[title] && BIS_dict[title][0] > BIS_dict[title][1]) {
-            //     //         var del_img = '<span id="delete_{{doc.id}}"><a title="Delete this document!" style="cursor: pointer;" onclick="deleteNode('+"{{doc.id}}"+',\''+title+'\')"><img width="20px" src="/static/img/delete-big.png"></img></a><span>'
-            //     //         $("#"+id).prepend( del_img )
-            //     //     }
-            //     // {% endfor %}
-            // });
-
-        },
-        error: function(result) {
-            console.log("Data not found");
-            console.log(result)
-        }
-    });
-}
-
-
-
-//'+"{{doc.id}}"+',\"'+title+'\"
-
-current_docs = {}
-if(Object.keys(BIS_dict).length>0) {
-    $("#delAll").css("visibility", "visible"); 
-    $("#delAll").show();
-} else $("#delAll").remove()
-
-{% for doc in documents %}
-    id = "doc_{{doc.id}}"
-    title = $("<div/>").html("{{doc.name}}").text()
-    current_docs["{{doc.id}}"] = title;
-    if(BIS_dict[title] && BIS_dict[title][0] > BIS_dict[title][1]) {
-        jspart = 'onclick= "deleteNode( {{doc.id}} );"'
-        var del_img = '<a title="Delete this document!" style="cursor: pointer;" '+jspart+'><img width="20px" src="/static/img/delete-big.png"></img></a>'
-        $("#"+id).prepend( del_img )
-    }
-{% endfor %}
-
-</script>
-{% endif %}

tests/AlexandreTests.py

@@ -185,7 +185,7 @@ science.children.count()
 
 # In[25]:
 
-science.children.last().metadata
+science.children.last().hyperdata
 
 
 # In[26]: