# coding: utf-8

# In[1]:

from node.models import Node, NodeType,                        Project, Corpus, Document,                        Ngram, Node_Ngram,                        User, Language, ResourceType


# In[2]:

import pycountry

for language in pycountry.languages:
    try:
        implemented = 1 if language.alpha2 in ['en', 'fr'] else 0
        Language(iso2=language.alpha2, iso3=language.terminology, fullname=language.name, implemented=implemented).save()
    except:
        pass


# In[3]:

english = Language.objects.get(iso2='en')
french  = Language.objects.get(iso2='fr')


# In[4]:

try:
    me = User.objects.get(username='alexandre')
except:
    me = User(username='alexandre')
    me.save()


# In[5]:

try:
    typeProject = NodeType.objects.get(name='Project')
except Exception as error:
    print(error)
    typeProject = NodeType(name='Project')
    typeProject.save()  

try:
    typeCorpus  = NodeType.objects.get(name='Corpus')
except Exception as error:
    print(error)
    typeCorpus  = NodeType(name='Corpus')
    typeCorpus.save()
    
try:
    typeDoc     = NodeType.objects.get(name='Document')
except Exception as error:
    print(error)
    typeDoc     = NodeType(name='Document')
    typeDoc.save()


# In[6]:

try:
    typePubmed = ResourceType.objects.get(name='pubmed')
    typeIsi    = ResourceType.objects.get(name='isi')
    typeRis    = ResourceType.objects.get(name='ris')
    typePresse = ResourceType.objects.get(name='europress')

except Exception as error:
    print(error)
    
    typePubmed = ResourceType(name='pubmed')
    typePubmed.save()  
    
    typeIsi    = ResourceType(name='isi')
    typeIsi.save()
    
    typeRis    = ResourceType(name='ris')
    typeRis.save()
    
    typePresse = ResourceType(name='europress')
    typePresse.save()


# In[10]:

Node.objects.all().delete()


# In[8]:

try:
    project = Node.objects.get(name='Bees project')
except:
    project = Node(name='Bees project', type=typeProject, user=me)
    project.save()


# ### Pubmed

# In[18]:

try:
    corpus_pubmed = Node.objects.get(name='PubMed corpus')
except:
    corpus_pubmed = Node(parent=project, name='PubMed corpus', type=typeCorpus, user=me)
    corpus_pubmed.save()


# In[19]:

corpus_pubmed.add_resource(file='/srv/gargantext_lib/data_samples/pubmedBig.zip', type=typePubmed)


# In[20]:

#corpus_abeille.add_resource(file='/srv/gargantext_lib/data_samples/pubmed.zip', type=typePubmed)


# In[21]:

corpus_pubmed.parse_resources()
corpus_pubmed.children.count()


# In[22]:

corpus_pubmed.children.all().extract_ngrams(['title',])
Node_Ngram.objects.filter(node=corpus_pubmed.children.all()[0]).count()


# ### RIS

# In[9]:

try:
    corpus_ris = Node.objects.get(name='RIS corpus')
except:
    corpus_ris = Node(parent=project, name='RIS corpus', type=typeCorpus, user=me)
    corpus_ris.save()


# In[10]:

corpus_ris.add_resource(file='/srv/gargantext_lib/data_samples/risUnix.zip', type=typeRis)


# In[15]:

corpus_ris.parse_resources()


# In[16]:

corpus_ris.children.count()


# In[40]:

corpus_ris.children.all()


# In[28]:

corpus_ris.name = "ZOTERO CORPUS (CIRDEM)"
corpus_ris.save()


# ### Science

# In[23]:

try:
    science = Node.objects.get(name='WOS corpus')
except:
    science = Node(parent=project, name='WOS corpus', type=typeCorpus, user=me)
    science.save()


# In[24]:

science.add_resource(file='/srv/gargantext_lib/data_samples/isi.zip', type=typeIsi)
science.parse_resources()
science.children.count()


# In[25]:

science.children.last().metadata


# In[26]:

science.children.all().extract_ngrams(['abstract',])
Node_Ngram.objects.filter(node=science.children.all()[0]).count()


# ### Press

# In[29]:

try:
    presse = Node.objects.get(name='Presse corpus')
except:
    presse = Node(parent=project, name='Presse corpus', type=typeCorpus, user=me)
    presse.save()


# In[30]:

presse.add_resource(file='/srv/gargantext_lib/data_samples/html/html_french.zip', type=typePresse)


# In[31]:

presse.parse_resources()


# In[32]:

presse.children.count()


# In[33]:

presse.children.all().extract_ngrams(['title',])


# In[34]:

project.children.all()


# In[37]:

corpus.children.all()


# In[46]:

liste_ordered = collections.OrderedDict(sorted(liste.items()), key=lambda t: t[1])


# In[52]:

#liste_ordered


# # Création des Listes

# In[57]:

import collections


# In[58]:

liste = collections.defaultdict(int)


# In[59]:

try:
    whitelist_type  = NodeType.objects.get(name='WhiteList')
    blacklist_type = NodeType.objects.get(name='BlackList')
except:
    whitelist_type = NodeType(name='WhiteList')
    whitelist_type.save()
    
    blacklist_type = NodeType(name='BlackList')
    blacklist_type.save()

white_node = Node.objects.create(name='WhiteList Pubmed', user=me, parent=corpus_pubmed, type=whitelist_type)
black_node = Node.objects.create(name='BlackList Pubmed', user=me, parent=corpus_pubmed, type=blacklist_type)


# In[60]:

Node_Ngram.objects.filter(node=white_node).count()


# # Création de la white list

# In[61]:

with transaction.atomic():
    for node in corpus_pubmed.children.all():
        for node_ngram in Node_Ngram.objects.filter(node=node):
            if node_ngram.ngram.n > 1:
                #liste[node_ngram.ngram.terms] += node_ngram.weight
                Node_Ngram.objects.create(node=white_node, ngram=node_ngram.ngram, weight=1)


# In[62]:

white_node.pk


# In[63]:

Node_Ngram.objects.filter(node=white_node).count()


# # Création de la black list

# In[64]:

with transaction.atomic():
    for node_ngram_object in Node_Ngram.objects.all()[101:150]:
        Node_Ngram.objects.create(node=black_node, ngram=node_ngram_object.ngram, occurences=1)


# In[12]:

Node_Ngram.objects.filter(node=black_node)


# # Création des synonymes

# In[13]:

syno_type  = NodeType.objects.get(name='Synonyme')
syno_node = Node.objects.create(name='Syno Pubmed',
                                user=user, 
                                parent=corpus, 
                                type=syno_type)


# In[23]:

synonyme1, synonyme2 = Node_Ngram.objects.filter(node=white_node)[3:5]


# In[24]:

NodeNgramNgram.objects.create(node=syno_node, ngramX=synonyme1.ngram, ngramY=synonyme2.ngram)


# # Cooccurrence

# In[65]:

white_node.children.count()


# In[66]:

black_node.pk


# In[67]:

try:
    cooc_type  = NodeType.objects.get(name='Cooccurrence')
except:
    cooc_type = NodeType(name='Cooccurrence')
    cooc_type.save()


# In[68]:

cooc = Node.objects.create(user=me,                           parent=corpus_pubmed,                           type=cooc_type,                           name="Cooccurrences calcul Alpha")


# In[69]:

cooc.pk


# In[152]:

white_node.children.all().delete()


# In[70]:

from django.db import connection
cursor = connection.cursor()
# LOCK TABLE documents_ngramtemporary IN EXCLUSIVE MODE;
query_string = """
INSERT INTO node_nodengramngram (node_id, "ngramX_id", "ngramY_id", score)

SELECT 
%d as node_id, x.ngram_id, y.ngram_id, COUNT(*) AS score

FROM
node_node_ngram AS x

INNER JOIN 
node_node_ngram AS y 
ON x.node_id = y.node_id


WHERE
x.id in (select id from node_node_ngram WHERE node_id = %d )
AND
y.id in (select id from node_node_ngram WHERE node_id = %d )
AND
x.ngram_id <> y.ngram_id


GROUP BY
x.ngram_id, y.ngram_id

HAVING count(*) > 1

ORDER BY score

LIMIT 300

             """ % (cooc.pk, white_node.pk, white_node.pk)

cursor.execute(query_string)

try:
    while True:
        row = cursor.fetchone()
        if row is None:
            break
        print(row)
except:
    pass


# In[1]:




# In[45]:




# In[71]:

from copy import copy
import numpy as np
import pandas as pd
import networkx as nx
from collections import defaultdict
from analysis.louvain import *
import matplotlib.pyplot as plt
get_ipython().magic('matplotlib inline')


# In[ ]:

matrix = ""


# In[72]:

matrix = defaultdict(lambda : defaultdict(float))
for cooccurrence in NodeNgramNgram.objects.filter(node=cooc):
    if cooccurrence.score > 1 :
        #print(x.ngramX.terms, x.ngramY.terms)
        matrix[cooccurrence.ngramX.terms][cooccurrence.ngramY.terms] = cooccurrence.score
        matrix[cooccurrence.ngramY.terms][cooccurrence.ngramX.terms] = cooccurrence.score


# In[73]:

df = pd.DataFrame(matrix).T.fillna(0)
x = copy(df.values)


# In[74]:

x = np.where((x.sum(axis=1) > x.shape[0] / 2), 0, x )
x = np.where((x.sum(axis=1) > x.shape[0] / 10), 0, x )


# In[75]:

x = x / x.sum(axis=1)


# In[76]:

matrix_filtered = np.where(x > .4, 1, 0)


# In[77]:

matrix_filtered


# In[78]:

G = nx.from_numpy_matrix(matrix_filtered)
G = nx.relabel_nodes(G, dict(enumerate(df.columns)))


# In[79]:

nx.draw(G, with_labels=True)
plt.show()


# In[80]:

partition = best_partition(G)


# In[ ]:

#partition


# In[81]:

pos = nx.spring_layout(G)


# In[82]:

count = 0.0
node_min = 3
for com in set(partition.values()) :
    count = count + 1
    list_nodes = [nodes for nodes in partition.keys() if partition[nodes] == com]
            
    if len(list_nodes) > node_min:
        nx.draw_networkx_nodes(G, pos, list_nodes, node_size = 20, with_labels=True)#, node_color = str(count / size))
        nx.draw_networkx_edges(G, pos, alpha=0.5)
        plt.title("Clique " + str(count))
                
        for node in list_nodes: 
            print(node)
            plt.show()
            print("-" * 30)


# In[ ]:




# In[ ]: