Skip to content

gargantext
gargantext
Commit b358f8e9 authored by Mathieu Rodic's avatar Mathieu Rodic
Browse files
Options

Merge branch 'master' of ssh://delanoe.org:1979/gargantext into mat

parents 2514ecdf 79c77e3e
Show whitespace changes
Inline Side-by-side
Showing with 9473 additions and 120 deletions
SQL_TESTS.ipynb
View file @ b358f8e9
{ {
"metadata": { "metadata": {
"name": "", "name": "",
"signature": "sha256:33c2f41e3ea5983e768350b4012544242c5df9b394091647362f00929812a921" "signature": "sha256:65f487ee62067486e4f832ed088fe02ede3daa27052c9dcaf58b3edffa169245"
}, },
"nbformat": 3, "nbformat": 3,
"nbformat_minor": 0, "nbformat_minor": 0,
...@@ -27,6 +27,17 @@ ...@@ -27,6 +27,17 @@
"outputs": [], "outputs": [],
"prompt_number": 1 "prompt_number": 1
}, },
{
"cell_type": "code",
"collapsed": false,
"input": [
"me = User.objects.get(username='alexandre')"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 2
},
{ {
"cell_type": "code", "cell_type": "code",
"collapsed": false, "collapsed": false,
...@@ -37,7 +48,7 @@ ...@@ -37,7 +48,7 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 15 "prompt_number": 2
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -48,7 +59,7 @@ ...@@ -48,7 +59,7 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 1 "prompt_number": 3
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -227,12 +238,12 @@ ...@@ -227,12 +238,12 @@
"collapsed": false, "collapsed": false,
"input": [ "input": [
"#\u00a0corpus = Node.objects.filter(type=typeCorpus).first()\n", "#\u00a0corpus = Node.objects.filter(type=typeCorpus).first()\n",
"corpus = Node.objects.get(id=44338)" "corpus = Node.objects.get(id=13064)"
], ],
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 4 "prompt_number": 3
}, },
{ {
"cell_type": "heading", "cell_type": "heading",
...@@ -279,30 +290,30 @@ ...@@ -279,30 +290,30 @@
"output_type": "stream", "output_type": "stream",
"stream": "stdout", "stream": "stdout",
"text": [ "text": [
"'2004/01/02','2'\n", "'1954/11/18','2'\n",
"'2004/01/03','1'\n", "'1958/11/18','1'\n",
"'2004/01/06','8'\n", "'1959/11/18','1'\n",
"'2004/01/07','5'\n", "'1968/11/18','1'\n",
"'2004/01/08','8'\n", "'1969/11/18','2'\n",
"'2004/01/09','1'\n", "'1971/11/18','4'\n",
"'2004/01/10','3'\n", "'1972/11/18','1'\n",
"'2004/01/12','2'\n", "'1974/11/18','2'\n",
"'2004/01/13','6'\n", "'1975/11/18','3'\n",
"'2004/01/15','2'\n", "'1976/11/18','1'\n",
"'2004/01/16','1'\n", "'1977/11/18','6'\n",
"'2004/01/17','5'\n", "'1978/11/18','11'\n",
"'2004/01/19','2'\n", "'1979/11/18','9'\n",
"'2004/01/20','2'\n", "'1980/11/18','6'\n",
"'2004/01/21','7'\n", "'1981/11/18','4'\n",
"'2004/01/23','1'\n", "'1982/11/18','7'\n",
"'2004/01/24','4'\n", "'1983/11/18','14'\n",
"'2004/01/25','4'\n", "'1984/11/18','17'\n",
"'2004/01/26','5'\n", "'1985/11/18','18'\n",
"'2004/01/27','2'\n" "'1986/02/21','1'\n"
] ]
} }
], ],
"prompt_number": 37 "prompt_number": 6
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -320,12 +331,17 @@ ...@@ -320,12 +331,17 @@
" node_ngram AS ngX ON ngX.id = nngX.ngram_id\n", " node_ngram AS ngX ON ngX.id = nngX.ngram_id\n",
" WHERE\n", " WHERE\n",
" n.parent_id = %s\n", " n.parent_id = %s\n",
" AND\n",
" ngX.n >= 2\n",
" GROUP BY\n", " GROUP BY\n",
" ngX.terms\n", " ngX.terms\n",
" Having\n",
" COUNT(*) > 7\n",
" ORDER BY\n", " ORDER BY\n",
" occurrences DESC\n", " occurrences DESC\n",
" LIMIT\n", " LIMIT\n",
" 20\n", " 100\n",
" \n",
"\"\"\", [corpus.id])\n", "\"\"\", [corpus.id])\n",
"\n", "\n",
"while True:\n", "while True:\n",
...@@ -341,30 +357,74 @@ ...@@ -341,30 +357,74 @@
"output_type": "stream", "output_type": "stream",
"stream": "stdout", "stream": "stdout",
"text": [ "text": [
"(196, 'patients')\n", "(138, 'honey bees')\n",
"(135, 'voice')\n", "(132, 'apis mellifera')\n",
"(129, 'study')\n", "(69, 'honey bee')\n",
"(111, 'disease')\n", "(66, 'apis mellifera l')\n",
"(69, 'treatment')\n", "(45, 'pesticide residues')\n",
"(66, 'life')\n", "(39, 'gas chromatography')\n",
"(58, 'patient')\n", "(36, 'varroa destructor')\n",
"(53, 'quality')\n", "(36, 'honey bee colonies')\n",
"(49, 'care')\n", "(30, 'sublethal effects')\n",
"(45, 'use')\n", "(27, 'apidae )')\n",
"(44, 'Patients')\n", "(21, 'neonicotinoid insecticides')\n",
"(43, 'people')\n", "(21, 'honey bee ( hymenoptera')\n",
"(41, 'development')\n", "(18, 'bee products')\n",
"(41, 'purpose')\n", "(18, 'megachile rotundata')\n",
"(40, 's disease')\n", "(18, 'solid-phase extraction')\n",
"(39, 's')\n", "(18, 'simultaneous determination')\n",
"(38, 'results')\n", "(18, 'mass spectrometric')\n",
"(37, 'diagnosis')\n", "(15, 'case study')\n",
"(36, 'years')\n", "(15, 'honey samples')\n",
"(34, 'women')\n" "(15, 'liquid chromatography')\n",
"(15, 'high performance liquid chromatography')\n",
"(15, 'varroa mites')\n",
"(12, 'organochlorine pesticides')\n",
"(12, 'gas chromatography-mass spectrometry')\n",
"(12, 'liquid chromatography-mass spectrometry')\n",
"(12, 'colony health')\n",
"(12, 'gas chromatographic')\n",
"(12, 'colony collapse disorder')\n",
"(12, 'bumble bees')\n",
"(12, 'varroa jacobsoni')\n",
"(9, 'chemiluminescent elisa')\n",
"(9, 'diversionary plantings for reduction of pesticide related bee mortality')\n",
"(9, 'pesticides and law')\n",
"(9, 'plant protection products')\n",
"(9, 'nomia melanderi')\n",
"(9, 'electron-capture detection')\n",
"(9, 'managed pollinator cap coordinated agricultural project a national research')\n",
"(9, 'apis florea f')\n",
"(9, 'solid-phase microextraction')\n",
"(9, 'extension initiative')\n",
"(9, 'crop pollination')\n",
"(9, 'non-apis bees')\n",
"(9, 'honey bees ( apis mellifera')\n",
"(9, 'liquid chromatography-tandem mass spectrometry')\n",
"(9, 'bee pollen')\n",
"(9, 'foraging behavior')\n",
"(9, 'biological control')\n",
"(9, 'nosema ceranae')\n",
"(9, 'organophosphorus pesticides')\n",
"(9, 'field conditions')\n",
"(9, 'honey bee apis mellifera l')\n",
"(9, 'laboratory tests')\n",
"(9, 'beauveria bassiana')\n",
"(9, 'comparative toxicity')\n",
"(9, 'high levels')\n",
"(9, 'pesticide exposure')\n",
"(9, 'fluvalinate residues')\n",
"(9, 'insecticide residues')\n",
"(9, 'osmia lignaria')\n",
"(9, 'bombus impatiens')\n",
"(9, 'honey bee health')\n",
"(9, 'agricultural landscape')\n",
"(9, 'dispersive liquid-liquid microextraction')\n",
"(9, 'matrix solid-phase dispersion')\n"
] ]
} }
], ],
"prompt_number": 104 "prompt_number": 28
}, },
{ {
"cell_type": "heading", "cell_type": "heading",
...@@ -455,7 +515,7 @@ ...@@ -455,7 +515,7 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 21 "prompt_number": 17
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -470,7 +530,7 @@ ...@@ -470,7 +530,7 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 22 "prompt_number": 18
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -485,7 +545,7 @@ ...@@ -485,7 +545,7 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 23 "prompt_number": 19
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -497,7 +557,7 @@ ...@@ -497,7 +557,7 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 24 "prompt_number": 22
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -514,7 +574,7 @@ ...@@ -514,7 +574,7 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 25 "prompt_number": 23
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -528,13 +588,13 @@ ...@@ -528,13 +588,13 @@
{ {
"metadata": {}, "metadata": {},
"output_type": "pyout", "output_type": "pyout",
"prompt_number": 26, "prompt_number": 24,
"text": [ "text": [
"6" "61297"
] ]
} }
], ],
"prompt_number": 26 "prompt_number": 24
}, },
{ {
"cell_type": "heading", "cell_type": "heading",
...@@ -592,7 +652,7 @@ ...@@ -592,7 +652,7 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 47 "prompt_number": 26
}, },
{ {
"cell_type": "heading", "cell_type": "heading",
...@@ -627,7 +687,60 @@ ...@@ -627,7 +687,60 @@
" WHERE\n", " WHERE\n",
" n.parent_id = %s\n", " n.parent_id = %s\n",
" AND\n", " AND\n",
" nngX.ngram_id > nngY.ngram_id\n", " nngX.ngram_id in (select id from node_node_ngram WHERE node_id = 61298 )\n",
" AND\n",
" nngY.ngram_id in (select id from node_node_ngram WHERE node_id = 61298 )\n",
" AND\n",
" nngX.ngram_id <> nngY.ngram_id\n",
" \n",
" GROUP BY\n",
" ngX.id,\n",
" ngX.terms,\n",
" ngY.id,\n",
" ngY.terms\n",
" ORDER BY\n",
" cooccurrences DESC\n",
" LIMIT\n",
" 200\n",
"\"\"\", [corpus.id])\n",
"\n",
"while True:\n",
" row = cursor.fetchone()\n",
" if row is None:\n",
" break\n",
" print(row)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 15
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"cursor.execute(\"\"\"\n",
" SELECT\n",
" COUNT(*) AS cooccurrences,\n",
" ngX.terms,\n",
" ngY.terms\n",
" FROM\n",
" node_node AS n\n",
" \n",
" INNER JOIN\n",
" node_node_ngram AS nngX ON nngX.node_id = n.id\n",
" INNER JOIN\n",
" node_ngram AS ngX ON ngX.id = nngX.ngram_id\n",
" \n",
" INNER JOIN\n",
" node_node_ngram AS nngY ON nngY.node_id = n.id\n",
" INNER JOIN\n",
" node_ngram AS ngY ON ngY.id = nngY.ngram_id\n",
"\n",
" WHERE\n",
" n.parent_id = %s\n",
" AND\n",
" nngX.ngram_id <> nngY.ngram_id\n",
" \n", " \n",
" GROUP BY\n", " GROUP BY\n",
" ngX.id,\n", " ngX.id,\n",
...@@ -651,32 +764,39 @@ ...@@ -651,32 +764,39 @@
"outputs": [ "outputs": [
{ {
"output_type": "stream", "output_type": "stream",
"stream": "stdout", "stream": "stderr",
"text": [ "text": [
"(98, 'patients', 'study')\n", "ERROR: An unexpected error occurred while tokenizing input\n",
"(88, 'patients', 'disease')\n", "The following traceback may be corrupted or invalid\n",
"(78, 'voice', 'patients')\n", "The error message is: ('EOF in multi-line string', (1, 0))\n",
"(76, 'Parkinson', 's disease')\n", "\n"
"(64, 'life', 'patients')\n", ]
"(62, 'life', 'quality')\n", },
"(60, 'treatment', 'patients')\n", {
"(56, 'patient', 'patients')\n", "ename": "OperationalError",
"(56, 'voice', 'study')\n", "evalue": "arr\u00eat des connexions suite \u00e0 la demande de l'administrateur\nSSL connection has been closed unexpectedly\n",
"(54, 'Patients', 'patients')\n", "output_type": "pyerr",
"(54, 'purpose', 'study')\n", "traceback": [
"(54, 'voice', 'disease')\n", "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mOperationalError\u001b[0m Traceback (most recent call last)",
"(52, 'study', 'disease')\n", "\u001b[1;32m<ipython-input-11-752593da5735>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m 31\u001b[0m \u001b[0mLIMIT\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 32\u001b[0m \u001b[1;36m20\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 33\u001b[1;33m \"\"\", [corpus.id])\n\u001b[0m\u001b[0;32m 34\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 35\u001b[0m \u001b[1;32mwhile\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"(48, 'voice', 'treatment')\n", "\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/django/db/backends/util.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, sql, params)\u001b[0m\n\u001b[0;32m 67\u001b[0m \u001b[0mstart\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 68\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 69\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCursorDebugWrapper\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msql\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 70\u001b[0m \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 71\u001b[0m \u001b[0mstop\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"(46, 'treatment', 'disease')\n", "\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/django/db/backends/util.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, sql, params)\u001b[0m\n\u001b[0;32m 51\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcursor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msql\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 52\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 53\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcursor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msql\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 54\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 55\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mexecutemany\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msql\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparam_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"(42, 'quality', 'patients')\n", "\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/django/db/utils.py\u001b[0m in \u001b[0;36m__exit__\u001b[1;34m(self, exc_type, exc_value, traceback)\u001b[0m\n\u001b[0;32m 97\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mdj_exc_type\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mDataError\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mIntegrityError\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 98\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrapper\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merrors_occurred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 99\u001b[1;33m \u001b[0msix\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreraise\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdj_exc_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdj_exc_value\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtraceback\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 100\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 101\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"(42, 'life', 'study')\n", "\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/django/utils/six.py\u001b[0m in \u001b[0;36mreraise\u001b[1;34m(tp, value, tb)\u001b[0m\n\u001b[0;32m 547\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mreraise\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtb\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 548\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__traceback__\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mtb\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 549\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 550\u001b[0m \u001b[1;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 551\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
"(40, 'care', 'patients')\n", "\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/django/db/backends/util.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, sql, params)\u001b[0m\n\u001b[0;32m 51\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcursor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msql\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 52\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 53\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcursor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msql\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 54\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 55\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mexecutemany\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msql\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparam_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"(40, 'PD', 'Parkinson')\n", "\u001b[1;31mOperationalError\u001b[0m: arr\u00eat des connexions suite \u00e0 la demande de l'administrateur\nSSL connection has been closed unexpectedly\n"
"(40, 'PD', 's disease')\n"
] ]
} }
], ],
"prompt_number": 108 "prompt_number": 11
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"language": "python",
"metadata": {},
"outputs": []
} }
], ],
"metadata": {} "metadata": {}
......
WorkFlow2.ipynb 0 → 100644
View file @ b358f8e9
This source diff could not be displayed because it is too large. You can view the blob instead.
gargantext_web/api.py
View file @ b358f8e9
...@@ -66,7 +66,7 @@ class CorpusController: ...@@ -66,7 +66,7 @@ class CorpusController:
corpus = corpusQuery.first() corpus = corpusQuery.first()
if corpus.type.name != 'Corpus': if corpus.type.name != 'Corpus':
raise Http404("No such corpus: %d" % (corpus_id, )) raise Http404("No such corpus: %d" % (corpus_id, ))
# if corpus.user != {currentUser}: # if corpus.user != request.user:
# raise Http403("Unauthorized access.") # raise Http403("Unauthorized access.")
return corpus return corpus
...@@ -177,10 +177,13 @@ class CorpusController: ...@@ -177,10 +177,13 @@ class CorpusController:
else: else:
raise ValidationError('Unrecognized "filter[]=%s"' % (filter, )) raise ValidationError('Unrecognized "filter[]=%s"' % (filter, ))
# query building: initializing SQL # query building: initializing SQL
sql_0 = _sql_cte #sql_0 = _sql_cte
sql_0 = ''
sql_1 = '\nSELECT ' sql_1 = '\nSELECT '
sql_2 = '\nFROM %s\nINNER JOIN cte ON cte."id" = %s.id' % (Node._meta.db_table, Node._meta.db_table, ) #sql_2 = '\nFROM %s\nINNER JOIN cte ON cte."id" = %s.id' % (Node._meta.db_table, Node._meta.db_table, )
sql_3 = '\nWHERE ((NOT cte.id = \'%d\') AND (\'%d\' = ANY(cte."path")))' % (corpus.id, corpus.id, ) sql_2 = '\nFROM %s' % (Node._meta.db_table, )
#sql_3 = '\nWHERE ((NOT cte.id = \'%d\') AND (\'%d\' = ANY(cte."path")))' % (corpus.id, corpus.id, )
sql_3 = '\nWHERE (%s.parent_id = %d)' % (Node._meta.db_table, corpus.id, )
# query building: assembling SQL # query building: assembling SQL
sql_1 += ", ".join(columns) sql_1 += ", ".join(columns)
sql_2 += "\nINNER JOIN %s ON %s.id = %s.type_id" % (NodeType._meta.db_table, NodeType._meta.db_table, Node._meta.db_table, ) sql_2 += "\nINNER JOIN %s ON %s.id = %s.type_id" % (NodeType._meta.db_table, NodeType._meta.db_table, Node._meta.db_table, )
......
gargantext_web/settings.py
View file @ b358f8e9
...@@ -63,7 +63,6 @@ INSTALLED_APPS = ( ...@@ -63,7 +63,6 @@ INSTALLED_APPS = (
'django.contrib.staticfiles', 'django.contrib.staticfiles',
'django_extensions', 'django_extensions',
#'south', #'south',
#'documents',
'cte_tree', 'cte_tree',
'node', 'node',
'ngram', 'ngram',
......
gargantext_web/urls.py
View file @ b358f8e9
...@@ -35,13 +35,13 @@ urlpatterns = patterns('', ...@@ -35,13 +35,13 @@ urlpatterns = patterns('',
url(r'^exploration$', views.exploration), url(r'^exploration$', views.exploration),
url(r'^chart/corpus/(\d+)/data.csv$', views.send_csv), url(r'^chart/corpus/(\d+)/data.csv$', views.send_csv),
url(r'^graph.json$', views.send_graph), url(r'^graph.json$', views.json_node_link),
url(r'^api/corpus/(\d+)/ngrams$', gargantext_web.api.CorpusController.ngrams), url(r'^api/corpus/(\d+)/ngrams$', gargantext_web.api.CorpusController.ngrams),
url(r'^api/corpus/(\d+)/metadata$', gargantext_web.api.CorpusController.metadata), url(r'^api/corpus/(\d+)/metadata$', gargantext_web.api.CorpusController.metadata),
url(r'^api/corpus/(\d+)/data$', gargantext_web.api.CorpusController.data), url(r'^api/corpus/(\d+)/data$', gargantext_web.api.CorpusController.data),
# url(r'^graph-it$', gargantext_web.api.CorpusController.data), url(r'^graph-it$', views.graph_it),
) )
from django.conf import settings from django.conf import settings
......
gargantext_web/views.py
View file @ b358f8e9
...@@ -8,7 +8,7 @@ from django.template import Context ...@@ -8,7 +8,7 @@ from django.template import Context
#from documents.models import Project, Corpus, Document #from documents.models import Project, Corpus, Document
from node.models import Language, ResourceType, Resource from node.models import Language, ResourceType, Resource
from node.models import Node, NodeType, Node_Resource, Project, Corpus from node.models import Node, NodeType, Node_Resource, Project, Corpus, NodeNgramNgram
from node.admin import CorpusForm, ProjectForm, ResourceForm from node.admin import CorpusForm, ProjectForm, ResourceForm
from django.contrib.auth.models import User from django.contrib.auth.models import User
...@@ -140,9 +140,11 @@ def project(request, project_id): ...@@ -140,9 +140,11 @@ def project(request, project_id):
user = request.user user = request.user
date = datetime.datetime.now() date = datetime.datetime.now()
type_corpus = NodeType.objects.get(name='Corpus')
project = Node.objects.get(id=project_id) project = Node.objects.get(id=project_id)
corpora = project.children.all() corpora = project.children.filter(type=type_corpus)
number = project.children.count() number = len(corpora)
# DONUT corpora representation # DONUT corpora representation
...@@ -228,7 +230,8 @@ def project(request, project_id): ...@@ -228,7 +230,8 @@ def project(request, project_id):
) )
try: try:
corpus.parse_resources.apply_async((), countdown=1) #corpus.parse_resources.apply_async((), countdown=1)
corpus.parse_resources()
except Exception as error: except Exception as error:
print(error) print(error)
...@@ -413,18 +416,78 @@ def send_csv(request, corpus_id): ...@@ -413,18 +416,78 @@ def send_csv(request, corpus_id):
return response return response
def send_graph(request): def json_node_link(request):
''' '''
Create the HttpResponse object with the graph dataset. Create the HttpResponse object with the graph dataset.
''' '''
response = HttpResponse(content_type='text/json')
response['Content-Disposition'] = 'attachment; filename="graph.json"'
writer = csv.writer(response) import pandas as pd
from copy import copy
import numpy as np
import networkx as nx
from networkx.readwrite import json_graph
from gargantext_web.api import JsonHttpResponse
#from analysis.louvain import *
file = open('/srv/gargantext/tests/graphsam/randomgraphgen.json', 'r') matrix = defaultdict(lambda : defaultdict(float))
for line in file.readlines(): labels = dict()
writer.writerow(line) cooc = Node.objects.get(id=61314)
return response for cooccurrence in NodeNgramNgram.objects.filter(node=cooc):
labels[cooccurrence.ngramx.id] = cooccurrence.ngramx.terms
labels[cooccurrence.ngramy.id] = cooccurrence.ngramy.terms
matrix[cooccurrence.ngramx.id][cooccurrence.ngramy.id] = cooccurrence.score
matrix[cooccurrence.ngramy.id][cooccurrence.ngramx.id] = cooccurrence.score
df = pd.DataFrame(matrix).T.fillna(0)
x = copy(df.values)
x = x / x.sum(axis=1)
# Removing unconnected nodes
threshold = min(x.max(axis=1))
matrix_filtered = np.where(x > threshold, 1, 0)
#matrix_filtered = np.where(x > threshold, x, 0)
G = nx.from_numpy_matrix(matrix_filtered)
G = nx.relabel_nodes(G, dict(enumerate([ labels[x] for x in list(df.columns)])))
#G = nx.relabel_nodes(G, dict(enumerate(df.columns)))
# Removing too connected nodes (find automatic way to do it)
outdeg = G.degree()
to_remove = [n for n in outdeg if outdeg[n] >= 10]
G.remove_nodes_from(to_remove)
for node in G.nodes():
try:
#node,type(labels[node])
G.node[node]['label'] = node
# G.node[node]['color'] = '19,180,300'
except Exception as error:
print(error)
data = json_graph.node_link_data(G)
# data = json_graph.node_link_data(G, attrs={\
# 'source':'source',\
# 'target':'target',\
# 'weight':'weight',\
# #'label':'label',\
# #'color':'color',\
# 'id':'id',})
#print(data)
return JsonHttpResponse(data)
def graph_it(request):
'''The new multimodal graph.'''
t = get_template('graph-it.html')
user = request.user
date = datetime.datetime.now()
html = t.render(Context({
'user': user,
'date': date,
}))
return HttpResponse(html)
init/cte_tree.models.diff 0 → 100644
View file @ b358f8e9
--- models.old.py 2014-11-18 12:12:12.289169335 +0100
+++ models.py 2014-11-18 12:11:31.157171030 +0100
@@ -118,7 +118,7 @@
**kwargs)
def maybe_alias(table):
- if self.query.table_map.has_key(table):
+ if table in self.query.table_map:
return self.query.table_map[table][0]
return table
@@ -153,11 +153,7 @@
name = field[0]
else:
name = field
- if self.query.model._meta.get_field_by_name(name)[
- 0].db_type().startswith('varchar'):
- return 'CAST (T."%s" AS TEXT)' % name
- else:
- return 'T."%s"' % name
+ return 'T."%s"' % name
# The primary key is used in the path; in case it is of a
# custom type, ensure appropriate casting is performed. This
init/graph.pdf 0 → 100644
View file @ b358f8e9
File added
init/graph.sh
View file @ b358f8e9
#Usage : ./manage.ph > graph.pdf #Usage : ./manage.ph > graph.pdf
./manage.py graph_models documents | dot -Tpdf ./manage.py graph_models node | dot -Tpdf
node/admin.py
View file @ b358f8e9
...@@ -2,12 +2,19 @@ from django.contrib import admin ...@@ -2,12 +2,19 @@ from django.contrib import admin
from django.forms import ModelForm, ModelChoiceField from django.forms import ModelForm, ModelChoiceField
from nested_inlines.admin import NestedModelAdmin, NestedStackedInline, NestedTabularInline from nested_inlines.admin import NestedModelAdmin, NestedStackedInline, NestedTabularInline
from node.models import NodeType, Language, Node, Project, Corpus, Document, ResourceType, Resource, Node_Ngram, Node_Resource from node.models import NodeType, Language, Node, \
Project, Corpus, Document, \
ResourceType, Resource, \
Ngram, Node_Ngram, Node_Resource
class ResourceInLine(admin.TabularInline): class ResourceInLine(admin.TabularInline):
model = Resource model = Resource
extra = 0 extra = 0
class NgramAdmin(admin.ModelAdmin):
list_display = ('N', 'Terms')
search_fields = ('Terms',)
class NodeAdmin(admin.ModelAdmin): class NodeAdmin(admin.ModelAdmin):
exclude = ('user', 'path', 'depth', 'numchild', 'ngrams') exclude = ('user', 'path', 'depth', 'numchild', 'ngrams')
list_display = ('name', 'date') list_display = ('name', 'date')
...@@ -137,6 +144,8 @@ admin.site.register(Project, ProjectAdmin) ...@@ -137,6 +144,8 @@ admin.site.register(Project, ProjectAdmin)
admin.site.register(Corpus, CorpusAdmin) admin.site.register(Corpus, CorpusAdmin)
admin.site.register(Document, DocumentAdmin) admin.site.register(Document, DocumentAdmin)
admin.site.register(Node_Resource) admin.site.register(Node_Resource)
admin.site.register(Ngram)
admin.site.register(Node_Ngram) admin.site.register(Node_Ngram)
node/models.py
View file @ b358f8e9
...@@ -46,6 +46,9 @@ class Ngram(models.Model): ...@@ -46,6 +46,9 @@ class Ngram(models.Model):
n = models.IntegerField() n = models.IntegerField()
terms = models.CharField(max_length=255) terms = models.CharField(max_length=255)
nodes = models.ManyToManyField(through='Node_Ngram', to='Node') nodes = models.ManyToManyField(through='Node_Ngram', to='Node')
def __str__(self):
return self.terms
class Resource(models.Model): class Resource(models.Model):
user = models.ForeignKey(User) user = models.ForeignKey(User)
...@@ -207,8 +210,6 @@ class Node_Ngram(models.Model): ...@@ -207,8 +210,6 @@ class Node_Ngram(models.Model):
def __str__(self): def __str__(self):
return "%s: %s" % (self.node.name, self.ngram.terms) return "%s: %s" % (self.node.name, self.ngram.terms)
class Project(Node): class Project(Node):
class Meta: class Meta:
proxy=True proxy=True
......
templates/explorer.html
View file @ b358f8e9
...@@ -21,16 +21,6 @@ ...@@ -21,16 +21,6 @@
{% block content %} {% block content %}
<link rel="stylesheet" href="{% static "js/libs/css2/freshslider.css" %} media="screen">
<link rel="stylesheet" href="{% static "js/libs/css2/custom.css" %} media="screen">
<link rel="stylesheet" href="{% static "js/libs/css2/sidebar.css" %} media="screen">
<!-- this is the tweakbar --> <!-- this is the tweakbar -->
<div id="defaultop" class="navbar navbar-default"> <div id="defaultop" class="navbar navbar-default">
<div class="navbar-collapse collapse navbar-responsive-collapse"> <div class="navbar-collapse collapse navbar-responsive-collapse">
...@@ -68,6 +58,12 @@ ...@@ -68,6 +58,12 @@
</a> </a>
</li> </li>
<li>
<a>
<div id="graphid" style="visibility: hidden;">/graph.json</div>
</a>
</li>
<!-- <!--
<li> <li>
<a> <a>
...@@ -123,6 +119,8 @@ ...@@ -123,6 +119,8 @@
</a></li> </a></li>
</ul> </ul>
<!--
<ul id="colorGraph" class="nav navbar-nav navbar-right"> <ul id="colorGraph" class="nav navbar-nav navbar-right">
<li class="dropdown"> <li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown"> <a href="#" class="dropdown-toggle" data-toggle="dropdown">
...@@ -137,6 +135,7 @@ ...@@ -137,6 +135,7 @@
</ul> </ul>
</li> </li>
</ul> </ul>
-->
...@@ -173,7 +172,9 @@ ...@@ -173,7 +172,9 @@
<div style="visibility: hidden;" id="sigma-othergraph"></div> <div style="visibility: hidden;" id="sigma-othergraph"></div>
<img id="semLoader" style="position:absolute; top:50%; left:40%; width:80px;" src="{% static "js/libs/img2/loading-bar.gif" %}" ></img> <div id="semLoader" style="position:absolute; top:50%; left:40%; width:80px; visibility: hidden;">
<img src="{% static "js/libs/img2/loading-bar.gif" %}"></img>
</div>
<ul id="ctlzoom"> <ul id="ctlzoom">
...@@ -185,12 +186,13 @@ ...@@ -185,12 +186,13 @@
<a href="#" id="aUnfold" class="rightarrow"></a> <a href="#" id="aUnfold" class="rightarrow"></a>
</div> </div>
</li> </li>
<!--
<li> <li>
<a href="#" id="geomapicon" onclick="$('#geomapmodal').modal('show'); callGeomap();"> <a href="#" id="geomapicon" onclick="$('#geomapmodal').modal('show'); callGeomap();">
<img title="World Map Distribution" width="34px" src="{% static "js/libs/img2/world.png" %}" ></img> <img title="World Map Distribution" width="34px" src="{% static "js/libs/img2/world.png" %}" ></img>
</a> </a>
</li> </li>
-->
<li> <li>
...@@ -232,7 +234,27 @@ ...@@ -232,7 +234,27 @@
<div id="leftcolumn"> <div id="leftcolumn">
<div id="tips"></div> <div id="tips"></div>
<div id="names"></div> <div id="names"></div>
<br>
<div id="tab-container" class='tab-container' style="display: none;">
<ul class='etabs'>
<li id="taboppos" class='tab'><a href="#tabs1">Opposite-Neighbors</a></li>
<li id="tabneigh" class='tab'><a href="#tabs2">Neighbors</a></li>
</ul>
<div class='panel-container'>
<div id="tabs1">
<div id="opossiteNodes"></div> <div id="opossiteNodes"></div>
</div>
<div id="tabs2">
<div id="sameNodes"></div>
</div>
</div>
</div>
<!-- <div id="topPapers"></div> --> <!-- <div id="topPapers"></div> -->
<div id="information"></div> <div id="information"></div>
</div> </div>
...@@ -331,13 +353,14 @@ ...@@ -331,13 +353,14 @@
<script src="{% static "js/libs/jquery/jquery-1.10.2.js" %}" type="text/javascript"></script> <script src="{% static "js/libs/jquery/jquery-1.10.2.js" %}" type="text/javascript"></script>
<script src="{% static "js/libs/jquery/jquery-ui.js" %}" type="text/javascript"></script> <script src="{% static "js/libs/jquery/jquery-ui.js" %}" type="text/javascript"></script>
<script src="{% static "js/libs/jquery/jquery.ba-dotimeout.min.js" %}" type="text/javascript"></script> <script src="{% static "js/libs/jquery/jquery.ba-dotimeout.min.js" %}" type="text/javascript"></script>
<script src="{% static "js/libs/jquery/jquery.mousewheel.min.js" %}" type="text/javascript"></script> <script src="{% static "js/libs/jquery/jquery.mousewheel.min.js" %}" type="text/javascript"></script>
<script type="text/javascript" src="{% static "js/libs/freshslider.1.0.js" %}"></script> <script type="text/javascript" src="{% static "js/libs/freshslider.1.0.js" %}"></script>
<script type="text/javascript" src="{% static "js/libs/readmore.js" %}"></script>
<script type="text/javascript" src="{% static "js/libs/jquery/jquery.easytabs.min.js" %}"></script>
<script src="{% static "js/libs/bootstrap/js/bootstrap.min.js" %}"></script> <script src="{% static "js/libs/bootstrap/js/bootstrap.min.js" %}"></script>
<script src="{% static "js/libs/bootstrap/js/bootstrap-modal.js" %}" type="text/javascript"></script> <script src="{% static "js/libs/bootstrap/js/bootstrap-modal.js" %}" type="text/javascript"></script>
...@@ -346,7 +369,6 @@ ...@@ -346,7 +369,6 @@
<script src="{% static "js/tinawebJS/globalUtils.js" %}" type="text/javascript"></script> <script src="{% static "js/tinawebJS/globalUtils.js" %}" type="text/javascript"></script>
<script src="{% static "js/tinawebJS/sigma.min.js" %}" type="text/javascript" language="javascript"></script> <script src="{% static "js/tinawebJS/sigma.min.js" %}" type="text/javascript" language="javascript"></script>
<script src="{% static "js/tinawebJS/sigma.forceatlas2.js" %}" type="text/javascript" language="javascript"></script> <script src="{% static "js/tinawebJS/sigma.forceatlas2.js" %}" type="text/javascript" language="javascript"></script>
<script src="{% static "js/settings_explorerjs.js" %}" type="text/javascript" language="javascript"></script> <script src="{% static "js/settings_explorerjs.js" %}" type="text/javascript" language="javascript"></script>
...@@ -360,5 +382,14 @@ ...@@ -360,5 +382,14 @@
<script src="{% static "js/tinawebJS/DataScanner.js" %}" type="text/javascript" language="javascript"></script> <script src="{% static "js/tinawebJS/DataScanner.js" %}" type="text/javascript" language="javascript"></script>
<script src="{% static "js/tinawebJS/main.js" %}" type="text/javascript" language="javascript"></script> <script src="{% static "js/tinawebJS/main.js" %}" type="text/javascript" language="javascript"></script>
<script type="text/javascript">
$('#tab-container').easytabs({updateHash:false});
// Popup window code
function newPopup(url) {
popupWindow = window.open(
url,'popUpWindow','height=700,width=800,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=no,menubar=no,location=no,directories=no,status=no')
}
</script>
{% endblock %} {% endblock %}
templates/menu.html
View file @ b358f8e9
...@@ -60,7 +60,8 @@ ...@@ -60,7 +60,8 @@
{% load staticfiles %} {% load staticfiles %}
<script src="{% static "js/jquery/jquery-1.9.1.min.js"%}"></script> <!-- <script src="{% static "js/jquery/jquery-1.9.1.min.js"%}"></script> -->
<script src="{% static "js/easy/jquery.easypiechart.js"%}"></script> <script src="{% static "js/easy/jquery.easypiechart.js"%}"></script>
<script type="text/javascript"> <script type="text/javascript">
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment