Skip to content

gargantext
gargantext
Commit 4838d9b9 authored by Administrator's avatar Administrator
Browse files
Options

Merge branch 'alex' 

Intégration des mes modifs locales
parents d4bb2189 30edf470
Hide whitespace changes
Inline Side-by-side
Showing with 1978 additions and 86 deletions
SQL_TESTS.ipynb
View file @ 4838d9b9
{ {
"metadata": { "metadata": {
"name": "", "name": "",
"signature": "sha256:33c2f41e3ea5983e768350b4012544242c5df9b394091647362f00929812a921" "signature": "sha256:65f487ee62067486e4f832ed088fe02ede3daa27052c9dcaf58b3edffa169245"
}, },
"nbformat": 3, "nbformat": 3,
"nbformat_minor": 0, "nbformat_minor": 0,
...@@ -27,6 +27,17 @@ ...@@ -27,6 +27,17 @@
"outputs": [], "outputs": [],
"prompt_number": 1 "prompt_number": 1
}, },
{
"cell_type": "code",
"collapsed": false,
"input": [
"me = User.objects.get(username='alexandre')"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 2
},
{ {
"cell_type": "code", "cell_type": "code",
"collapsed": false, "collapsed": false,
...@@ -37,7 +48,7 @@ ...@@ -37,7 +48,7 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 15 "prompt_number": 2
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -48,7 +59,7 @@ ...@@ -48,7 +59,7 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 1 "prompt_number": 3
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -227,12 +238,12 @@ ...@@ -227,12 +238,12 @@
"collapsed": false, "collapsed": false,
"input": [ "input": [
"#\u00a0corpus = Node.objects.filter(type=typeCorpus).first()\n", "#\u00a0corpus = Node.objects.filter(type=typeCorpus).first()\n",
"corpus = Node.objects.get(id=44338)" "corpus = Node.objects.get(id=13064)"
], ],
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 4 "prompt_number": 3
}, },
{ {
"cell_type": "heading", "cell_type": "heading",
...@@ -279,30 +290,30 @@ ...@@ -279,30 +290,30 @@
"output_type": "stream", "output_type": "stream",
"stream": "stdout", "stream": "stdout",
"text": [ "text": [
"'2004/01/02','2'\n", "'1954/11/18','2'\n",
"'2004/01/03','1'\n", "'1958/11/18','1'\n",
"'2004/01/06','8'\n", "'1959/11/18','1'\n",
"'2004/01/07','5'\n", "'1968/11/18','1'\n",
"'2004/01/08','8'\n", "'1969/11/18','2'\n",
"'2004/01/09','1'\n", "'1971/11/18','4'\n",
"'2004/01/10','3'\n", "'1972/11/18','1'\n",
"'2004/01/12','2'\n", "'1974/11/18','2'\n",
"'2004/01/13','6'\n", "'1975/11/18','3'\n",
"'2004/01/15','2'\n", "'1976/11/18','1'\n",
"'2004/01/16','1'\n", "'1977/11/18','6'\n",
"'2004/01/17','5'\n", "'1978/11/18','11'\n",
"'2004/01/19','2'\n", "'1979/11/18','9'\n",
"'2004/01/20','2'\n", "'1980/11/18','6'\n",
"'2004/01/21','7'\n", "'1981/11/18','4'\n",
"'2004/01/23','1'\n", "'1982/11/18','7'\n",
"'2004/01/24','4'\n", "'1983/11/18','14'\n",
"'2004/01/25','4'\n", "'1984/11/18','17'\n",
"'2004/01/26','5'\n", "'1985/11/18','18'\n",
"'2004/01/27','2'\n" "'1986/02/21','1'\n"
] ]
} }
], ],
"prompt_number": 37 "prompt_number": 6
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -320,12 +331,17 @@ ...@@ -320,12 +331,17 @@
" node_ngram AS ngX ON ngX.id = nngX.ngram_id\n", " node_ngram AS ngX ON ngX.id = nngX.ngram_id\n",
" WHERE\n", " WHERE\n",
" n.parent_id = %s\n", " n.parent_id = %s\n",
" AND\n",
" ngX.n >= 2\n",
" GROUP BY\n", " GROUP BY\n",
" ngX.terms\n", " ngX.terms\n",
" Having\n",
" COUNT(*) > 7\n",
" ORDER BY\n", " ORDER BY\n",
" occurrences DESC\n", " occurrences DESC\n",
" LIMIT\n", " LIMIT\n",
" 20\n", " 100\n",
" \n",
"\"\"\", [corpus.id])\n", "\"\"\", [corpus.id])\n",
"\n", "\n",
"while True:\n", "while True:\n",
...@@ -341,30 +357,74 @@ ...@@ -341,30 +357,74 @@
"output_type": "stream", "output_type": "stream",
"stream": "stdout", "stream": "stdout",
"text": [ "text": [
"(196, 'patients')\n", "(138, 'honey bees')\n",
"(135, 'voice')\n", "(132, 'apis mellifera')\n",
"(129, 'study')\n", "(69, 'honey bee')\n",
"(111, 'disease')\n", "(66, 'apis mellifera l')\n",
"(69, 'treatment')\n", "(45, 'pesticide residues')\n",
"(66, 'life')\n", "(39, 'gas chromatography')\n",
"(58, 'patient')\n", "(36, 'varroa destructor')\n",
"(53, 'quality')\n", "(36, 'honey bee colonies')\n",
"(49, 'care')\n", "(30, 'sublethal effects')\n",
"(45, 'use')\n", "(27, 'apidae )')\n",
"(44, 'Patients')\n", "(21, 'neonicotinoid insecticides')\n",
"(43, 'people')\n", "(21, 'honey bee ( hymenoptera')\n",
"(41, 'development')\n", "(18, 'bee products')\n",
"(41, 'purpose')\n", "(18, 'megachile rotundata')\n",
"(40, 's disease')\n", "(18, 'solid-phase extraction')\n",
"(39, 's')\n", "(18, 'simultaneous determination')\n",
"(38, 'results')\n", "(18, 'mass spectrometric')\n",
"(37, 'diagnosis')\n", "(15, 'case study')\n",
"(36, 'years')\n", "(15, 'honey samples')\n",
"(34, 'women')\n" "(15, 'liquid chromatography')\n",
"(15, 'high performance liquid chromatography')\n",
"(15, 'varroa mites')\n",
"(12, 'organochlorine pesticides')\n",
"(12, 'gas chromatography-mass spectrometry')\n",
"(12, 'liquid chromatography-mass spectrometry')\n",
"(12, 'colony health')\n",
"(12, 'gas chromatographic')\n",
"(12, 'colony collapse disorder')\n",
"(12, 'bumble bees')\n",
"(12, 'varroa jacobsoni')\n",
"(9, 'chemiluminescent elisa')\n",
"(9, 'diversionary plantings for reduction of pesticide related bee mortality')\n",
"(9, 'pesticides and law')\n",
"(9, 'plant protection products')\n",
"(9, 'nomia melanderi')\n",
"(9, 'electron-capture detection')\n",
"(9, 'managed pollinator cap coordinated agricultural project a national research')\n",
"(9, 'apis florea f')\n",
"(9, 'solid-phase microextraction')\n",
"(9, 'extension initiative')\n",
"(9, 'crop pollination')\n",
"(9, 'non-apis bees')\n",
"(9, 'honey bees ( apis mellifera')\n",
"(9, 'liquid chromatography-tandem mass spectrometry')\n",
"(9, 'bee pollen')\n",
"(9, 'foraging behavior')\n",
"(9, 'biological control')\n",
"(9, 'nosema ceranae')\n",
"(9, 'organophosphorus pesticides')\n",
"(9, 'field conditions')\n",
"(9, 'honey bee apis mellifera l')\n",
"(9, 'laboratory tests')\n",
"(9, 'beauveria bassiana')\n",
"(9, 'comparative toxicity')\n",
"(9, 'high levels')\n",
"(9, 'pesticide exposure')\n",
"(9, 'fluvalinate residues')\n",
"(9, 'insecticide residues')\n",
"(9, 'osmia lignaria')\n",
"(9, 'bombus impatiens')\n",
"(9, 'honey bee health')\n",
"(9, 'agricultural landscape')\n",
"(9, 'dispersive liquid-liquid microextraction')\n",
"(9, 'matrix solid-phase dispersion')\n"
] ]
} }
], ],
"prompt_number": 104 "prompt_number": 28
}, },
{ {
"cell_type": "heading", "cell_type": "heading",
...@@ -455,7 +515,7 @@ ...@@ -455,7 +515,7 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 21 "prompt_number": 17
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -470,7 +530,7 @@ ...@@ -470,7 +530,7 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 22 "prompt_number": 18
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -485,7 +545,7 @@ ...@@ -485,7 +545,7 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 23 "prompt_number": 19
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -497,7 +557,7 @@ ...@@ -497,7 +557,7 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 24 "prompt_number": 22
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -514,7 +574,7 @@ ...@@ -514,7 +574,7 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 25 "prompt_number": 23
}, },
{ {
"cell_type": "code", "cell_type": "code",
...@@ -528,13 +588,13 @@ ...@@ -528,13 +588,13 @@
{ {
"metadata": {}, "metadata": {},
"output_type": "pyout", "output_type": "pyout",
"prompt_number": 26, "prompt_number": 24,
"text": [ "text": [
"6" "61297"
] ]
} }
], ],
"prompt_number": 26 "prompt_number": 24
}, },
{ {
"cell_type": "heading", "cell_type": "heading",
...@@ -592,7 +652,7 @@ ...@@ -592,7 +652,7 @@
"language": "python", "language": "python",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"prompt_number": 47 "prompt_number": 26
}, },
{ {
"cell_type": "heading", "cell_type": "heading",
...@@ -627,7 +687,60 @@ ...@@ -627,7 +687,60 @@
" WHERE\n", " WHERE\n",
" n.parent_id = %s\n", " n.parent_id = %s\n",
" AND\n", " AND\n",
" nngX.ngram_id > nngY.ngram_id\n", " nngX.ngram_id in (select id from node_node_ngram WHERE node_id = 61298 )\n",
" AND\n",
" nngY.ngram_id in (select id from node_node_ngram WHERE node_id = 61298 )\n",
" AND\n",
" nngX.ngram_id <> nngY.ngram_id\n",
" \n",
" GROUP BY\n",
" ngX.id,\n",
" ngX.terms,\n",
" ngY.id,\n",
" ngY.terms\n",
" ORDER BY\n",
" cooccurrences DESC\n",
" LIMIT\n",
" 200\n",
"\"\"\", [corpus.id])\n",
"\n",
"while True:\n",
" row = cursor.fetchone()\n",
" if row is None:\n",
" break\n",
" print(row)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 15
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"cursor.execute(\"\"\"\n",
" SELECT\n",
" COUNT(*) AS cooccurrences,\n",
" ngX.terms,\n",
" ngY.terms\n",
" FROM\n",
" node_node AS n\n",
" \n",
" INNER JOIN\n",
" node_node_ngram AS nngX ON nngX.node_id = n.id\n",
" INNER JOIN\n",
" node_ngram AS ngX ON ngX.id = nngX.ngram_id\n",
" \n",
" INNER JOIN\n",
" node_node_ngram AS nngY ON nngY.node_id = n.id\n",
" INNER JOIN\n",
" node_ngram AS ngY ON ngY.id = nngY.ngram_id\n",
"\n",
" WHERE\n",
" n.parent_id = %s\n",
" AND\n",
" nngX.ngram_id <> nngY.ngram_id\n",
" \n", " \n",
" GROUP BY\n", " GROUP BY\n",
" ngX.id,\n", " ngX.id,\n",
...@@ -651,32 +764,39 @@ ...@@ -651,32 +764,39 @@
"outputs": [ "outputs": [
{ {
"output_type": "stream", "output_type": "stream",
"stream": "stdout", "stream": "stderr",
"text": [ "text": [
"(98, 'patients', 'study')\n", "ERROR: An unexpected error occurred while tokenizing input\n",
"(88, 'patients', 'disease')\n", "The following traceback may be corrupted or invalid\n",
"(78, 'voice', 'patients')\n", "The error message is: ('EOF in multi-line string', (1, 0))\n",
"(76, 'Parkinson', 's disease')\n", "\n"
"(64, 'life', 'patients')\n", ]
"(62, 'life', 'quality')\n", },
"(60, 'treatment', 'patients')\n", {
"(56, 'patient', 'patients')\n", "ename": "OperationalError",
"(56, 'voice', 'study')\n", "evalue": "arr\u00eat des connexions suite \u00e0 la demande de l'administrateur\nSSL connection has been closed unexpectedly\n",
"(54, 'Patients', 'patients')\n", "output_type": "pyerr",
"(54, 'purpose', 'study')\n", "traceback": [
"(54, 'voice', 'disease')\n", "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mOperationalError\u001b[0m Traceback (most recent call last)",
"(52, 'study', 'disease')\n", "\u001b[1;32m<ipython-input-11-752593da5735>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m 31\u001b[0m \u001b[0mLIMIT\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 32\u001b[0m \u001b[1;36m20\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 33\u001b[1;33m \"\"\", [corpus.id])\n\u001b[0m\u001b[0;32m 34\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 35\u001b[0m \u001b[1;32mwhile\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"(48, 'voice', 'treatment')\n", "\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/django/db/backends/util.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, sql, params)\u001b[0m\n\u001b[0;32m 67\u001b[0m \u001b[0mstart\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 68\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 69\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCursorDebugWrapper\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msql\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 70\u001b[0m \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 71\u001b[0m \u001b[0mstop\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"(46, 'treatment', 'disease')\n", "\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/django/db/backends/util.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, sql, params)\u001b[0m\n\u001b[0;32m 51\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcursor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msql\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 52\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 53\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcursor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msql\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 54\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 55\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mexecutemany\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msql\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparam_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"(42, 'quality', 'patients')\n", "\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/django/db/utils.py\u001b[0m in \u001b[0;36m__exit__\u001b[1;34m(self, exc_type, exc_value, traceback)\u001b[0m\n\u001b[0;32m 97\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mdj_exc_type\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mDataError\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mIntegrityError\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 98\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrapper\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merrors_occurred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 99\u001b[1;33m \u001b[0msix\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreraise\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdj_exc_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdj_exc_value\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtraceback\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 100\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 101\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"(42, 'life', 'study')\n", "\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/django/utils/six.py\u001b[0m in \u001b[0;36mreraise\u001b[1;34m(tp, value, tb)\u001b[0m\n\u001b[0;32m 547\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mreraise\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtb\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 548\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__traceback__\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mtb\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 549\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 550\u001b[0m \u001b[1;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 551\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
"(40, 'care', 'patients')\n", "\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/django/db/backends/util.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, sql, params)\u001b[0m\n\u001b[0;32m 51\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcursor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msql\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 52\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 53\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcursor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msql\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 54\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 55\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mexecutemany\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msql\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparam_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"(40, 'PD', 'Parkinson')\n", "\u001b[1;31mOperationalError\u001b[0m: arr\u00eat des connexions suite \u00e0 la demande de l'administrateur\nSSL connection has been closed unexpectedly\n"
"(40, 'PD', 's disease')\n"
] ]
} }
], ],
"prompt_number": 108 "prompt_number": 11
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"language": "python",
"metadata": {},
"outputs": []
} }
], ],
"metadata": {} "metadata": {}
......
WorkFlow2.ipynb 0 → 100644
View file @ 4838d9b9
This source diff could not be displayed because it is too large. You can view the blob instead.
gargantext_web/settings.py
View file @ 4838d9b9
...@@ -63,7 +63,6 @@ INSTALLED_APPS = ( ...@@ -63,7 +63,6 @@ INSTALLED_APPS = (
'django.contrib.staticfiles', 'django.contrib.staticfiles',
'django_extensions', 'django_extensions',
#'south', #'south',
#'documents',
'cte_tree', 'cte_tree',
'node', 'node',
'ngram', 'ngram',
......
gargantext_web/views.py
View file @ 4838d9b9
...@@ -228,7 +228,8 @@ def project(request, project_id): ...@@ -228,7 +228,8 @@ def project(request, project_id):
) )
try: try:
corpus.parse_resources.apply_async((), countdown=1) #corpus.parse_resources.apply_async((), countdown=1)
corpus.parse_resources()
except Exception as error: except Exception as error:
print(error) print(error)
......
init/graph.pdf 0 → 100644
View file @ 4838d9b9
File added
init/graph.sh
View file @ 4838d9b9
#Usage : ./manage.ph > graph.pdf #Usage : ./manage.ph > graph.pdf
./manage.py graph_models documents | dot -Tpdf ./manage.py graph_models node | dot -Tpdf
node/admin.py
View file @ 4838d9b9
...@@ -2,12 +2,19 @@ from django.contrib import admin ...@@ -2,12 +2,19 @@ from django.contrib import admin
from django.forms import ModelForm, ModelChoiceField from django.forms import ModelForm, ModelChoiceField
from nested_inlines.admin import NestedModelAdmin, NestedStackedInline, NestedTabularInline from nested_inlines.admin import NestedModelAdmin, NestedStackedInline, NestedTabularInline
from node.models import NodeType, Language, Node, Project, Corpus, Document, ResourceType, Resource, Node_Ngram, Node_Resource from node.models import NodeType, Language, Node, \
Project, Corpus, Document, \
ResourceType, Resource, \
Ngram, Node_Ngram, Node_Resource
class ResourceInLine(admin.TabularInline): class ResourceInLine(admin.TabularInline):
model = Resource model = Resource
extra = 0 extra = 0
class NgramAdmin(admin.ModelAdmin):
list_display = ('N', 'Terms')
search_fields = ('Terms',)
class NodeAdmin(admin.ModelAdmin): class NodeAdmin(admin.ModelAdmin):
exclude = ('user', 'path', 'depth', 'numchild', 'ngrams') exclude = ('user', 'path', 'depth', 'numchild', 'ngrams')
list_display = ('name', 'date') list_display = ('name', 'date')
...@@ -137,6 +144,8 @@ admin.site.register(Project, ProjectAdmin) ...@@ -137,6 +144,8 @@ admin.site.register(Project, ProjectAdmin)
admin.site.register(Corpus, CorpusAdmin) admin.site.register(Corpus, CorpusAdmin)
admin.site.register(Document, DocumentAdmin) admin.site.register(Document, DocumentAdmin)
admin.site.register(Node_Resource) admin.site.register(Node_Resource)
admin.site.register(Ngram)
admin.site.register(Node_Ngram) admin.site.register(Node_Ngram)
node/models.py
View file @ 4838d9b9
...@@ -46,6 +46,9 @@ class Ngram(models.Model): ...@@ -46,6 +46,9 @@ class Ngram(models.Model):
n = models.IntegerField() n = models.IntegerField()
terms = models.CharField(max_length=255) terms = models.CharField(max_length=255)
nodes = models.ManyToManyField(through='Node_Ngram', to='Node') nodes = models.ManyToManyField(through='Node_Ngram', to='Node')
def __str__(self):
return self.terms
class Resource(models.Model): class Resource(models.Model):
user = models.ForeignKey(User) user = models.ForeignKey(User)
...@@ -207,8 +210,6 @@ class Node_Ngram(models.Model): ...@@ -207,8 +210,6 @@ class Node_Ngram(models.Model):
def __str__(self): def __str__(self):
return "%s: %s" % (self.node.name, self.ngram.terms) return "%s: %s" % (self.node.name, self.ngram.terms)
class Project(Node): class Project(Node):
class Meta: class Meta:
proxy=True proxy=True
......
tests/cooc_whitelist.sql 0 → 100644
View file @ 4838d9b9
SELECT
COUNT(*) AS cooccurrences,
ngX.terms,
ngY.terms
FROM
node_node AS n -- the nodes who are direct children of the corpus
INNER JOIN
node_node_ngram AS nngX ON nngX.node_id = n.id -- list of ngrams contained in the node
INNER JOIN
node_node_ngram AS whitelistX ON whitelistX.ngram_id = nngX.ngram_id -- list of ngrams contained in the whitelist and in the node
INNER JOIN
node_ngram AS ngX ON ngX.id = whitelistX.ngram_id -- ngrams which are in both
INNER JOIN
node_node_ngram AS nngY ON nngY.node_id = n.id
INNER JOIN
node_node_ngram AS whitelistY ON whitelistY.ngram_id = nngY.ngram_id
INNER JOIN
node_ngram AS ngY ON ngY.id = whitelistY.ngram_id
WHERE
n.parent_id = %s
AND
whitelistX.node_id = %s
AND
whitelistY.node_id = %s
AND
nngX.ngram_id < nngY.ngram_id -- so we only get distinct pairs of ngrams
GROUP BY
ngX.id,
ngX.terms,
ngY.id,
ngY.terms
ORDER BY
cooccurrences DESC
LIMIT
200
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment