Skip to content

gargantext
gargantext
Commit e558ccba authored by Administrator's avatar Administrator
Browse files
Options

[FEATURE] SQL > GRAPH on view Corpus.

parent 87ec7c16
Hide whitespace changes
Inline Side-by-side
Showing with 94 additions and 96 deletions
AlexandreTests.ipynb
View file @ e558ccba
{
"metadata": {
"name": "",
"signature": "sha256:2993a87a967d66a7ddb13adeee8f61edcc427c30048a4b1f235ebdc95f0713cf"
"signature": "sha256:6d467c42666e98d7f142756ddaee6715f400591ac3004e45c17d204c2f2d0b58"
},
"nbformat": 3,
"nbformat_minor": 0,
......@@ -22,72 +22,6 @@
"outputs": [],
"prompt_number": 27
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"corpus = Node.objects.get(name='tst')"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 18
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"corpus.parse_resources()"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 19
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"c = corpus.children.all()[1]"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 21
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"c.parse_resources()"
],
"language": "python",
"metadata": {},
"outputs": [
{
"ename": "OSError",
"evalue": "Error reading file 'corpora/pubmed_trawoMk.zip': failed to load external entity \"corpora/pubmed_trawoMk.zip\"",
"output_type": "pyerr",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mOSError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m<ipython-input-22-397b854d45cf>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mc\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparse_resources\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[1;32m/srv/gargantext/node/models.py\u001b[0m in \u001b[0;36mparse_resources\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 113\u001b[0m \u001b[1;34m'ris'\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0mRisFileParser\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 114\u001b[0m \u001b[1;34m'europress'\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0mEuropressFileParser\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 115\u001b[1;33m })[resource.type.name]()\n\u001b[0m\u001b[0;32m 116\u001b[0m \u001b[0mmetadata_list\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mparser\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparse\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresource\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfile\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 117\u001b[0m \u001b[1;31m# insert the new resources in the database!\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m/srv/gargantext/parsing/FileParsers/FileParser.py\u001b[0m in \u001b[0;36mparse\u001b[1;34m(self, file)\u001b[0m\n\u001b[0;32m 105\u001b[0m \u001b[1;31m# ...otherwise, let's parse it directly!\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 106\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 107\u001b[1;33m \u001b[0mmetadata_list\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_parse\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 108\u001b[0m \u001b[1;31m# return the list of formatted metadata\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 109\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mmap\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat_metadata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmetadata_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m/srv/gargantext/parsing/FileParsers/PubmedFileParser.py\u001b[0m in \u001b[0;36m_parse\u001b[1;34m(self, file)\u001b[0m\n\u001b[0;32m 9\u001b[0m \u001b[1;31m# open the file as XML\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 10\u001b[0m \u001b[0mxml_parser\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0metree\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mXMLParser\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresolve_entities\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrecover\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m \u001b[0mxml\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0metree\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparse\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparser\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mxml_parser\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 12\u001b[0m \u001b[0mxml_articles\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mxml\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfindall\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'PubmedArticle'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 13\u001b[0m \u001b[1;31m# initialize the list of metadata\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/lxml/etree.cpython-34m.so\u001b[0m in \u001b[0;36mlxml.etree.parse (src/lxml/lxml.etree.c:70141)\u001b[1;34m()\u001b[0m\n",
"\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/lxml/etree.cpython-34m.so\u001b[0m in \u001b[0;36mlxml.etree._parseDocument (src/lxml/lxml.etree.c:102252)\u001b[1;34m()\u001b[0m\n",
"\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/lxml/etree.cpython-34m.so\u001b[0m in \u001b[0;36mlxml.etree._parseDocumentFromURL (src/lxml/lxml.etree.c:102516)\u001b[1;34m()\u001b[0m\n",
"\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/lxml/etree.cpython-34m.so\u001b[0m in \u001b[0;36mlxml.etree._parseDocFromFile (src/lxml/lxml.etree.c:101551)\u001b[1;34m()\u001b[0m\n",
"\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/lxml/etree.cpython-34m.so\u001b[0m in \u001b[0;36mlxml.etree._BaseParser._parseDocFromFile (src/lxml/lxml.etree.c:97003)\u001b[1;34m()\u001b[0m\n",
"\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/lxml/etree.cpython-34m.so\u001b[0m in \u001b[0;36mlxml.etree._ParserContext._handleParseResultDoc (src/lxml/lxml.etree.c:91461)\u001b[1;34m()\u001b[0m\n",
"\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/lxml/etree.cpython-34m.so\u001b[0m in \u001b[0;36mlxml.etree._handleParseResult (src/lxml/lxml.etree.c:92647)\u001b[1;34m()\u001b[0m\n",
"\u001b[1;32m/home/alexandre/projets/gargantext.py/env/lib/python3.4/site-packages/lxml/etree.cpython-34m.so\u001b[0m in \u001b[0;36mlxml.etree._raiseParseError (src/lxml/lxml.etree.c:91908)\u001b[1;34m()\u001b[0m\n",
"\u001b[1;31mOSError\u001b[0m: Error reading file 'corpora/pubmed_trawoMk.zip': failed to load external entity \"corpora/pubmed_trawoMk.zip\""
]
}
],
"prompt_number": 22
},
{
"cell_type": "code",
"collapsed": false,
......
SQL_TESTS.ipynb
View file @ e558ccba
{
"metadata": {
"name": "",
"signature": "sha256:a5bd8dfc6ee8fc121f5aec3f45e591fc715917cc12bb133015d3f91f7337cbc7"
"signature": "sha256:9cffba735acaf6f9511e0a0a5e08831945944a5fd5f5da3122be9ab3b0e03327"
},
"nbformat": 3,
"nbformat_minor": 0,
......@@ -25,7 +25,7 @@
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 30
"prompt_number": 1
},
{
"cell_type": "code",
......@@ -68,12 +68,12 @@
"collapsed": false,
"input": [
"#\u00a0corpus = Node.objects.filter(type=typeCorpus).first()\n",
"corpus = Node.objects.get(id=698)"
"corpus = Node.objects.get(id=44338)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 59
"prompt_number": 4
},
{
"cell_type": "heading",
......@@ -83,6 +83,65 @@
"Occurences"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"cursor.execute(\"\"\"\n",
" SELECT\n",
" metadata -> 'publication_day' as day,\n",
" metadata -> 'publication_month' as month, \n",
" metadata -> 'publication_year' as year, \n",
" COUNT(*)\n",
" FROM\n",
" node_node AS n\n",
" WHERE\n",
" n.parent_id = %s\n",
" GROUP BY\n",
" day, month, year\n",
" LIMIT\n",
" 20\n",
"\"\"\", [corpus.id])\n",
"\n",
"\n",
"while True:\n",
" row = cursor.fetchone()\n",
" if row is None:\n",
" break\n",
" print(row)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"('08', '01', '2004', 8)\n",
"('03', '01', '2004', 1)\n",
"('21', '01', '2004', 7)\n",
"('09', '01', '2004', 1)\n",
"('06', '01', '2004', 8)\n",
"('07', '01', '2004', 5)\n",
"('23', '01', '2004', 1)\n",
"('19', '01', '2004', 2)\n",
"('01', '02', '2004', 3)\n",
"('16', '01', '2004', 1)\n",
"('02', '01', '2004', 2)\n",
"('31', '01', '2004', 2)\n",
"('20', '01', '2004', 2)\n",
"('25', '01', '2004', 4)\n",
"('29', '01', '2004', 2)\n",
"('24', '01', '2004', 4)\n",
"('17', '01', '2004', 5)\n",
"('26', '01', '2004', 5)\n",
"('27', '01', '2004', 2)\n",
"('28', '01', '2004', 5)\n"
]
}
],
"prompt_number": 11
},
{
"cell_type": "code",
"collapsed": false,
......
gargantext_web/views.py
View file @ e558ccba
......@@ -427,31 +427,36 @@ def send_csv(request):
# file = open('/srv/gargantext/static/js/d3/ndx.csv', 'r')
# for line in file.readlines():
# writer.writerow(line)
writer.writerow(['date','open','high','low','close','volume','oi'])
writer.writerow(['12/19/2001','96.05','99.98','95.79','99.98','1260','0'])
writer.writerow(['12/20/2001','104.3','104.39','99.98','104.39','197','0'])
writer.writerow(['12/21/2001','109.07','109.13','103.73','109.13','28','0'])
writer.writerow(['12/24/2001','113.57','114.55','109.13','114.55','32','0'])
writer.writerow(['12/25/2001','120.09','120.25','114.55','120.25','15','0'])
writer.writerow(['12/26/2001','125.27','125.27','120.25','125.27','100','0'])
writer.writerow(['12/19/2002','96.05','99.98','95.79','99.98','1260','0'])
writer.writerow(['12/20/2002','104.3','104.39','99.98','104.39','197','0'])
writer.writerow(['12/21/2002','109.07','109.13','103.73','109.13','28','0'])
writer.writerow(['12/24/2002','113.57','114.55','109.13','114.55','32','0'])
writer.writerow(['12/25/2002','120.09','120.25','114.55','120.25','15','0'])
writer.writerow(['12/26/2002','125.27','125.27','120.25','125.27','100','0'])
writer.writerow(['12/19/2003','96.05','99.98','95.79','99.98','1260','0'])
writer.writerow(['12/20/2003','104.3','104.39','99.98','104.39','197','0'])
writer.writerow(['12/21/2003','109.07','109.13','103.73','109.13','28','0'])
writer.writerow(['12/24/2003','113.57','114.55','109.13','114.55','32','0'])
writer.writerow(['12/25/2003','120.09','120.25','114.55','120.25','15','0'])
writer.writerow(['12/26/2003','125.27','125.27','120.25','125.27','100','0'])
writer.writerow(['12/19/2004','96.05','99.98','95.79','99.98','1260','0'])
writer.writerow(['12/20/2004','104.3','104.39','99.98','104.39','197','0'])
writer.writerow(['12/21/2004','109.07','109.13','103.73','109.13','28','0'])
writer.writerow(['12/24/2004','113.57','114.55','109.13','114.55','32','0'])
writer.writerow(['12/25/2004','120.09','120.25','114.55','120.25','15','0'])
writer.writerow(['12/26/2004','125.27','125.27','120.25','125.27','100','0'])
# writer.writerow(['12/19/2001','96.05','99.98','95.79','99.98','1260','0'])
# writer.writerow(['12/20/2001','104.3','104.39','99.98','104.39','197','0'])
# writer.writerow(['12/21/2001','109.07','109.13','103.73','109.13','28','0'])
# writer.writerow(['12/24/2001','113.57','114.55','109.13','114.55','32','0'])
# writer.writerow(['12/25/2001','120.09','120.25','114.55','120.25','15','0'])
# writer.writerow(['12/26/2001','125.27','125.27','120.25','125.27','100','0'])
# writer.writerow(['12/19/2002','96.05','99.98','95.79','99.98','1260','0'])
# writer.writerow(['12/20/2002','104.3','104.39','99.98','104.39','197','0'])
# writer.writerow(['12/21/2002','109.07','109.13','103.73','109.13','28','0'])
# writer.writerow(['12/24/2002','113.57','114.55','109.13','114.55','32','0'])
# writer.writerow(['12/25/2002','120.09','120.25','114.55','120.25','15','0'])
# writer.writerow(['12/26/2002','125.27','125.27','120.25','125.27','100','0'])
# writer.writerow(['12/19/2003','96.05','99.98','95.79','99.98','1260','0'])
# writer.writerow(['12/20/2003','104.3','104.39','99.98','104.39','197','0'])
# writer.writerow(['12/21/2003','109.07','109.13','103.73','109.13','28','0'])
# writer.writerow(['12/24/2003','113.57','114.55','109.13','114.55','32','0'])
# writer.writerow(['12/25/2003','120.09','120.25','114.55','120.25','15','0'])
# writer.writerow(['12/26/2003','125.27','125.27','120.25','125.27','100','0'])
# writer.writerow(['12/19/2004','96.05','99.98','95.79','99.98','1260','0'])
# writer.writerow(['12/20/2004','104.3','104.39','99.98','104.39','197','0'])
# writer.writerow(['12/21/2004','109.07','109.13','103.73','109.13','28','0'])
# writer.writerow(['12/24/2004','113.57','114.55','109.13','114.55','32','0'])
# writer.writerow(['12/25/2004','120.09','120.25','114.55','120.25','15','0'])
# writer.writerow(['12/26/2004','125.27','125.27','120.25','125.27','100','0'])
#
return response
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment