Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
gargantext
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
humanities
gargantext
Commits
73289d0b
Commit
73289d0b
authored
Dec 12, 2015
by
PkSM3
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[UPDATE] occs par default
parent
b81f45fd
Changes
7
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
49 additions
and
28 deletions
+49
-28
workflow.py
ngram/workflow.py
+1
-1
NgramsExtractor.py
parsing/NgramsExtractors/NgramsExtractor.py
+2
-2
TurboNgramsExtractor.py
parsing/NgramsExtractors/TurboNgramsExtractor.py
+2
-2
ngrams.py
rest_v1_0/ngrams.py
+18
-5
NGrams_dyna_chart_and_table.js
static/js/NGrams_dyna_chart_and_table.js
+24
-16
menu.html
templates/corpus/menu.html
+1
-1
views.py
tests/ngramstable/views.py
+1
-1
No files found.
ngram/workflow.py
View file @
73289d0b
...
@@ -6,7 +6,7 @@ from ngram.stop import compute_stop
...
@@ -6,7 +6,7 @@ from ngram.stop import compute_stop
from
ngram.group
import
compute_groups
from
ngram.group
import
compute_groups
from
gargantext_web.db
import
get_or_create_node
from
gargantext_web.db
import
get_or_create_node
from
ngram.mapList
import
compute_mapList
from
ngram.mapList
import
compute_mapList
#
from ngram.occurrences import compute_occs
from
ngram.occurrences
import
compute_occs
from
gargantext_web.db
import
session
,
Node
,
NodeNgram
from
gargantext_web.db
import
session
,
Node
,
NodeNgram
from
admin.utils
import
WorkflowTracking
from
admin.utils
import
WorkflowTracking
...
...
parsing/NgramsExtractors/NgramsExtractor.py
View file @
73289d0b
from
..Taggers
import
TurboTagger
#
NltkTagger #,
from
..Taggers
import
NltkTagger
#,
import
nltk
import
nltk
...
@@ -21,7 +21,7 @@ class NgramsExtractor:
...
@@ -21,7 +21,7 @@ class NgramsExtractor:
self
.
stop
()
self
.
stop
()
def
start
(
self
):
def
start
(
self
):
self
.
tagger
=
Turbo
Tagger
()
self
.
tagger
=
Nltk
Tagger
()
def
stop
(
self
):
def
stop
(
self
):
pass
pass
...
...
parsing/NgramsExtractors/TurboNgramsExtractor.py
View file @
73289d0b
from
.NgramsExtractor
import
NgramsExtractor
from
.NgramsExtractor
import
NgramsExtractor
from
..Taggers
import
TurboTagger
#NltkTagger
from
..Taggers
import
NltkTagger
#TurboTagger
class
TurboNgramsExtractor
(
NgramsExtractor
):
class
TurboNgramsExtractor
(
NgramsExtractor
):
def
start
(
self
):
def
start
(
self
):
self
.
tagger
=
Turbo
Tagger
()
self
.
tagger
=
Nltk
Tagger
()
rest_v1_0/ngrams.py
View file @
73289d0b
...
@@ -91,17 +91,30 @@ class List(APIView):
...
@@ -91,17 +91,30 @@ class List(APIView):
"id"
:
node
.
id
,
"id"
:
node
.
id
,
"name"
:
node
.
terms
,
"name"
:
node
.
terms
,
"scores"
:
{
"scores"
:
{
"tfidf"
:
0
"tfidf"
:
0
,
"occs"
:
0
}
}
}
}
# occ_list = get_or_create_node(nodetype='Occurrences', corpus_id=parent_id).id
# occ_list = get_or_create_node(nodetype='Occurrences', corpus_id=parent_id).id
# print( occ_list )
# print( occ_list )
tfidf_list
=
get_or_create_node
(
nodetype
=
'Tfidf (global)'
,
corpus_id
=
parent_id
)
.
id
try
:
ngram_tfidf
=
session
.
query
(
NodeNodeNgram
.
ngram_id
,
NodeNodeNgram
.
score
)
.
filter
(
NodeNodeNgram
.
nodex_id
==
tfidf_list
,
NodeNodeNgram
.
ngram_id
.
in_
(
list
(
ngram_ids
.
keys
())
))
.
all
()
tfidf_list
=
get_or_create_node
(
nodetype
=
'Tfidf (global)'
,
corpus_id
=
parent_id
)
.
id
for
n
in
ngram_tfidf
:
ngram_tfidf
=
session
.
query
(
NodeNodeNgram
.
ngram_id
,
NodeNodeNgram
.
score
)
.
filter
(
NodeNodeNgram
.
nodex_id
==
tfidf_list
,
NodeNodeNgram
.
ngram_id
.
in_
(
list
(
ngram_ids
.
keys
())
))
.
all
()
for
n
in
ngram_tfidf
:
if
n
.
ngram_id
in
ngram_ids
:
ngram_ids
[
n
.
ngram_id
][
"scores"
][
"tfidf"
]
+=
n
.
score
except
:
pass
try
:
occ_list
=
get_or_create_node
(
nodetype
=
'Occurrences'
,
corpus_id
=
parent_id
)
.
id
ngram_occs
=
session
.
query
(
NodeNodeNgram
.
ngram_id
,
NodeNodeNgram
.
score
)
.
filter
(
NodeNodeNgram
.
nodex_id
==
occ_list
,
NodeNodeNgram
.
ngram_id
.
in_
(
list
(
ngram_ids
.
keys
())
))
.
all
()
for
n
in
ngram_occs
:
if
n
.
ngram_id
in
ngram_ids
:
if
n
.
ngram_id
in
ngram_ids
:
ngram_ids
[
n
.
ngram_id
][
"scores"
][
"tfidf"
]
+=
n
.
score
ngram_ids
[
n
.
ngram_id
][
"scores"
][
"occs"
]
+=
round
(
n
.
score
)
except
:
pass
end_
=
time
.
time
()
end_
=
time
.
time
()
...
...
static/js/NGrams_dyna_chart_and_table.js
View file @
73289d0b
...
@@ -97,14 +97,16 @@ var LineChart = dc.lineChart("#monthly-move-chart");
...
@@ -97,14 +97,16 @@ var LineChart = dc.lineChart("#monthly-move-chart");
var
volumeChart
=
dc
.
barChart
(
"#monthly-volume-chart"
);
var
volumeChart
=
dc
.
barChart
(
"#monthly-volume-chart"
);
//
Just for Garg
//
Get all projects and corpuses of the user
function
GetUserPortfolio
()
{
function
GetUserPortfolio
()
{
//http://localhost:8000/api/corpusintersection/1a50317a50145
//http://localhost:8000/api/corpusintersection/1a50317a50145
var
project_id
=
getIDFromURL
(
"project"
)
var
project_id
=
getIDFromURL
(
"project"
)
var
corpus_id
=
getIDFromURL
(
"corpus"
)
var
corpus_id
=
getIDFromURL
(
"corpus"
)
if
(
Object
.
keys
(
corpusesList
).
length
>
0
)
if
(
Object
.
keys
(
corpusesList
).
length
>
0
)
{
return
true
;
$
(
'#corpuses'
).
modal
(
'show'
);
return
true
;
}
var
query_url
=
window
.
location
.
origin
+
'/api/userportfolio/project/'
+
project_id
+
'/corpuses'
var
query_url
=
window
.
location
.
origin
+
'/api/userportfolio/project/'
+
project_id
+
'/corpuses'
$
.
ajax
({
$
.
ajax
({
...
@@ -158,8 +160,6 @@ function GetUserPortfolio() {
...
@@ -158,8 +160,6 @@ function GetUserPortfolio() {
});
});
$
(
'#corpuses'
).
modal
(
'show'
);
$
(
'#corpuses'
).
modal
(
'show'
);
},
},
error
:
function
(){
error
:
function
(){
pr
(
'Page Not found: TestFunction()'
);
pr
(
'Page Not found: TestFunction()'
);
...
@@ -167,6 +167,7 @@ function GetUserPortfolio() {
...
@@ -167,6 +167,7 @@ function GetUserPortfolio() {
});
});
}
}
//Getting a corpusB-list and intersecting it with current corpusA-miamlist.
function
printCorpuses
()
{
function
printCorpuses
()
{
console
.
log
(
"!!!!!!!! in printCorpuses() !!!!!!!! "
)
console
.
log
(
"!!!!!!!! in printCorpuses() !!!!!!!! "
)
pr
(
corpusesList
)
pr
(
corpusesList
)
...
@@ -585,6 +586,7 @@ function SaveGlobalChanges_Form( nodes2del) {
...
@@ -585,6 +586,7 @@ function SaveGlobalChanges_Form( nodes2del) {
$
(
"#pre_savechanges"
).
modal
(
"show"
)
$
(
"#pre_savechanges"
).
modal
(
"show"
)
}
}
// Save changes to all corpusA-lists
function
SaveLocalChanges
()
{
function
SaveLocalChanges
()
{
console
.
clear
()
console
.
clear
()
...
@@ -713,7 +715,7 @@ $("#Save_All").click(function(){
...
@@ -713,7 +715,7 @@ $("#Save_All").click(function(){
// }
// }
});
});
// For lists, all http-requests
function
CRUD
(
parent_id
,
action
,
nodes
,
args
,
http_method
,
callback
)
{
function
CRUD
(
parent_id
,
action
,
nodes
,
args
,
http_method
,
callback
)
{
var
the_url
=
window
.
location
.
origin
+
"/api/node/"
+
parent_id
+
"/ngrams"
+
action
+
"/"
+
nodes
.
join
(
"+"
);
var
the_url
=
window
.
location
.
origin
+
"/api/node/"
+
parent_id
+
"/ngrams"
+
action
+
"/"
+
nodes
.
join
(
"+"
);
the_url
=
the_url
.
replace
(
/
\/
$/
,
""
);
//remove trailing slash
the_url
=
the_url
.
replace
(
/
\/
$/
,
""
);
//remove trailing slash
...
@@ -1060,6 +1062,7 @@ function getIDFromURL( item ) {
...
@@ -1060,6 +1062,7 @@ function getIDFromURL( item ) {
return
pageurl
[
cid
+
1
];
return
pageurl
[
cid
+
1
];
}
}
// For lists, only GET requests
function
GET_
(
url
,
callback
)
{
function
GET_
(
url
,
callback
)
{
$
.
ajax
({
$
.
ajax
({
...
@@ -1107,18 +1110,23 @@ var url = [
...
@@ -1107,18 +1110,23 @@ var url = [
GET_
(
url
[
0
]
,
function
(
result
)
{
GET_
(
url
[
0
]
,
function
(
result
)
{
// = = = = MIAM = = = = //
// = = = = MIAM = = = = //
if
(
result
!=
false
)
{
if
(
result
!=
false
)
{
NGrams
[
"main"
]
=
{
NGrams
[
"main"
]
=
{
"ngrams"
:
[],
"ngrams"
:
[],
"scores"
:
{
"scores"
:
{
"initial"
:
"tfidf
"
,
"initial"
:
"occs
"
,
"nb_docs"
:
result
.
length
,
"nb_docs"
:
result
.
length
,
"orig_nb_ngrams"
:
1
,
"orig_nb_ngrams"
:
1
,
"nb_ngrams"
:
result
.
length
,
"nb_ngrams"
:
result
.
length
,
}
}
}
}
for
(
var
i
in
result
)
var
occs_sum
=
0
for
(
var
i
in
result
)
{
NGrams
[
"main"
].
ngrams
.
push
(
result
[
i
])
NGrams
[
"main"
].
ngrams
.
push
(
result
[
i
])
occs_sum
+=
result
[
i
].
scores
.
occs
}
if
(
occs_sum
==
0
)
NGrams
[
"main"
][
"scores"
][
"initial"
]
=
"tfidf"
;
}
}
// = = = = /MIAM = = = = //
// = = = = /MIAM = = = = //
...
...
templates/corpus/menu.html
View file @
73289d0b
...
@@ -49,7 +49,7 @@
...
@@ -49,7 +49,7 @@
<a
class=
"btn btn-primary btn-lg"
role=
"button"
href=
"/admin/documents/corpus/{{ corpus.id }}/"
>
Add documents
</a></p>
<a
class=
"btn btn-primary btn-lg"
role=
"button"
href=
"/admin/documents/corpus/{{ corpus.id }}/"
>
Add documents
</a></p>
{% endif %}
{% endif %}
{% if nb_groups != None and nb_groups > 0 %}
{% if nb_groups != None and nb_groups > 0 %}
<a
style=
"visibility: hidden;"
id=
"share_button"
class=
"btn btn-primary btn-lg"
role=
"button"
>
Share!!!
</a></p>
<a
id=
"share_button"
class=
"btn btn-primary btn-lg"
role=
"button"
>
Share!!!
</a></p>
{% endif %}
{% endif %}
</div>
</div>
</div>
</div>
...
...
tests/ngramstable/views.py
View file @
73289d0b
...
@@ -252,7 +252,7 @@ def graph_share(request, generic=100, specific=100):
...
@@ -252,7 +252,7 @@ def graph_share(request, generic=100, specific=100):
def
node_link_share
(
request
):
def
node_link_share
(
request
):
data
=
{
"hola"
:
"mundo"
}
data
=
{
"request"
:
"error"
}
if
request
.
method
==
'GET'
and
"token"
in
request
.
GET
:
if
request
.
method
==
'GET'
and
"token"
in
request
.
GET
:
import
json
import
json
le_token
=
json
.
loads
(
request
.
GET
[
"token"
])[
0
]
le_token
=
json
.
loads
(
request
.
GET
[
"token"
])[
0
]
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment