Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
haskell-gargantext
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
153
Issues
153
List
Board
Labels
Milestones
Merge Requests
12
Merge Requests
12
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
gargantext
haskell-gargantext
Commits
0ba19424
Commit
0ba19424
authored
Dec 21, 2020
by
Alexandre Delanoë
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[FEAT] SQL fun to insert postagging
parent
fbc1dbea
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
125 additions
and
13 deletions
+125
-13
schema.sql
devops/postgres/schema.sql
+1
-0
NgramsPostag.hs
src/Gargantext/Database/Query/Table/NgramsPostag.hs
+98
-0
NgramsPostag.hs
src/Gargantext/Database/Schema/NgramsPostag.hs
+26
-13
No files found.
devops/postgres/schema.sql
View file @
0ba19424
...
...
@@ -178,6 +178,7 @@ CREATE UNIQUE INDEX ON public.nodes USING btree (hash_id);
CREATE
UNIQUE
INDEX
ON
public
.
ngrams
(
terms
);
-- TEST GIN
CREATE
INDEX
ON
public
.
ngrams
USING
btree
(
id
,
terms
);
CREATE
UNIQUE
INDEX
ON
public
.
ngrams_postag
(
lang_id
,
algo_id
,
postag
,
ngrams_id
,
lemm_id
);
CREATE
INDEX
ON
public
.
node_ngrams
USING
btree
(
node_id
,
node_subtype
);
CREATE
UNIQUE
INDEX
ON
public
.
node_ngrams
USING
btree
(
node_id
,
node_subtype
,
ngrams_id
);
...
...
src/Gargantext/Database/Query/Table/NgramsPostag.hs
0 → 100644
View file @
0ba19424
{-|
Module : Gargantext.Database.Query.Table.NgramsPostag
Description : Deal with in Gargantext Database.
Copyright : (c) CNRS, 2017-Present
License : AGPL + CECILL v3
Maintainer : team@gargantext.org
Stability : experimental
Portability : POSIX
-}
{-# LANGUAGE Arrows #-}
{-# LANGUAGE QuasiQuotes #-}
{-# LANGUAGE TemplateHaskell #-}
module
Gargantext.Database.Query.Table.NgramsPostag
where
import
Data.Text
(
Text
)
import
Gargantext.Database.Prelude
(
Cmd
)
import
Gargantext.Database.Prelude
(
runPGSQuery
)
import
Gargantext.Database.Schema.Ngrams
import
Gargantext.Database.Schema.Prelude
import
Gargantext.Prelude
import
qualified
Database.PostgreSQL.Simple
as
PGS
type
NgramsPostagInsert
=
(
Int
,
Int
,
Text
,
Text
,
Int
,
Text
,
Int
)
insertNgramsPostag
::
[
NgramsPostagInsert
]
->
Cmd
err
[
NgramIds
]
insertNgramsPostag
ns
=
runPGSQuery
queryInsertNgramsPostag
(
PGS
.
Only
$
Values
fields
ns
)
where
fields
=
map
(
\
t
->
QualifiedIdentifier
Nothing
t
)
$
snd
fields_name
fields_name
::
(
[
Text
],
[
Text
])
fields_name
=
(
[
"lang_id"
,
"algo_id"
,
"postag"
,
"form"
,
"form_n"
,
"lem"
,
"lem_n"
]
,
[
"int4"
,
"int4"
,
"text"
,
"text"
,
"int4"
,
"text"
,
"int4"
]
)
----------------------
queryInsertNgramsPostag
::
PGS
.
Query
queryInsertNgramsPostag
=
[
sql
|
WITH input_rows(lang_id,algo_id,postag,form,form_n, lem, lem_n)
AS (?)
-- ((VALUES (1::"int4",2::"int4",'VB'::"text",'dansaient'::"text",1::"int4",'danser'::"text",1::"int4")))
------------------------------------------------
, ins_form AS (INSERT INTO ngrams (terms,n)
SELECT ir1.form, ir1.form_n
FROM input_rows as ir1
UNION ALL
SELECT ir2.lem, ir2.lem_n
FROM input_rows as ir2
ON CONFLICT (terms)
DO NOTHING
RETURNING id,terms
)
------------------------------------------------
, ins_form_ret AS (
SELECT id, terms
FROM ins_form
UNION ALL
SELECT n.id, ir.form
FROM input_rows ir
JOIN ngrams n ON n.terms = ir.form
)
, ins_lem_ret AS (
SELECT id, terms
FROM ins_form
UNION ALL
SELECT n.id, ir.lem
FROM input_rows ir
JOIN ngrams n ON n.terms = ir.lem
)
------------------------------------------------
------------------------------------------------
, ins_postag AS ( INSERT INTO ngrams_postag (lang_id, algo_id, postag, ngrams_id, lemm_id,score)
SELECT ir.lang_id, ir.algo_id, ir.postag, form.id, lem.id, 1
FROM input_rows ir
JOIN ins_form_ret form ON form.terms = ir.form
JOIN ins_lem_ret lem ON lem.terms = ir.lem
ON CONFLICT (lang_id,algo_id,postag,ngrams_id,lemm_id)
DO UPDATE SET score = ngrams_postag.score + 1
)
SELECT * FROM ins_form_ret
|]
src/Gargantext/Database/Schema/NgramsPostag.hs
View file @
0ba19424
...
...
@@ -20,29 +20,33 @@ ngrams in NgramsTerm Lists.
module
Gargantext.Database.Schema.NgramsPostag
where
import
Control.Lens
import
Data.Text
(
Text
)
import
Gargantext.Prelude
import
Gargantext.Database.Schema.Prelude
import
Gargantext.Prelude
import
qualified
Database.PostgreSQL.Simple
as
PGS
data
NgramsPos
T
agPoly
id
data
NgramsPos
t
agPoly
id
lang_id
algo_id
postag
ngrams_id
lemm_id
score
=
NgramsPos
TagDB
{
_ngramsPosT
ag_id
::
!
id
,
_ngramsPos
T
ag_lang_id
::
!
lang_id
,
_ngramsPos
T
ag_algo_id
::
!
algo_id
,
_ngramsPos
T
ag_postag
::
!
postag
,
_ngramsPos
T
ag_ngrams_id
::
!
ngrams_id
,
_ngramsPos
T
ag_lemm_id
::
!
lemm_id
,
_ngramsPos
T
ag_score
::
!
score
=
NgramsPos
tagDB
{
_ngramsPost
ag_id
::
!
id
,
_ngramsPos
t
ag_lang_id
::
!
lang_id
,
_ngramsPos
t
ag_algo_id
::
!
algo_id
,
_ngramsPos
t
ag_postag
::
!
postag
,
_ngramsPos
t
ag_ngrams_id
::
!
ngrams_id
,
_ngramsPos
t
ag_lemm_id
::
!
lemm_id
,
_ngramsPos
t
ag_score
::
!
score
}
deriving
(
Show
)
------------------------------------------------------------------------
type
NgramsPostagDB
=
NgramsPostagPoly
(
Maybe
Int
)
Int
Int
(
Maybe
Text
)
Int
Int
Int
type
NgramsPosTagWrite
=
NgramsPos
T
agPoly
(
Maybe
(
Column
PGInt4
))
type
NgramsPosTagWrite
=
NgramsPos
t
agPoly
(
Maybe
(
Column
PGInt4
))
(
Column
PGInt4
)
(
Column
PGInt4
)
(
Maybe
(
Column
PGText
))
...
...
@@ -50,7 +54,7 @@ type NgramsPosTagWrite = NgramsPosTagPoly (Maybe (Column PGInt4))
(
Column
PGInt4
)
(
Maybe
(
Column
PGInt4
))
type
NgramsPosTagRead
=
NgramsPos
T
agPoly
(
Column
PGInt4
)
type
NgramsPosTagRead
=
NgramsPos
t
agPoly
(
Column
PGInt4
)
(
Column
PGInt4
)
(
Column
PGInt4
)
(
Column
PGText
)
...
...
@@ -58,12 +62,21 @@ type NgramsPosTagRead = NgramsPosTagPoly (Column PGInt4)
(
Column
PGInt4
)
(
Column
PGInt4
)
type
NgramsPosTagReadNull
=
NgramsPos
T
agPoly
(
Column
(
Nullable
PGInt4
))
type
NgramsPosTagReadNull
=
NgramsPos
t
agPoly
(
Column
(
Nullable
PGInt4
))
(
Column
(
Nullable
PGInt4
))
(
Column
(
Nullable
PGInt4
))
(
Column
(
Nullable
PGText
))
(
Column
(
Nullable
PGInt4
))
(
Column
(
Nullable
PGInt4
))
(
Column
(
Nullable
PGInt4
))
makeLenses
''
N
gramsPostagPoly
type
NgramsPosTagDB
=
NgramsPosTagPoly
Int
Int
Int
Text
Int
Int
Int
instance
PGS
.
ToRow
NgramsPostagDB
where
toRow
(
NgramsPostagDB
f0
f1
f2
f3
f4
f5
f6
)
=
[
toField
f0
,
toField
f1
,
toField
f2
,
toField
f3
,
toField
f4
,
toField
f5
,
toField
f6
]
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment