Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
weboob
Project overview
Project overview
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
182
Issues
182
List
Boards
Labels
Milestones
Merge Requests
53
Merge Requests
53
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
weboob
weboob
Commits
4aafce01
Commit
4aafce01
authored
Sep 22, 2016
by
Edouard Lambert
Committed by
Romain Bignon
Jan 08, 2017
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
fixing spirica : stop splitting transactions and improvements
parent
c6b17ab8
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
79 additions
and
101 deletions
+79
-101
modules/spirica/browser.py
modules/spirica/browser.py
+31
-26
modules/spirica/pages.py
modules/spirica/pages.py
+48
-75
No files found.
modules/spirica/browser.py
View file @
4aafce01
...
...
@@ -26,52 +26,57 @@
class
SpiricaBrowser
(
LoginBrowser
):
TIMEOUT
=
60
login
=
URL
(
'/securite/login.xhtml'
,
LoginPage
)
accounts
=
URL
(
'/sylvea/client/synthese.xhtml'
,
AccountsPage
)
details
=
URL
(
'/sylvea/contrat/consultationContratEpargne.xhtml'
,
DetailsPage
)
maintenance
=
URL
(
'/maintenance.html'
,
MaintenancePage
)
def
__init__
(
self
,
website
,
username
,
password
,
*
args
,
**
kwargs
):
super
(
Login
Browser
,
self
)
.
__init__
(
*
args
,
**
kwargs
)
def
__init__
(
self
,
website
,
*
args
,
**
kwargs
):
super
(
Spirica
Browser
,
self
)
.
__init__
(
*
args
,
**
kwargs
)
self
.
BASEURL
=
website
self
.
username
=
username
self
.
password
=
password
self
.
cache
=
{}
self
.
cache
[
'invs'
]
=
{}
self
.
cache
[
'trs'
]
=
{}
def
do_login
(
self
):
self
.
login
.
go
()
.
login
(
self
.
username
,
self
.
password
)
if
self
.
login
.
is_here
():
raise
BrowserIncorrectPassword
error
=
self
.
page
.
get_error
()
raise
BrowserIncorrectPassword
(
error
)
def
get_subscription_list
(
self
):
return
iter
([])
@
need_login
def
iter_accounts
(
self
):
return
self
.
accounts
.
stay_or_go
()
.
iter_accounts
()
if
'accs'
not
in
self
.
cache
.
keys
():
self
.
cache
[
'accs'
]
=
[
a
for
a
in
self
.
accounts
.
stay_or_go
()
.
iter_accounts
()]
return
self
.
cache
[
'accs'
]
@
need_login
def
iter_investment
(
self
,
account
):
# Get form to show PRM
form
=
self
.
location
(
account
.
_link
)
.
page
.
get_investment_form
()
return
self
.
location
(
form
.
url
,
data
=
dict
(
form
))
.
page
.
iter_investment
()
if
account
.
id
not
in
self
.
cache
[
'invs'
]:
# Get form to show PRM
form
=
self
.
location
(
account
.
_link
)
.
page
.
get_investment_form
()
invs
=
[
i
for
i
in
self
.
location
(
form
.
url
,
data
=
dict
(
form
))
.
page
.
iter_investment
()]
self
.
cache
[
'invs'
][
account
.
id
]
=
invs
return
self
.
cache
[
'invs'
][
account
.
id
]
@
need_login
def
iter_history
(
self
,
account
):
# Get form to go to History's tab
form
=
self
.
location
(
account
.
_link
)
.
page
.
get_historytab_form
()
# Get form to show all transactions
form
=
self
.
location
(
form
.
url
,
data
=
dict
(
form
))
.
page
.
get_historyallpages_form
()
if
form
:
self
.
location
(
form
.
url
,
data
=
dict
(
form
))
# Get forms to expand details of all transactions
for
form
in
self
.
page
.
get_historyexpandall_form
():
self
.
location
(
form
.
url
,
data
=
dict
(
form
))
# Get all transactions
self
.
skipped
=
[]
transactions
=
[]
for
t
in
self
.
page
.
iter_history
():
transactions
.
append
(
t
)
for
t
in
self
.
page
.
iter_history_skipped
():
transactions
.
append
(
t
)
return
iter
(
sorted
(
transactions
,
key
=
lambda
t
:
t
.
date
,
reverse
=
True
))
if
account
.
id
not
in
self
.
cache
[
'trs'
]:
# Get form to go to History's tab
form
=
self
.
location
(
account
.
_link
)
.
page
.
get_historytab_form
()
# Get form to show all transactions
form
=
self
.
location
(
form
.
url
,
data
=
dict
(
form
))
.
page
.
get_historyallpages_form
()
if
form
:
self
.
location
(
form
.
url
,
data
=
dict
(
form
))
# Get forms to expand details of all transactions
for
form
in
self
.
page
.
get_historyexpandall_form
():
# Can't async because of ReadTimeout
self
.
location
(
form
.
url
,
data
=
dict
(
form
))
trs
=
[
t
for
t
in
self
.
page
.
iter_history
()]
self
.
cache
[
'trs'
][
account
.
id
]
=
trs
return
self
.
cache
[
'trs'
][
account
.
id
]
modules/spirica/pages.py
View file @
4aafce01
...
...
@@ -21,9 +21,9 @@
import
re
from
weboob.browser.pages
import
HTMLPage
,
LoggedPage
from
weboob.browser.elements
import
ItemElement
,
TableElement
,
SkipItem
,
method
from
weboob.browser.elements
import
ItemElement
,
TableElement
,
method
from
weboob.browser.filters.standard
import
CleanText
,
Date
,
Regexp
,
CleanDecimal
,
\
Env
,
TableCell
,
Field
,
Async
,
AsyncLoad
,
Eval
TableCell
,
Field
,
Async
,
AsyncLoad
,
Eval
from
weboob.browser.filters.html
import
Attr
,
Link
from
weboob.capabilities.bank
import
Account
,
Investment
,
Transaction
from
weboob.capabilities.base
import
NotAvailable
...
...
@@ -48,6 +48,9 @@ def login(self, login, password):
form
[
'loginForm:login'
]
=
"loginForm:login"
form
.
submit
()
def
get_error
(
self
):
return
CleanText
(
'//li[@class="erreurBox"]'
)(
self
.
doc
)
class
AccountsPage
(
LoggedPage
,
HTMLPage
):
TYPES
=
{
'Assurance Vie'
:
Account
.
TYPE_LIFE_INSURANCE
,
'Unknown'
:
Account
.
TYPE_UNKNOWN
}
...
...
@@ -78,6 +81,43 @@ def obj_type(self):
"Option fiscale")]/following-sibling::td'
,
default
=
"Unknown"
))(
self
)]
class
TableInvestment
(
TableElement
):
col_label
=
u'Support'
col_vdate
=
u'Date de valeur'
col_unitvalue
=
u'Valeur de part'
col_quantity
=
u'Nombre de parts'
col_portfolio_share
=
u'
%
'
class
ItemInvestment
(
ItemElement
):
klass
=
Investment
obj_label
=
CleanText
(
TableCell
(
'label'
))
obj_quantity
=
MyDecimal
(
TableCell
(
'quantity'
,
default
=
None
))
obj_unitvalue
=
MyDecimal
(
TableCell
(
'unitvalue'
,
default
=
None
))
obj_vdate
=
Date
(
CleanText
(
TableCell
(
'vdate'
,
default
=
""
)),
dayfirst
=
True
,
default
=
NotAvailable
)
def
obj_valuation
(
self
):
valuation
=
MyDecimal
(
TableCell
(
'valuation'
,
default
=
None
))(
self
)
h2
=
CleanText
(
'./ancestor::div[contains(@id, "Histo")][1]/preceding-sibling::h2[1]'
)(
self
)
return
-
valuation
if
valuation
and
any
(
word
in
h2
.
lower
()
for
word
in
self
.
page
.
DEBIT_WORDS
)
else
valuation
def
obj_portfolio_share
(
self
):
ps
=
MyDecimal
(
TableCell
(
'portfolio_share'
,
default
=
None
))(
self
)
return
Eval
(
lambda
x
:
x
/
100
,
ps
)(
self
)
if
ps
else
NotAvailable
class
TableTransactionsInvestment
(
TableInvestment
):
item_xpath
=
'./tbody/tr'
head_xpath
=
'./thead/tr/th'
col_code
=
u'ISIN'
col_valuation
=
[
u'Montant brut'
,
u'Montant net'
]
class
item
(
ItemInvestment
):
obj_code
=
Regexp
(
CleanText
(
TableCell
(
'code'
)),
pattern
=
'([A-Z]{2}
\
d{10})'
,
default
=
NotAvailable
)
class
DetailsPage
(
LoggedPage
,
HTMLPage
):
DEBIT_WORDS
=
[
u'arrêté'
,
'rachat'
,
'frais'
,
u'désinvestir'
]
...
...
@@ -90,27 +130,14 @@ def get_investment_form(self):
return
form
@
method
class
iter_investment
(
Table
Ele
ment
):
class
iter_investment
(
Table
Invest
ment
):
item_xpath
=
'//div[contains(@id, "INVESTISSEMENT")]//table/tbody/tr[@data-ri]'
head_xpath
=
'//div[contains(@id, "INVESTISSEMENT")]//table/thead/tr/th'
col_label
=
u'Support'
col_vdate
=
u'Date de valeur'
col_unitvalue
=
u'Valeur de part'
col_quantity
=
u'Nombre de parts'
col_valuation
=
re
.
compile
(
'Contre'
)
col_portfolio_share
=
u'
%
'
class
item
(
ItemElement
):
klass
=
Investment
obj_label
=
CleanText
(
TableCell
(
'label'
))
class
item
(
ItemInvestment
):
obj_code
=
Regexp
(
CleanText
(
'.//td[contains(text(), "Isin")]'
),
':[
\
s]+([
\
w]+)'
,
default
=
NotAvailable
)
obj_quantity
=
MyDecimal
(
TableCell
(
'quantity'
))
obj_unitvalue
=
MyDecimal
(
TableCell
(
'unitvalue'
))
obj_valuation
=
MyDecimal
(
TableCell
(
'valuation'
))
obj_vdate
=
Date
(
CleanText
(
TableCell
(
'vdate'
)),
dayfirst
=
True
,
default
=
NotAvailable
)
obj_portfolio_share
=
Eval
(
lambda
x
:
x
/
100
,
MyDecimal
(
TableCell
(
'portfolio_share'
)))
def
obj_unitprice
(
self
):
return
MyDecimal
(
'//div[contains(@id, "PRIX_REVIENT")]//a[contains(text(),
\
...
...
@@ -149,36 +176,6 @@ def get_historyexpandall_form(self):
form
[
'ongletHistoOperations:newoperations_expandedRowIndex'
]
=
data
yield
form
def
get_investments
(
self
,
el
,
xpath
=
'.'
):
# Get all positions of th
positions
=
{}
keys
=
{
'isin'
:
'code'
,
'support'
:
'label'
,
'supports'
:
'label'
,
'nombre de parts'
:
'quantity'
,
'valeur de part'
:
\
'unitvalue'
,
'montant brut'
:
'valuation'
,
'date de valeur'
:
'vdate'
,
'
%
'
:
'portfolio_share'
}
for
position
,
th
in
enumerate
(
el
.
xpath
(
"
%
s//thead//th"
%
xpath
)):
key
=
CleanText
()
.
filter
(
th
.
xpath
(
'.'
))
.
lower
()
if
key
in
keys
:
positions
[
keys
[
key
]]
=
position
+
1
investments
=
[]
for
tr
in
el
.
xpath
(
"
%
s//tbody/tr[@data-ri]"
%
xpath
):
i
=
Investment
()
i
.
label
=
CleanText
()
.
filter
(
tr
.
xpath
(
'./td[
%
s]'
%
positions
[
'label'
]))
\
if
"label"
in
positions
else
NotAvailable
i
.
code
=
Regexp
(
CleanText
(
'./td[
%
s]'
%
positions
[
'code'
]),
pattern
=
'([A-Z]{2}
\
d{10})'
,
default
=
NotAvailable
)(
tr
)
i
.
quantity
=
MyDecimal
()
.
filter
(
tr
.
xpath
(
'./td[
%
s]'
%
positions
[
'quantity'
]))
\
if
"quantity"
in
positions
else
NotAvailable
i
.
unitvalue
=
MyDecimal
()
.
filter
(
tr
.
xpath
(
'./td[
%
s]'
%
positions
[
'unitvalue'
]))
\
if
"unitvalue"
in
positions
else
NotAvailable
i
.
valuation
=
MyDecimal
()
.
filter
(
tr
.
xpath
(
'./td[
%
s]'
%
positions
[
'valuation'
]))
\
if
"valuation"
in
positions
else
NotAvailable
i
.
vdate
=
Date
(
CleanText
(
'./td[
%
s]'
%
positions
[
'vdate'
]),
dayfirst
=
True
,
default
=
NotAvailable
)(
tr
)
\
if
"vdate"
in
positions
else
NotAvailable
i
.
portfolio_share
=
Eval
(
lambda
x
:
x
/
100
)
.
filter
([
MyDecimal
()
.
filter
(
tr
.
xpath
(
'./td[
%
s]'
%
positions
[
'portfolio_share'
]))])
\
if
"portfolio_share"
in
positions
else
NotAvailable
investments
.
append
(
i
)
return
investments
@
method
class
iter_history
(
TableElement
):
item_xpath
=
'//table/tbody[@id and not(contains(@id, "j_idt"))]/tr[@data-ri]'
...
...
@@ -197,7 +194,6 @@ class item(ItemElement):
obj_label
=
CleanText
(
TableCell
(
'label'
))
obj_vdate
=
Date
(
CleanText
(
TableCell
(
'vdate'
)),
dayfirst
=
True
)
obj_type
=
Transaction
.
TYPE_BANK
obj_investments
=
Env
(
'investments'
)
def
obj_amount
(
self
):
amount
=
MyDecimal
(
TableCell
(
'net'
)
if
not
CleanText
(
TableCell
(
'brut'
))(
self
)
else
TableCell
(
'brut'
))(
self
)
...
...
@@ -209,30 +205,7 @@ def obj_date(self):
def
condition
(
self
):
return
u"Validé"
in
CleanText
(
TableCell
(
'status'
))(
self
)
def
parse
(
self
,
el
):
if
u"Désinvestir"
in
CleanText
(
'./following-sibling::tr[1]'
)(
self
):
self
.
page
.
browser
.
skipped
.
append
([
el
,
el
.
xpath
(
'./following-sibling::tr[1]'
)[
0
]])
raise
SkipItem
()
self
.
env
[
'investments'
]
=
self
.
page
.
get_investments
(
el
,
\
'./following-sibling::tr[1]//span[contains(text(), "ISIN")]/ancestor::table[1]'
)
def
iter_history_skipped
(
self
):
for
tr1
,
tr2
in
self
.
browser
.
skipped
:
for
table
,
h2
in
zip
(
tr2
.
xpath
(
'.//table[@role]'
),
tr2
.
xpath
(
u'.//h2'
)):
t
=
Transaction
()
t
.
vdate
=
Date
(
CleanText
(
'./td[8]'
),
dayfirst
=
True
)(
tr1
)
t
.
date
=
Date
(
CleanText
(
'./td[6]'
),
dayfirst
=
True
,
default
=
t
.
vdate
)(
tr1
)
t
.
type
=
Transaction
.
TYPE_BANK
t
.
label
=
u"
%
s -
%
s"
%
(
CleanText
()
.
filter
(
tr1
.
xpath
(
'./td[2]'
)),
\
CleanText
()
.
filter
(
h2
.
xpath
(
'.'
)))
t
.
amount
=
CleanDecimal
(
replace_dots
=
True
,
default
=
MyDecimal
()
.
filter
(
\
tr1
.
xpath
(
'./td[5]'
)))
.
filter
(
tr1
.
xpath
(
'./td[4]'
))
if
t
.
amount
and
any
(
word
in
t
.
label
.
lower
()
for
word
in
self
.
DEBIT_WORDS
):
t
.
amount
=
-
t
.
amount
t
.
investments
=
self
.
get_investments
(
table
)
yield
t
def
obj_investments
(
self
):
investments
=
[]
for
table
in
self
.
el
.
xpath
(
'./following-sibling::tr[1]//span[contains(text(), "ISIN")]/ancestor::table[1]'
):
investments
=
sum
([
investments
,
list
(
TableTransactionsInvestment
(
self
.
page
,
el
=
table
)())],
[])
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment