From 4d8239301f13305051ff6a65dd7a132c4aeecc51 Mon Sep 17 00:00:00 2001 From: woofer Date: Tue, 2 Apr 2024 12:12:47 +0200 Subject: [PATCH 1/7] [ccf] use distri-account-api, add savings accounts --- modules/ccf/browser.py | 28 +++++++++++++++++++++++++-- modules/ccf/pages.py | 44 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/modules/ccf/browser.py b/modules/ccf/browser.py index 4d4b39b3d..ee82096de 100644 --- a/modules/ccf/browser.py +++ b/modules/ccf/browser.py @@ -27,7 +27,7 @@ from woob.tools.decorators import retry -from .pages import SubscriptionsPage, DocumentsPage, RibPage, TransactionsPage +from .pages import SubscriptionsPage, DocumentsPage, RibPage, TransactionsPage, AccountsPage, BalancePage __all__ = ["CCFParBrowser", "CCFProBrowser"] @@ -37,6 +37,14 @@ class CCFBrowser(CmsoParBrowser): arkea_si = None AUTH_CLIENT_ID = "S4dgkKwTA7FQzWxGRHPXe6xNvihEATOY" + # accounts_: note the trailing underscore + # don't override super.accounts, used indirectly by get_ibans_from_ribs + accounts_ = URL( + r"/distri-account-api/api/v1/persons/me/accounts", AccountsPage + ) + balance = URL( + r"/distri-account-api/api/v1/customers/me/accounts/(?P.*)/balances", BalancePage + ) subscriptions = URL( r"/distri-account-api/api/v1/customers/me/accounts", SubscriptionsPage ) @@ -132,11 +140,27 @@ def update_iban(self, account): if not account.iban: account.iban = iban_number - def iter_accounts(self): + def get_ibans_from_ribs(self): accounts_list = super().iter_accounts() for account in accounts_list: account._original_id = account.id self.update_iban(account) + return { account.id: account.iban for account in accounts_list } + + @need_login + def iter_accounts(self): + ibans = self.get_ibans_from_ribs() + + go_accounts = retry(ClientError, tries=5)(self.accounts_.go) + go_accounts(params={'types': 'CHECKING,SAVING'}) + + accounts_list = list(self.page.iter_accounts()) + for account in accounts_list: + self.balance.go(account_id=account.id) + balance = list(self.page.iter_balances())[0] + account.balance = balance.amount + account.iban = ibans.get(account.id) + return accounts_list @need_login diff --git a/modules/ccf/pages.py b/modules/ccf/pages.py index 2c90ffbbf..e9fea8b76 100644 --- a/modules/ccf/pages.py +++ b/modules/ccf/pages.py @@ -33,6 +33,7 @@ ) from woob.browser.pages import JsonPage, LoggedPage +from woob.capabilities.bank import Account, AccountType, AccountOwnerType, AccountOwnership from woob.capabilities.bill import Document, DocumentTypes, Subscription from woob.tools.capabilities.bank.transactions import FrenchTransaction @@ -42,6 +43,49 @@ def get_iban(self): return Dict("iban")(self.doc) +ACCOUNT_TYPES = { + "CHECKING": AccountType.CHECKING, + "STOCK": AccountType.PEA, + "HAV": AccountType.LIFE_INSURANCE, +} + +class AccountsPage(LoggedPage, JsonPage): + @method + class iter_accounts(DictElement): + class item(ItemElement): + klass = Account + + obj_id = Dict("accountId") + obj_label = Dict("label") + + obj_type = Map(Dict("type"), ACCOUNT_TYPES, AccountType.UNKNOWN) + obj_ownership = AccountOwnership.OWNER + obj_owner_type = AccountOwnerType.PRIVATE + + obj__type = Dict("type") + obj__iban_encrypted = Dict("iban") + + +class Balance: pass + +class BalancePage(LoggedPage, JsonPage): + @method + class iter_balances(DictElement): + def store(self, obj): + obj.id = f"balance-{len(self.objects)}" + self.objects[obj.id] = obj + return obj + + class item(ItemElement): + klass = Balance + + obj_amount = CleanDecimal.SI(Dict('balanceAmount/amount')) + obj_currency = Dict('balanceAmount/currency') + + obj_refDate = Date(Dict('referenceDate')) + obj_balType = Dict("balanceType") + + class SubscriptionsPage(LoggedPage, JsonPage): @method class iter_subscriptions(DictElement): -- GitLab From 78eb430dc294854fd775cb97ec3b54b4c64b6817 Mon Sep 17 00:00:00 2001 From: woofer Date: Tue, 2 Apr 2024 19:15:15 +0200 Subject: [PATCH 2/7] [ccf] fix RibPage to not crash on no-iban accounts --- modules/ccf/pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ccf/pages.py b/modules/ccf/pages.py index e9fea8b76..476685c0b 100644 --- a/modules/ccf/pages.py +++ b/modules/ccf/pages.py @@ -40,7 +40,7 @@ class RibPage(LoggedPage, JsonPage): def get_iban(self): - return Dict("iban")(self.doc) + return Dict("iban", default=None)(self.doc) ACCOUNT_TYPES = { -- GitLab From 12e5acedcb14c2bcb378f98d1709d25796655775 Mon Sep 17 00:00:00 2001 From: woofer Date: Wed, 3 Apr 2024 14:39:18 +0200 Subject: [PATCH 3/7] [ccf] add note about potential common browser w/ allianzbanque --- modules/ccf/browser.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/ccf/browser.py b/modules/ccf/browser.py index ee82096de..c1a80498e 100644 --- a/modules/ccf/browser.py +++ b/modules/ccf/browser.py @@ -37,6 +37,11 @@ class CCFBrowser(CmsoParBrowser): arkea_si = None AUTH_CLIENT_ID = "S4dgkKwTA7FQzWxGRHPXe6xNvihEATOY" + # Use CmsoParBrowser as base, but rely on /distri-account-api/api + # for accounts list & balance. Like modules/allianzbanque/browser.py + # We should probably extract a common browser. + + # accounts_: note the trailing underscore # don't override super.accounts, used indirectly by get_ibans_from_ribs accounts_ = URL( -- GitLab From ad0ad5d33a48206bd5693c788b93d2177da4b088 Mon Sep 17 00:00:00 2001 From: woofer Date: Wed, 3 Apr 2024 14:51:56 +0200 Subject: [PATCH 4/7] [ccf] use schwifty to reconstruct accounts .iban --- modules/ccf/browser.py | 17 ++++++++++++++--- modules/ccf/pages.py | 2 ++ modules/ccf/requirements.txt | 1 + 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/modules/ccf/browser.py b/modules/ccf/browser.py index c1a80498e..8e50abaab 100644 --- a/modules/ccf/browser.py +++ b/modules/ccf/browser.py @@ -19,6 +19,8 @@ from base64 import b64encode from hashlib import sha256 +from schwifty import IBAN + from woob.browser import URL, need_login from woob.browser.browsers import ClientError, ServerError from woob_modules.cmso.par.browser import CmsoParBrowser @@ -154,8 +156,6 @@ def get_ibans_from_ribs(self): @need_login def iter_accounts(self): - ibans = self.get_ibans_from_ribs() - go_accounts = retry(ClientError, tries=5)(self.accounts_.go) go_accounts(params={'types': 'CHECKING,SAVING'}) @@ -164,7 +164,7 @@ def iter_accounts(self): self.balance.go(account_id=account.id) balance = list(self.page.iter_balances())[0] account.balance = balance.amount - account.iban = ibans.get(account.id) + account.iban = mk_iban(account) return accounts_list @@ -177,6 +177,17 @@ def iter_history(self, account): go_transactions(account_id=account._original_id) return self.page.iter_transactions() +def mk_iban(account): + _iban = account._iban_offended + _contract = account._contract_id + + return str(IBAN.generate( + country_code=_iban[:2], + bank_code=_iban[4:9], + branch_code=_iban[9:14], + account_code=_contract[0]+_contract[4:14]) + ) if _iban else None + class CCFParBrowser(CCFBrowser): BASEURL = "https://api.ccf.fr" diff --git a/modules/ccf/pages.py b/modules/ccf/pages.py index 476685c0b..8a3498fef 100644 --- a/modules/ccf/pages.py +++ b/modules/ccf/pages.py @@ -64,6 +64,8 @@ class item(ItemElement): obj__type = Dict("type") obj__iban_encrypted = Dict("iban") + obj__iban_offended = Dict("offendedIBAN") + obj__contract_id = Dict("sourceContractId") class Balance: pass diff --git a/modules/ccf/requirements.txt b/modules/ccf/requirements.txt index 1ef9ec69a..2ab9712ca 100644 --- a/modules/ccf/requirements.txt +++ b/modules/ccf/requirements.txt @@ -1 +1,2 @@ woob ~= 3.6 +schwifty -- GitLab From b8f4c4a44c6e41d3a05178b048902cb928413a62 Mon Sep 17 00:00:00 2001 From: Ludovic LANGE Date: Fri, 5 Apr 2024 11:45:07 +0200 Subject: [PATCH 5/7] fix subscription handling --- modules/ccf/browser.py | 17 ++++++----------- modules/ccf/pages.py | 25 +++++++++++++++++++------ 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/modules/ccf/browser.py b/modules/ccf/browser.py index 8e50abaab..3839ee4c9 100644 --- a/modules/ccf/browser.py +++ b/modules/ccf/browser.py @@ -119,17 +119,12 @@ def build_request(self, *args, **kwargs): @need_login def get_subscription_list(self): - accounts_list = self.iter_accounts() - subscriptions = [] - for account in accounts_list: - s = Subscription() - s.label = account._lib - if account.number: - s.label = f"{s.label} {account.number}" - s.subscriber = account._owner_name - s.id = account.id - subscriptions.append(s) - return subscriptions + params = { + 'types': 'CHECKING', + 'roles': 'TIT,COT', + } + self.subscriptions.go(params=params) + return self.page.iter_subscriptions() @need_login def iter_documents(self, subscription): diff --git a/modules/ccf/pages.py b/modules/ccf/pages.py index 8a3498fef..d8e04117a 100644 --- a/modules/ccf/pages.py +++ b/modules/ccf/pages.py @@ -94,16 +94,29 @@ class iter_subscriptions(DictElement): class item(ItemElement): klass = Subscription - obj_id = Dict("sourceContractId") + obj_id = Dict("accountId") + obj_label = Dict("label") # there can be several "participants" but no matter what _contract_id is, # list of related documents will be the same, so we can simply take the first one obj__contract_id = Dict("participants/0/id") # CAUTION non persistant - obj_subscriber = Format( - "%s %s", - CleanText(Dict("participants/0/firstName")), - CleanText(Dict("participants/0/lastName")), - ) + def obj_subscriber(self): + def key_participants(participant): + role = participant.get("role", None) + if role == "TIT": + role_idx = 0 + else: + role_idx = 1 + return '%s-%s-%s' % (role_idx, participant.get("lastName"), participant.get("firstName")) + + result = "" + for participant in sorted(Dict("participants")(self), key=key_participants): + result += Format( + "%s %s / ", + CleanText(Dict("lastName")), + CleanText(Dict("firstName")), + )(participant) + return result.strip("/ ") DOCUMENT_TYPES = { -- GitLab From ba38e4c9138761ea98248440808c807ea8eedf0c Mon Sep 17 00:00:00 2001 From: Ludovic LANGE Date: Fri, 5 Apr 2024 11:48:08 +0200 Subject: [PATCH 6/7] re-use axabanque logic and AccountsPage definition - slightly modified --- modules/ccf/browser.py | 20 ++++++++++----- modules/ccf/pages.py | 56 +++++++++++++++--------------------------- 2 files changed, 34 insertions(+), 42 deletions(-) diff --git a/modules/ccf/browser.py b/modules/ccf/browser.py index 3839ee4c9..3fc56a909 100644 --- a/modules/ccf/browser.py +++ b/modules/ccf/browser.py @@ -15,6 +15,9 @@ # You should have received a copy of the GNU Lesser General Public License # along with this woob module. If not, see . +from datetime import date +from dateutil.relativedelta import relativedelta + import random from base64 import b64encode from hashlib import sha256 @@ -29,7 +32,7 @@ from woob.tools.decorators import retry -from .pages import SubscriptionsPage, DocumentsPage, RibPage, TransactionsPage, AccountsPage, BalancePage +from .pages import SubscriptionsPage, DocumentsPage, RibPage, TransactionsPage, AccountsPage __all__ = ["CCFParBrowser", "CCFProBrowser"] @@ -50,7 +53,11 @@ class CCFBrowser(CmsoParBrowser): r"/distri-account-api/api/v1/persons/me/accounts", AccountsPage ) balance = URL( - r"/distri-account-api/api/v1/customers/me/accounts/(?P.*)/balances", BalancePage + r"/distri-account-api/api/v1/customers/me/accounts/(?P.*)/balances", AccountsPage + ) + balances_comings = URL( + r'/distri-account-api/api/v1/persons/me/accounts/(?P[A-Z0-9]{10})/total-upcoming-transactions', + AccountsPage ) subscriptions = URL( r"/distri-account-api/api/v1/customers/me/accounts", SubscriptionsPage @@ -157,11 +164,12 @@ def iter_accounts(self): accounts_list = list(self.page.iter_accounts()) for account in accounts_list: self.balance.go(account_id=account.id) - balance = list(self.page.iter_balances())[0] - account.balance = balance.amount + self.page.fill_balance(account) + date_to = (date.today() + relativedelta(months=1)).strftime('%Y-%m-%dT00:00:00.000Z') + self.balances_comings.go(account_id=account.id, params={'dateTo': date_to}) + self.page.fill_coming(account) account.iban = mk_iban(account) - - return accounts_list + yield account @need_login def iter_history(self, account): diff --git a/modules/ccf/pages.py b/modules/ccf/pages.py index d8e04117a..1e618c223 100644 --- a/modules/ccf/pages.py +++ b/modules/ccf/pages.py @@ -28,14 +28,18 @@ Eval, Field, Format, + Lower, Map, + MapIn, Regexp, + Upper, ) from woob.browser.pages import JsonPage, LoggedPage from woob.capabilities.bank import Account, AccountType, AccountOwnerType, AccountOwnership from woob.capabilities.bill import Document, DocumentTypes, Subscription from woob.tools.capabilities.bank.transactions import FrenchTransaction +from woob_modules.axabanque.pages.bank import AccountsPage as _AccountsPage, ACCOUNT_TYPES as ACCOUNT_TYPES_GENERIC class RibPage(LoggedPage, JsonPage): @@ -43,49 +47,29 @@ def get_iban(self): return Dict("iban", default=None)(self.doc) -ACCOUNT_TYPES = { - "CHECKING": AccountType.CHECKING, - "STOCK": AccountType.PEA, - "HAV": AccountType.LIFE_INSURANCE, +ACCOUNT_TYPES_OVERRIDE = { + "checking": AccountType.CHECKING, + "stock": AccountType.PEA, # 'compte titres' and 'plan d'epargne en actions' have both the same type with the type field + "hav": AccountType.LIFE_INSURANCE, } -class AccountsPage(LoggedPage, JsonPage): - @method - class iter_accounts(DictElement): - class item(ItemElement): - klass = Account - - obj_id = Dict("accountId") - obj_label = Dict("label") - - obj_type = Map(Dict("type"), ACCOUNT_TYPES, AccountType.UNKNOWN) - obj_ownership = AccountOwnership.OWNER - obj_owner_type = AccountOwnerType.PRIVATE +ACCOUNT_TYPES = {**ACCOUNT_TYPES_GENERIC, **ACCOUNT_TYPES_OVERRIDE} - obj__type = Dict("type") +class AccountsPage(_AccountsPage): + @method + class iter_accounts(_AccountsPage.iter_accounts.klass): + class item(_AccountsPage.iter_accounts.klass.item): + obj_type = Coalesce( + MapIn(Lower(Dict('type')), ACCOUNT_TYPES, Account.TYPE_UNKNOWN), + MapIn(Lower(Dict('label')), ACCOUNT_TYPES, Account.TYPE_UNKNOWN), + default=Account.TYPE_UNKNOWN, + ) obj__iban_encrypted = Dict("iban") obj__iban_offended = Dict("offendedIBAN") obj__contract_id = Dict("sourceContractId") - -class Balance: pass - -class BalancePage(LoggedPage, JsonPage): - @method - class iter_balances(DictElement): - def store(self, obj): - obj.id = f"balance-{len(self.objects)}" - self.objects[obj.id] = obj - return obj - - class item(ItemElement): - klass = Balance - - obj_amount = CleanDecimal.SI(Dict('balanceAmount/amount')) - obj_currency = Dict('balanceAmount/currency') - - obj_refDate = Date(Dict('referenceDate')) - obj_balType = Dict("balanceType") + def validate(self, obj): + return True class SubscriptionsPage(LoggedPage, JsonPage): -- GitLab From 49113b4c58e9dc5f3f23fb5e0cc908916ca29f88 Mon Sep 17 00:00:00 2001 From: Ludovic LANGE Date: Fri, 5 Apr 2024 11:49:24 +0200 Subject: [PATCH 7/7] fill IBAN when possible --- modules/ccf/browser.py | 52 +++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/modules/ccf/browser.py b/modules/ccf/browser.py index 3fc56a909..0965d4749 100644 --- a/modules/ccf/browser.py +++ b/modules/ccf/browser.py @@ -143,21 +143,42 @@ def download_document(self, document): params = {"flattenDoc": False} return self.open(document.url, params=params).content - def update_iban(self, account): - self.rib_details.go(json={"numeroContratSouscritCrypte": account._index}) - iban_number = self.page.get_iban() - if not account.iban: - account.iban = iban_number - - def get_ibans_from_ribs(self): + def _update_iban(self, account, ibans): + iban_obj = None + if account.id in ibans: + account.iban = ibans.get(account.id) + iban_obj = IBAN(account.iban) if account.iban else None + else: + _iban = account._iban_offended + _contract = account._contract_id + + if _iban: + iban_obj = IBAN.generate( + country_code=_iban[:2], + bank_code=_iban[4:9], + branch_code=_iban[9:14], + account_code=_contract[0]+_contract[4:14]) + + if iban_obj: + account.iban = iban_obj.formatted + account.number = iban_obj.account_code + account.bank_name = iban_obj.bank_name + if not account.bank_name: + account.bank_name = 'CCF' + + def _get_ibans_from_ribs(self): accounts_list = super().iter_accounts() for account in accounts_list: - account._original_id = account.id - self.update_iban(account) + self.rib_details.go(json={"numeroContratSouscritCrypte": account._index}) + iban_number = self.page.get_iban() + if not account.iban: + account.iban = iban_number return { account.id: account.iban for account in accounts_list } @need_login def iter_accounts(self): + ibans = self._get_ibans_from_ribs() + go_accounts = retry(ClientError, tries=5)(self.accounts_.go) go_accounts(params={'types': 'CHECKING,SAVING'}) @@ -168,7 +189,7 @@ def iter_accounts(self): date_to = (date.today() + relativedelta(months=1)).strftime('%Y-%m-%dT00:00:00.000Z') self.balances_comings.go(account_id=account.id, params={'dateTo': date_to}) self.page.fill_coming(account) - account.iban = mk_iban(account) + self._update_iban(account, ibans) yield account @need_login @@ -180,17 +201,6 @@ def iter_history(self, account): go_transactions(account_id=account._original_id) return self.page.iter_transactions() -def mk_iban(account): - _iban = account._iban_offended - _contract = account._contract_id - - return str(IBAN.generate( - country_code=_iban[:2], - bank_code=_iban[4:9], - branch_code=_iban[9:14], - account_code=_contract[0]+_contract[4:14]) - ) if _iban else None - class CCFParBrowser(CCFBrowser): BASEURL = "https://api.ccf.fr" -- GitLab