From 3f7044238829b4f8dc4c4d85f30a9a9105932fd9 Mon Sep 17 00:00:00 2001 From: Antti Jaakkola Date: Mon, 9 Jul 2018 07:05:35 +0300 Subject: [PATCH] Improve procountor import parsing --- membership/management/commands/csvbills.py | 27 ++++++++++++++++--- .../test_data/procountor-csv-empty-rows.csv | 6 +++++ .../test_data/procountor-csv-uknown-rows.csv | 5 ++++ membership/tests.py | 16 +++++++++++ 4 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 membership/test_data/procountor-csv-empty-rows.csv create mode 100644 membership/test_data/procountor-csv-uknown-rows.csv diff --git a/membership/management/commands/csvbills.py b/membership/management/commands/csvbills.py index ab8915ee..59cf38e0 100644 --- a/membership/management/commands/csvbills.py +++ b/membership/management/commands/csvbills.py @@ -105,10 +105,21 @@ def _get_row(self, row): """ return row + def _filter_row(self, row): + """ + Function for custom data filtering + """ + return False + def next(self): - row = self._get_row(UnicodeDictReader.next(self)) - if len(row) == 0: - return None + while True: + row = self._get_row(UnicodeDictReader.next(self)) + if len(row) == 0: + return None + # Skip over invalid rows + if self._filter_row(row): + continue + break row['amount'] = Decimal(row['amount'].replace(",", ".")) row['date'] = datetime.strptime(row['date'], "%d.%m.%Y") row['reference'] = row['reference'].replace(' ', '').lstrip('0') @@ -117,6 +128,7 @@ def next(self): row['value_date'] = datetime.strptime(row['value_date'], "%d.%m.%Y") return row + class OpDictReader(BillDictReader): '''Reader for Osuuspankki CSV file format @@ -172,6 +184,15 @@ def _get_row(self, row): row['reference'] = row['real_reference'] return row + def _filter_row(self, row): + if row['amount'] == "" or row['amount'] is None: + logger.warning("Skipping invalid row %s" % (row,)) + return True + if row['event_type_description'].lower() not in ['viitemaksu', 'tilisiirto']: + logger.warning("Skipping uknown event row %s" % (row,)) + return True + return False + def row_to_payment(row): try: diff --git a/membership/test_data/procountor-csv-empty-rows.csv b/membership/test_data/procountor-csv-empty-rows.csv new file mode 100644 index 00000000..9685c2de --- /dev/null +++ b/membership/test_data/procountor-csv-empty-rows.csv @@ -0,0 +1,6 @@ +Arvopäivä;Maksupäivä;Summa;Kirjausselite;Nimi;Tilinro;Viitenumero;Viesti;Kohdistettu;Arkistointitunnus +01.10.2014;01.10.2014;40,00;Viitemaksu;TEST PERSON;;2800001;;Myyntilasku 123456;1001553397672111 +;;;;;;;;; +30.09.2014;30.09.2014;40,00;Viitemaksu;MEIKÄLÄINEN MAIJA TERTTU;;703314010;;;0918593731241980 +;;;;;;;;; +25.09.2014;25.09.2014;40,00;Viitemaksu;MEIKÄLÄINEN TERO ANTERO;;1114018;;;0925593731AB1320 diff --git a/membership/test_data/procountor-csv-uknown-rows.csv b/membership/test_data/procountor-csv-uknown-rows.csv new file mode 100644 index 00000000..da767d1d --- /dev/null +++ b/membership/test_data/procountor-csv-uknown-rows.csv @@ -0,0 +1,5 @@ +Arvopäivä;Maksupäivä;Summa;Kirjausselite;Nimi;Tilinro;Viitenumero;Viesti;Kohdistettu;Arkistointitunnus +01.10.2014;01.10.2014;40,00;Viitemaksu;TEST PERSON;;2800001;;Myyntilasku 123456;1001553397672111 +30.09.2014;30.09.2014;40,00;Viitemaksu;MEIKÄLÄINEN MAIJA TERTTU;;703314010;;;0918593731241980 +30.09.2014;30.09.2014;40,00;Testimaksu;MEIKÄLÄINEN MAIJA TERTTU;;703314010;;;0918593731241980 +25.09.2014;25.09.2014;40,00;Viitemaksu;MEIKÄLÄINEN TERO ANTERO;;1114018;;;0925593731AB1320 diff --git a/membership/tests.py b/membership/tests.py index f6f2dfde..226f0dcb 100644 --- a/membership/tests.py +++ b/membership/tests.py @@ -1085,6 +1085,22 @@ def test_future_payment(self): with open_test_data("procountor-csv-future.csv") as f: self.assertRaises(PaymentFromFutureException, process_procountor_csv, f) + def test_empty_rows(self): + with open_test_data("procountor-csv-empty-rows.csv") as f: + process_procountor_csv(f) + no_cycle_q = Q(billingcycle=None) + payment_count = Payment.objects.filter(~no_cycle_q).count() + error = "The payment in the sample file should have matched" + self.assertEqual(payment_count, 1, error) + + def test_uknown_rows(self): + with open_test_data("procountor-csv-uknown-rows.csv") as f: + process_procountor_csv(f) + no_cycle_q = Q(billingcycle=None) + payment_count = Payment.objects.filter(~no_cycle_q).count() + error = "The payment in the sample file should have matched" + self.assertEqual(payment_count, 1, error) + def test_csv_header_processing(self): error = "Should fail on invalid header" with open_test_data("procountor-csv-invalid.csv") as f: