Improve procountor import parsing

membership/management/commands/csvbills.py

@@ -105,10 +105,21 @@ def _get_row(self, row):
         """
         return row
 
+    def _filter_row(self, row):
+        """
+        Function for custom data filtering
+        """
+        return False
+
     def next(self):
-        row = self._get_row(UnicodeDictReader.next(self))
-        if len(row) == 0:
-            return None
+        while True:
+            row = self._get_row(UnicodeDictReader.next(self))
+            if len(row) == 0:
+                return None
+            # Skip over invalid rows
+            if self._filter_row(row):
+                continue
+            break
         row['amount'] = Decimal(row['amount'].replace(",", "."))
         row['date'] = datetime.strptime(row['date'], "%d.%m.%Y")
         row['reference'] = row['reference'].replace(' ', '').lstrip('0')
@@ -117,6 +128,7 @@ def next(self):
             row['value_date'] = datetime.strptime(row['value_date'], "%d.%m.%Y")
         return row
 
+
 class OpDictReader(BillDictReader):
     '''Reader for Osuuspankki CSV file format
 
@@ -172,6 +184,15 @@ def _get_row(self, row):
             row['reference'] = row['real_reference']
         return row
 
+    def _filter_row(self, row):
+        if row['amount'] == "" or row['amount'] is None:
+            logger.warning("Skipping invalid row %s" % (row,))
+            return True
+        if row['event_type_description'].lower() not in ['viitemaksu', 'tilisiirto']:
+            logger.warning("Skipping unknown event row %s" % (row,))
+            return True
+        return False
+
 
 def row_to_payment(row):
     try:

membership/test_data/procountor-csv-empty-rows.csv

@@ -0,0 +1,6 @@
+Arvop�iv�;Maksup�iv�;Summa;Kirjausselite;Nimi;Tilinro;Viitenumero;Viesti;Kohdistettu;Arkistointitunnus
+01.10.2014;01.10.2014;40,00;Viitemaksu;TEST PERSON;;2800001;;Myyntilasku 123456;1001553397672111
+;;;;;;;;;
+30.09.2014;30.09.2014;40,00;Viitemaksu;MEIK�L�INEN MAIJA TERTTU;;703314010;;;0918593731241980
+;;;;;;;;;
+25.09.2014;25.09.2014;40,00;Viitemaksu;MEIK�L�INEN TERO ANTERO;;1114018;;;0925593731AB1320

membership/test_data/procountor-csv-uknown-rows.csv

@@ -0,0 +1,5 @@
+Arvop�iv�;Maksup�iv�;Summa;Kirjausselite;Nimi;Tilinro;Viitenumero;Viesti;Kohdistettu;Arkistointitunnus
+01.10.2014;01.10.2014;40,00;Viitemaksu;TEST PERSON;;2800001;;Myyntilasku 123456;1001553397672111
+30.09.2014;30.09.2014;40,00;Viitemaksu;MEIK�L�INEN MAIJA TERTTU;;703314010;;;0918593731241980
+30.09.2014;30.09.2014;40,00;Testimaksu;MEIK�L�INEN MAIJA TERTTU;;703314010;;;0918593731241980
+25.09.2014;25.09.2014;40,00;Viitemaksu;MEIK�L�INEN TERO ANTERO;;1114018;;;0925593731AB1320

membership/tests.py

@@ -1085,6 +1085,22 @@ def test_future_payment(self):
         with open_test_data("procountor-csv-future.csv") as f:
             self.assertRaises(PaymentFromFutureException, process_procountor_csv, f)
 
+    def test_empty_rows(self):
+        with open_test_data("procountor-csv-empty-rows.csv") as f:
+            process_procountor_csv(f)
+        no_cycle_q = Q(billingcycle=None)
+        payment_count = Payment.objects.filter(~no_cycle_q).count()
+        error = "The payment in the sample file should have matched"
+        self.assertEqual(payment_count, 1, error)
+
+    def test_uknown_rows(self):
+        with open_test_data("procountor-csv-uknown-rows.csv") as f:
+            process_procountor_csv(f)
+        no_cycle_q = Q(billingcycle=None)
+        payment_count = Payment.objects.filter(~no_cycle_q).count()
+        error = "The payment in the sample file should have matched"
+        self.assertEqual(payment_count, 1, error)
+
     def test_csv_header_processing(self):
         error = "Should fail on invalid header"
         with open_test_data("procountor-csv-invalid.csv") as f: