Merge pull request #94 from chosak/unicode-csv-exports

Properly support Unicode in CSV exports

complaint_search/export.py

@@ -1,3 +1,4 @@
+import six
 from six import text_type
 from six.moves import cStringIO as StringIO
 
@@ -7,6 +8,12 @@
 from django.http import StreamingHttpResponse
 
 
+if six.PY2:  # pragma: no cover
+    from unicodecsv import DictWriter
+else:  # pragma: no cover
+    from csv import DictWriter
+
+
 class ElasticSearchExporter(object):
 
     # export_csv - Stream an Elsticsearch response as a CSV file
@@ -28,8 +35,8 @@ def read_and_flush(writer, buffer_, row):
 
         def stream():
             buffer_ = StringIO()
-            writer = csv.DictWriter(buffer_, header_dict.keys(),
-                                    delimiter=",", quoting=csv.QUOTE_MINIMAL)
+            writer = DictWriter(buffer_, header_dict.keys(),
+                                delimiter=",", quoting=csv.QUOTE_MINIMAL)
 
             # Write Header Row
             data = read_and_flush(writer, buffer_, header_dict)

complaint_search/stream_content.py

@@ -1,3 +1,6 @@
+import six
+
+
 class StreamCSVContent(object):
 
     def __init__(self, header, content):
@@ -15,8 +18,9 @@ def __next__(self):
         else:
             return next(self.content)
 
-    def next(self):
-        return self.__next__()
+    if six.PY2:  # pragma: no cover
+        def next(self):
+            return self.__next__()
 
 
 class StreamJSONContent(object):
@@ -83,5 +87,6 @@ def __next__(self):
                 else:
                     raise StopIteration
 
-    def next(self):
-        return self.__next__()
+    if six.PY2:  # pragma: no cover
+        def next(self):
+            return self.__next__()

complaint_search/tests/test_export.py

@@ -80,3 +80,22 @@ def test_export_json_request_response(self, length):
         self.assertTrue('map' in str(type(res.streaming_content)))
         downloaded_file = io.BytesIO(b"".join(res.streaming_content))
         self.assertFalse(downloaded_file is None)
+
+
+class TestCSVExportWithUnicodeCharacters(TestCase):
+    def test_export_contains_unicode_chacter(self):
+        headers = OrderedDict([
+            ('key', 'Key'),
+        ])
+
+        def unicode_results():
+            yield {
+                '_source': {
+                    'key': u'\u2019',
+                },
+            }
+
+        exporter = ElasticSearchExporter()
+        response = exporter.export_csv(unicode_results(), headers)
+        content = io.BytesIO(b"".join(response.streaming_content)).read()
+        self.assertEqual(content, b'Key\r\n\xe2\x80\x99\r\n')

setup.py

@@ -17,6 +17,7 @@
     'elasticsearch>=2.4.1,<3',
     'django-localflavor>=1.1,<2',
     'django-flags>=4.0.1,<5',
+    'unicodecsv>=0.14.1,<1',
 ]
 
 testing_extras = [