From 95657d8215467cddc7d0fdc98059f628d1b22657 Mon Sep 17 00:00:00 2001 From: Jeffrey Farley Date: Wed, 29 Nov 2017 14:32:44 -0500 Subject: [PATCH 1/6] feat: Provide API documentation This is in OpenAPI (Swagger) format --- complaint_search/swagger.yml | 580 +++++++++++++++++++++++++++++++++++ 1 file changed, 580 insertions(+) create mode 100644 complaint_search/swagger.yml diff --git a/complaint_search/swagger.yml b/complaint_search/swagger.yml new file mode 100644 index 00000000..29bf00fc --- /dev/null +++ b/complaint_search/swagger.yml @@ -0,0 +1,580 @@ +swagger: "2.0" +info: + version: "1.0.0" + title: "Consumer Complaint Database API" + description: "The API for searching the Consumer Complaint Database" + termsOfService: "https://cfpb.github.io/source-code-policy/" + contact: + email: "tech@consumerfinance.gov" + license: + name: "CC0" + url: "https://github.com/cfpb/ccdb5-api/blob/master/LICENSE" +host: "www.consumerfinance.gov" +basePath: "/data-research/consumer-complaints/search/api/v1/" +schemes: +- http +- https +produces: +- "application/json" +paths: + /: + get: + tags: + - Complaints + summary: "Search consumer complaints" + description: "Search through everything in consumer complaints" + produces: + - "application/json" + - "text/csv" + - "application/vnd.ms-excel" + - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + parameters: + - $ref: "#/parameters/search_term" + - $ref: "#/parameters/field" + - $ref: "#/parameters/from" + - $ref: "#/parameters/size" + - $ref: "#/parameters/sort" + - $ref: "#/parameters/format" + - $ref: "#/parameters/no_aggs" + - $ref: "#/parameters/no_highlight" + - $ref: "#/parameters/company" + - $ref: "#/parameters/company_public_response" + - $ref: "#/parameters/company_received_max" + - $ref: "#/parameters/company_received_min" + - $ref: "#/parameters/company_response" + - $ref: "#/parameters/consumer_consent_provided" + - $ref: "#/parameters/consumer_disputed" + - $ref: "#/parameters/date_received_max" + - $ref: "#/parameters/date_received_min" + - $ref: "#/parameters/has_narrative" + - $ref: "#/parameters/issue" + - $ref: "#/parameters/product" + - $ref: "#/parameters/state" + - $ref: "#/parameters/submitted_via" + - $ref: "#/parameters/tag" + responses: + 200: + description: "successful operation" + schema: + type: array + items: + $ref: '#/definitions/SearchResult' + 400: + description: "Invalid status value" + /_suggest: + get: + tags: + - Typeahead + summary: "Suggest possible searches" + description: "The endpoint for the main search box in the UI" + parameters: + - $ref: "#/parameters/size" + - $ref: "#/parameters/suggest_text" + responses: + 200: + description: "successful operation" + schema: + $ref: "#/definitions/SuggestResult" + 400: + description: "Invalid input" + /_suggest_company: + get: + tags: + - Typeahead + summary: "Suggest possible companies" + description: "Provide a list of companies that match the input text" + parameters: + - $ref: "#/parameters/suggest_text" + - $ref: "#/parameters/size" + - $ref: "#/parameters/company_public_response" + - $ref: "#/parameters/company_received_max" + - $ref: "#/parameters/company_received_min" + - $ref: "#/parameters/company_response" + - $ref: "#/parameters/consumer_consent_provided" + - $ref: "#/parameters/consumer_disputed" + - $ref: "#/parameters/date_received_max" + - $ref: "#/parameters/date_received_min" + - $ref: "#/parameters/has_narrative" + - $ref: "#/parameters/issue" + - $ref: "#/parameters/product" + - $ref: "#/parameters/state" + - $ref: "#/parameters/submitted_via" + - $ref: "#/parameters/tag" + responses: + 200: + description: "successful operation" + schema: + $ref: "#/definitions/SuggestResult" + 400: + description: "Invalid input" + /_suggest_zip: + get: + tags: + - Typeahead + summary: "Suggest possible zip codes" + description: "Provide a list of zip codes that match the input text" + parameters: + - $ref: "#/parameters/suggest_text" + - $ref: "#/parameters/size" + - $ref: "#/parameters/company_public_response" + - $ref: "#/parameters/company_received_max" + - $ref: "#/parameters/company_received_min" + - $ref: "#/parameters/company_response" + - $ref: "#/parameters/consumer_consent_provided" + - $ref: "#/parameters/consumer_disputed" + - $ref: "#/parameters/date_received_max" + - $ref: "#/parameters/date_received_min" + - $ref: "#/parameters/has_narrative" + - $ref: "#/parameters/issue" + - $ref: "#/parameters/product" + - $ref: "#/parameters/state" + - $ref: "#/parameters/submitted_via" + - $ref: "#/parameters/tag" + responses: + 200: + description: "successful operation" + schema: + $ref: "#/definitions/SuggestResult" + 400: + description: "Invalid input" + /{complaintId}: + get: + tags: + - Complaints + summary: "Find comsumer complaint by ID" + description: "Get complaint details for a specific ID" + parameters: + - name: complaintId + in: path + description: ID of the complaint + required: true + type: integer + maximum: 9999999999 + minimum: 0 + format: int64 + responses: + 200: + description: "successful operation" + schema: + $ref: "#/definitions/Complaint" + 400: + description: "Invalid ID supplied" + 404: + description: "Complaint not found" + /docs: + get: + summary: "Returns this document" + description: "Produces the OpenAPI specification for the API" + produces: + - "text/yaml" + responses: + 200: + description: "successful operation" +definitions: + Aggregation: + type: object + description: "An Elasticsearch aggregation" + properties: + doc_count: + type: integer + description: "The total number of complaints covered in this aggregation" + "": + type: object + description: "The name of the field being aggregated" + properties: + buckets: + type: array + items: + $ref: "#/definitions/Bucket" + doc_count_error_upper_bound: + type: integer + description: "The number of possible errors that occurred when searching the shards" + sum_other_doc_count: + type: integer + description: "The number of complaints that were not included in this aggregation." + Bucket: + type: object + properties: + doc_count: + type: integer + description: "The number of complaints that match this key" + key: + type: string + Complaint: + type: object + externalDocs: + description: "Official documentation" + url: "https://cfpb.github.io/api/ccdb/fields.html" + properties: + company: + type: string + description: "The complaint is about this company" + company_public_response: + type: string + description: "The company's optional, public-facing response to a consumer's complaint" + company_response: + type: string + description: "The response from the company about this complaint" + complaint_id: + type: integer + description: "The unique identification number for a complaint" + complaint_what_happened: + type: string + description: "A description of the complaint provided by the consumer" + consumer_consent_provided: + type: string + description: "dentifies whether the consumer opted in to publish their complaint narrative" + consumer_disputed: + type: string + description: "Whether the consumer disputed the company’s response" + date_received: + type: string + format: date + description: "The date the CFPB received the complaint" + date_sent_to_company: + type: string + description: "The date the CFPB sent the complaint to the company" + has_narrative: + type: boolean + description: "Indicates this complaint has a narrative" + issue: + type: string + description: "The issue the consumer identified in the complaint" + product: + type: string + description: "The type of product the consumer identified in the complaint" + state: + type: string + description: "The state of the mailing address provided by the consumer" + sub_issue: + type: string + description: "The sub-issue the consumer identified in the complaint" + sub_product: + type: string + description: "The type of sub-product the consumer identified in the complaint" + submitted_via: + type: string + description: "How the complaint was submitted to the CFPB" + tags: + type: string + description: "Data that supports easier searching and sorting of complaints" + timely: + type: string + description: "Indicates whether the company gave a timely response or not" + zip_code: + type: string + description: "The mailing ZIP code provided by the consumer" + Hit: + type: object + description: "A single Elasticsearch result" + properties: + _source: + $ref: "#/definitions/Complaint" + Hits: + type: object + description: "A set of complaints that matched the query" + properties: + hits: + type: array + items: + $ref: "#/definitions/Hit" + max_score: + type: number + description: "The highest score in the results" + format: float + total: + type: integer + description: "The totol number of complaints that matched the query" + Meta: + type: object + properties: + has_data_issue: + type: boolean + description: "Indicates there has been an issue with the most recent data load" + is_data_stale: + type: boolean + description: "Indicates the most recent data is over 5 business days old" + is_narrative_stale: + type: boolean + description: "Indicates the most recent narratives are over 5 busines days old" + last_indexed: + type: string + description: "The timestamp of the most recently indexed complaint" + format: dateTime + last_updated: + type: string + description: "The timestamp of the most recent complaint" + format: dateTime + license: + type: string + description: "The open source license under which the API operates" + total_record_count: + type: integer + description: "The total number of complaints currently indexed" + MultiLevelAggregation: + properties: + doc_count: + type: integer + description: "The total number of complaints covered in this aggregation" + "": + type: object + description: "The name of the field being aggregated" + properties: + buckets: + type: array + items: + $ref: "#/definitions/MultiLevelBucket" + doc_count_error_upper_bound: + type: integer + description: "The number of possible errors that occurred when searching the shards" + sum_other_doc_count: + type: integer + description: "The number of complaints that were not included in this aggregation." + MultiLevelBucket: + type: object + properties: + doc_count: + type: integer + description: "The number of complaints that match this key" + "": + type: object + description: "The next level of aggregations" + properties: + buckets: + type: array + items: + $ref: "#/definitions/Aggregation" + key: + type: string + SearchResult: + type: object + properties: + _meta: + $ref: "#/definitions/Meta" + aggregations: + type: object + properties: + company_public_response: + $ref: "#/definitions/Aggregation" + company_response: + $ref: "#/definitions/Aggregation" + consumer_consent_provided: + $ref: "#/definitions/Aggregation" + consumer_disputed: + $ref: "#/definitions/Aggregation" + has_narrative: + $ref: "#/definitions/Aggregation" + issue: + $ref: "#/definitions/MultiLevelAggregation" + product: + $ref: "#/definitions/MultiLevelAggregation" + state: + $ref: "#/definitions/Aggregation" + submitted_via: + $ref: "#/definitions/Aggregation" + tags: + $ref: "#/definitions/Aggregation" + timely: + $ref: "#/definitions/Aggregation" + zip_code: + $ref: "#/definitions/Aggregation" + hits: + $ref: "#/definitions/Hits" + SuggestResult: + type: array + items: + type: string +parameters: + company: + name: company + in: query + description: Filter the results to only return these companies + type: array + items: + type: string + collectionFormat: multi + company_public_response: + name: company_public_response + in: query + description: Filter the results to only return these types of public response by the company + type: array + items: + type: string + collectionFormat: multi + company_received_max: + name: company_received_max + in: query + description: Return results with date < company_received_max (i.e. 2017-03-04) + type: string + format: date + company_received_min: + name: company_received_min + in: query + description: Return results with date >= company_received_min (i.e. 2017-03-04) + type: string + format: date + company_response: + name: company_response + in: query + description: Filter the results to only return these types of response by the company + type: array + items: + type: string + collectionFormat: multi + consumer_consent_provided: + name: consumer_consent_provided + in: query + description: Filter the results to only return these types of consent consumer provided + type: array + items: + type: string + collectionFormat: multi + consumer_disputed: + name: consumer_disputed + in: query + description: Filter the results to only return the specified state of consumer disputed, i.e. yes, no + type: array + items: + type: string + collectionFormat: multi + date_received_max: + name: date_received_max + in: query + description: Return results with date < date_received_max (i.e. 2017-03-04) + type: string + format: date + date_received_min: + name: date_received_min + in: query + description: Return results with date >= date_received_min (i.e. 2017-03-04) + type: string + format: date + field: + name: "field" + in: "query" + description: "Search by particular field" + type: string + enum: + - "complaint_what_happened" + - "company_public_response" + - "all" + default: "complaint_what_happened" + format: + name: "format" + in: "query" + description: "Format to be returned, if this parameter is not specified, frm/size parameters can be used properly, but if a format is specified for exporting, frm/size will be ignored" + type: "string" + enum: + - "json" + - "csv" + - "xls" + - "xlsx" + default: "json" + from: + name: frm + in: "query" + description: "Return results starting from a specific index, only if format parameter is not specified, ignore otherwise" + type: "integer" + maximum: 100000 + minimum: 1 + format: "int64" + default: 0 + has_narrative: + name: has_narrative + in: query + description: Filter the results to only return the specified state of whether it has narrative in the complaint or not, i.e. yes, no + type: array + items: + type: string + collectionFormat: multi + issue: + name: issue + in: query + description: "Filter the results to only return these types of issue and subissue, i.e. product-only: Getting a Loan, subproduct needs to include product, separated by '•', U+2022: Getting a Loan•Can't qualify for a loan" + type: array + items: + type: string + collectionFormat: multi + no_aggs: + name: no_aggs + in: query + description: Include aggregations in result or not, True means no aggregations will be included, False means aggregations will be included. + type: boolean + default: False + no_highlight: + name: no_highlight + in: query + description: Include highlight of search term in result or not, True means no highlighting will be included, False means highlighting will be included. + type: boolean + default: False + product: + name: product + in: query + description: "Filter the results to only return these types of product and subproduct, i.e. product-only: Mortgage, subproduct needs to include product, separated by '•', U+2022: Mortgage•FHA mortgage" + type: array + items: + type: string + collectionFormat: multi + search_term: + name: "search_term" + in: "query" + description: "Return results containing specific term" + type: "string" + size: + name: "size" + in: "query" + description: "Limit the size of the results" + type: "integer" + maximum: 100000 + minimum: 1 + format: "int64" + default: 10 + sort: + name: "sort" + in: "query" + description: "Return results sort in a particular order" + type: "string" + enum: + - "relevance_desc" + - "relevance_asc" + - "created_date_desc" + - "created_date_asc" + default: "relevance_desc" + state: + name: state + in: query + description: Filter the results to only return these states (use abbreviation, i.e. CA, VA) + type: array + items: + type: string + collectionFormat: multi + submitted_via: + name: submitted_via + in: query + description: Filter the results to only return these types of way consumers submitted their complaints + type: array + items: + type: string + collectionFormat: multi + suggest_text: + name: text + in: query + description: "text to use for suggestions" + required: true + type: string + tag: + name: tag + in: query + description: Filter the results to only return these types of tag + type: array + items: + type: string + collectionFormat: multi +tags: + - name: Complaints + description: These endpoints provide access to consumer complaints + - name: Typeahead + description: These endpoints support the typeahead boxes in the UI +externalDocs: + description: "Full documentation for the API" + url: "https://cfpb.github.io/api/ccdb/" \ No newline at end of file From 01183f3e1521d4653d8bf38bfa0613234a9a690c Mon Sep 17 00:00:00 2001 From: Jeffrey Farley Date: Wed, 29 Nov 2017 14:36:01 -0500 Subject: [PATCH 2/6] feat: Provide endpoint to serve API documentation --- complaint_search/urls.py | 3 ++- complaint_search/views.py | 25 ++++++++++++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/complaint_search/urls.py b/complaint_search/urls.py index 7b6a8794..1d68d538 100644 --- a/complaint_search/urls.py +++ b/complaint_search/urls.py @@ -13,6 +13,7 @@ name="suggest_zip" ), url(r'^_suggest', complaint_search.views.suggest, name="suggest"), - url(r'^(?P[0-9]+)$', complaint_search.views.document, name="document"), + url(r'^docs', complaint_search.views.swagger, name="swagger"), + url(r'^(?P[0-9]+)$', complaint_search.views.document, name="complaint"), url(r'^$', complaint_search.views.search, name="search"), ] diff --git a/complaint_search/views.py b/complaint_search/views.py index fae470eb..fa3e8253 100644 --- a/complaint_search/views.py +++ b/complaint_search/views.py @@ -156,6 +156,25 @@ def search(request): return response +@api_view(['GET']) +def swagger(request): + from django.http import FileResponse + import os + + swagger_yml_path = os.path.join( + settings.BASE_DIR, + 'complaint_search', + 'swagger.yml', + ) + + swagger_yml = open(swagger_yml_path, 'rb') + + return FileResponse( + swagger_yml, + content_type='text/yaml' + ) + + @api_view(['GET']) @catch_es_error def suggest(request): @@ -195,7 +214,7 @@ def suggest_zip(request): @api_view(['GET']) @catch_es_error def suggest_company(request): - + # Key removal that takes mutation into account in case of other reference def removekey(d, key): r = dict(d) @@ -206,14 +225,14 @@ def removekey(d, key): validVars.append('text') data = _parse_query_params(request.query_params, validVars) - + # Company filters should not be applied to their own aggregation filter if 'company' in data: data = removekey(data, 'company') if data.get('text'): data['text'] = data['text'].upper() - + return _suggest_field(data, 'company.suggest', 'company.raw') From 5073f6150e8fa98c8b12b215724672ef6415a7d7 Mon Sep 17 00:00:00 2001 From: Jeffrey Farley Date: Wed, 29 Nov 2017 14:38:26 -0500 Subject: [PATCH 3/6] test: Fix document endpoint tests These broke when the endpoint's name changed --- complaint_search/tests/test_views_document.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/complaint_search/tests/test_views_document.py b/complaint_search/tests/test_views_document.py index 56649a96..fe5e2b19 100644 --- a/complaint_search/tests/test_views_document.py +++ b/complaint_search/tests/test_views_document.py @@ -27,7 +27,7 @@ def test_document__valid(self, mock_esdocument): """ documenting with an ID """ - url = reverse('complaint_search:document', kwargs={"id": "123456"}) + url = reverse('complaint_search:complaint', kwargs={"id": "123456"}) mock_esdocument.return_value = 'OK' response = self.client.get(url) self.assertEqual(response.status_code, status.HTTP_200_OK) @@ -37,7 +37,7 @@ def test_document__valid(self, mock_esdocument): @mock.patch('complaint_search.es_interface.document') def test_document_with_document_anon_rate_throttle(self, mock_esdocument): - url = reverse('complaint_search:document', kwargs={"id": "123456"}) + url = reverse('complaint_search:complaint', kwargs={"id": "123456"}) mock_esdocument.return_value = 'OK' DocumentAnonRateThrottle.rate = self.orig_document_anon_rate limit = int(self.orig_document_anon_rate.split('/')[0]) @@ -55,7 +55,7 @@ def test_document_with_document_anon_rate_throttle(self, mock_esdocument): @mock.patch('complaint_search.es_interface.document') def test_document_with_document_ui_rate_throttle(self, mock_esdocument): - url = reverse('complaint_search:document', kwargs={"id": "123456"}) + url = reverse('complaint_search:complaint', kwargs={"id": "123456"}) mock_esdocument.return_value = 'OK' DocumentAnonRateThrottle.rate = self.orig_document_anon_rate @@ -74,7 +74,7 @@ def test_document_with_document_ui_rate_throttle(self, mock_esdocument): @mock.patch('complaint_search.es_interface.document') def test_document__transport_error_with_status_code(self, mock_esdocument): mock_esdocument.side_effect = TransportError(status.HTTP_404_NOT_FOUND, "Error") - url = reverse('complaint_search:document', kwargs={"id": "123456"}) + url = reverse('complaint_search:complaint', kwargs={"id": "123456"}) response = self.client.get(url) self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) self.assertDictEqual({"error": "Elasticsearch error: Error"}, response.data) @@ -82,7 +82,7 @@ def test_document__transport_error_with_status_code(self, mock_esdocument): @mock.patch('complaint_search.es_interface.document') def test_document__transport_error_without_status_code(self, mock_esdocument): mock_esdocument.side_effect = TransportError('N/A', "Error") - url = reverse('complaint_search:document', kwargs={"id": "123456"}) + url = reverse('complaint_search:complaint', kwargs={"id": "123456"}) response = self.client.get(url) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertDictEqual({"error": "Elasticsearch error: Error"}, response.data) From a99c8744c38be19967b9c4e1621238eee82202ae Mon Sep 17 00:00:00 2001 From: Jeffrey Farley Date: Wed, 29 Nov 2017 15:08:27 -0500 Subject: [PATCH 4/6] test: Provide tests for `docs` endpoint --- complaint_search/tests/test_views_swagger.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 complaint_search/tests/test_views_swagger.py diff --git a/complaint_search/tests/test_views_swagger.py b/complaint_search/tests/test_views_swagger.py new file mode 100644 index 00000000..516bd741 --- /dev/null +++ b/complaint_search/tests/test_views_swagger.py @@ -0,0 +1,15 @@ +from django.core.urlresolvers import reverse +from rest_framework import status +from rest_framework.test import APITestCase +import io + + +class SwaggerTests(APITestCase): + def test_provide(self): + url = reverse('complaint_search:swagger') + + response = self.client.get(url) + data = ''.join([l for l in response.streaming_content]) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual('swagger', data[:7]) From d9e83920f5509f9dc14311d2f34abd19e06f4896 Mon Sep 17 00:00:00 2001 From: Jeffrey Farley Date: Wed, 29 Nov 2017 15:18:19 -0500 Subject: [PATCH 5/6] fix: Determine the directory a different way The previous version did not work when included in CFGOV --- complaint_search/views.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/complaint_search/views.py b/complaint_search/views.py index fa3e8253..02425b32 100644 --- a/complaint_search/views.py +++ b/complaint_search/views.py @@ -161,9 +161,10 @@ def swagger(request): from django.http import FileResponse import os + thisDir = os.path.dirname(os.path.abspath(__file__)) + swagger_yml_path = os.path.join( - settings.BASE_DIR, - 'complaint_search', + thisDir, 'swagger.yml', ) From 82205d85e8eb97449bd08549ab1bfdb299164b96 Mon Sep 17 00:00:00 2001 From: Jeffrey Farley Date: Mon, 4 Dec 2017 14:01:17 -0500 Subject: [PATCH 6/6] fix: Update the description of the search endpoint --- complaint_search/swagger.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/complaint_search/swagger.yml b/complaint_search/swagger.yml index 29bf00fc..558dea98 100644 --- a/complaint_search/swagger.yml +++ b/complaint_search/swagger.yml @@ -22,7 +22,7 @@ paths: tags: - Complaints summary: "Search consumer complaints" - description: "Search through everything in consumer complaints" + description: "Search the contents of the consumer complaint database" produces: - "application/json" - "text/csv"