From fce81ca7d1cbda9f743df8b1a555d6ce074e42a5 Mon Sep 17 00:00:00 2001 From: charlieDurnsfordHollands Date: Wed, 9 Aug 2023 09:43:44 +0100 Subject: [PATCH 01/11] generate wordcloud from ticket summaries --- word_cloud_generator/word_cloud_generator.py | 164 +++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 word_cloud_generator/word_cloud_generator.py diff --git a/word_cloud_generator/word_cloud_generator.py b/word_cloud_generator/word_cloud_generator.py new file mode 100644 index 00000000..9a06e521 --- /dev/null +++ b/word_cloud_generator/word_cloud_generator.py @@ -0,0 +1,164 @@ +from argparse import ArgumentParser, RawDescriptionHelpFormatter +from datetime import datetime +from dateutil.relativedelta import relativedelta +from pathlib import Path +from sys import argv +from time import sleep +from os import path +from wordcloud import WordCloud + +import json +import matplotlib.pyplot as pyplot +import requests +import re + + +def parse_args(inp_args): + """ + Function to parse commandline args + :param inp_args: a set of commandline args to parse (dict) + :returns: A dictionary of parsed args + """ + # Get arguments passed to the script + parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter) + + parser.add_argument( + "-u", + "--username", + metavar="USERNAME", + help="FedID of the user", + required=True, + ) + parser.add_argument( + "-p", + "--password", + metavar="PASSWORD", + help="Password of the user", + required=True, + ) + parser.add_argument( + "-o", + "--output", + metavar="OUTPUT", + help="Directory to create the output files in", + default="output", + ) + parser.add_argument( + "-d", + "--date", + metavar="Date", + help="Date to get issues since", + default=datetime.now() - relativedelta(months=1), + ) + args = parser.parse_args(inp_args) + return args + + +def get_response_json(auth, headers, url): + """ + Function to send a get request to a url and return the response as json + :param auth: A HTTPBasicAuth object for authentication (HTTPBasicAuth) + :param headers: A request Header (dict) + :param url: The URL to send the request (string) + :returns: A dictionary of JSON values + """ + session = requests.session() + session.headers = headers + session.auth = auth + + attempts = 5 + + while attempts > 0: + response = session.get(url, timeout=5) + if ( + response.content != b'{"status":"RUNNING"}' + and response.content != b'{"status":"ENQUEUED"}' + ): + break + else: + sleep(1) + attempts = attempts - 1 + + if attempts == 0: + raise requests.exceptions.Timeout( + "Get request status not completed before timeout" + ) + + return json.loads(response.text) + + +def get_issues_contents_after_time(auth, headers, host, date): + """ + Function to get the number of issues using a loop, as only 50 can be checked at a time + :param date: Time to get issues since (datetime.datetime) + :param auth: A HTTPBasicAuth object for authentication (HTTPBasicAuth) + :param headers: A request Header (dict) + :param host: The host used to create the URL to send the request (string) + :returns: A list with only a string containing the number of issues + """ + issues_contents = [] + return_amount = 50 + issues_amount = 0 + while return_amount == 50: + url = f"{host}/rest/servicedeskapi/servicedesk/6/queue/182/issue?start={issues_amount}" + + json_load = get_response_json(auth, headers, url) + + issues = json_load.get("values") + + for issue in issues: + issue_date = datetime.strptime(issue.get("fields").get("created")[:10], "%Y-%m-%d") + if issue_date < date: + return issues_contents + issue_contents = issue.get("fields").get("summary") + if issue_contents: + issues_contents.append(issue_contents) + + return_amount = json_load.get("size") + issues_amount = issues_amount + return_amount + + return issues_contents + + +def generate_word_cloud(issues_contents): + matches = re.findall(r"((\w+([.'](?![ \n']))*[-_]*)+)", issues_contents) + issues_contents = " ".join(list(list(zip(*matches))[0])) + word_cloud = WordCloud( + width=2000, + height=1000, + min_font_size=25, + max_words=10000, + background_color="white", + collocations=False, + regexp=r"\w*\S*", + ).generate(issues_contents) + + pyplot.imshow(word_cloud, interpolation="bilinear") + pyplot.axis("off") + pyplot.show() + + +def word_cloud_generator(): + args = parse_args(argv[1:]) + host = "https://stfc.atlassian.net" + username = args.username + password = args.password + output_location = args.output + date = args.date + + Path(output_location).mkdir(exist_ok=True) + + word_cloud_output_location = path.join(output_location, "JSM Metric Data.csv") + + auth = requests.auth.HTTPBasicAuth(username, password) + headers = { + "Accept": "application/json", + } + + issues_contents = get_issues_contents_after_time(auth, headers, host, date) + + generate_word_cloud(" ".join(issues_contents)) + + +if __name__ == "__main__": + word_cloud_generator() From 690b7bdb7a48dc3f726aefb7169003bd84c04ca9 Mon Sep 17 00:00:00 2001 From: charlieDurnsfordHollands Date: Thu, 10 Aug 2023 09:37:59 +0100 Subject: [PATCH 02/11] Add filters --- word_cloud_generator/word_cloud_generator.py | 88 +++++++++++++++----- 1 file changed, 67 insertions(+), 21 deletions(-) diff --git a/word_cloud_generator/word_cloud_generator.py b/word_cloud_generator/word_cloud_generator.py index 9a06e521..a5e62af5 100644 --- a/word_cloud_generator/word_cloud_generator.py +++ b/word_cloud_generator/word_cloud_generator.py @@ -44,11 +44,36 @@ def parse_args(inp_args): default="output", ) parser.add_argument( - "-d", - "--date", - metavar="Date", - help="Date to get issues since", - default=datetime.now() - relativedelta(months=1), + "-s", + "--start_date", + metavar="START_DATE", + help="Date to get issues from", + default=datetime.now().strftime("%Y-%m-%d"), + ) + parser.add_argument( + "-e", + "--end_date", + metavar="END_DATE", + help="Date to get issues to", + default=(datetime.now() - relativedelta(months=1)).strftime("%Y-%m-%d"), + ) + parser.add_argument( + "-a", + "--assigned", + metavar="ASSIGNED", + help="Assigned user to get tickets from", + ) + parser.add_argument( + "-f", + "--filter_for", + metavar="FILTER_FOR", + help="Strings to filter the word cloud for", + ) + parser.add_argument( + "-n", + "--filter_not", + metavar="FILTER_NOT", + help="Strings to filter the word cloud to not have", ) args = parser.parse_args(inp_args) return args @@ -87,14 +112,14 @@ def get_response_json(auth, headers, url): return json.loads(response.text) -def get_issues_contents_after_time(auth, headers, host, date): +def get_issues_contents_after_time(auth, headers, host, issue_filter): """ - Function to get the number of issues using a loop, as only 50 can be checked at a time - :param date: Time to get issues since (datetime.datetime) + Function to get the contents of through issues using a loop, as only 50 can be checked at a time + :param issue_filter: Dict of filters to check the issues against (dict) :param auth: A HTTPBasicAuth object for authentication (HTTPBasicAuth) :param headers: A request Header (dict) :param host: The host used to create the URL to send the request (string) - :returns: A list with only a string containing the number of issues + :returns: A list with the contents of all valid issues """ issues_contents = [] return_amount = 50 @@ -108,11 +133,12 @@ def get_issues_contents_after_time(auth, headers, host, date): for issue in issues: issue_date = datetime.strptime(issue.get("fields").get("created")[:10], "%Y-%m-%d") - if issue_date < date: + if issue_date < datetime.strptime(issue_filter.get("end_date"), "%Y-%m-%d"): return issues_contents - issue_contents = issue.get("fields").get("summary") - if issue_contents: - issues_contents.append(issue_contents) + if filter_issue(issue, issue_filter, issue_date): + issue_contents = issue.get("fields").get("summary") + if issue_contents: + issues_contents.append(issue_contents) return_amount = json_load.get("size") issues_amount = issues_amount + return_amount @@ -120,9 +146,27 @@ def get_issues_contents_after_time(auth, headers, host, date): return issues_contents -def generate_word_cloud(issues_contents): +def filter_issue(issue, issue_filter, issue_date): + issue_assigned = issue.get("fields").get("assignee") + if issue_date > datetime.strptime(issue_filter.get("start_date"), "%Y-%m-%d"): + return False + if issue.get("assigned") and issue_assigned != issue_assigned.get("assigned"): + return False + return True + + +def generate_word_cloud(issues_contents, issue_filter): matches = re.findall(r"((\w+([.'](?![ \n']))*[-_]*)+)", issues_contents) - issues_contents = " ".join(list(list(zip(*matches))[0])) + if matches: + issues_contents = " ".join(list(list(zip(*matches))[0])) + if issue_filter.get("filter_not"): + issues_contents = re.sub(issue_filter.get("filter_not").lower(), "", issues_contents, flags=re.I) + if issue_filter.get("filter_for"): + issues_contents = " ".join(re.findall( + issue_filter.get("filter_for").lower(), + issues_contents, + flags=re.IGNORECASE + )) word_cloud = WordCloud( width=2000, height=1000, @@ -143,21 +187,23 @@ def word_cloud_generator(): host = "https://stfc.atlassian.net" username = args.username password = args.password - output_location = args.output - date = args.date + issue_filter = {} + for arg in args.__dict__: + if args.__dict__[arg] is not None and arg != "username" and arg != "password": + issue_filter.update({arg: args.__dict__[arg]}) - Path(output_location).mkdir(exist_ok=True) + Path(issue_filter.get("output")).mkdir(exist_ok=True) - word_cloud_output_location = path.join(output_location, "JSM Metric Data.csv") + word_cloud_output_location = path.join(issue_filter["output"], "JSM Metric Data.csv") auth = requests.auth.HTTPBasicAuth(username, password) headers = { "Accept": "application/json", } - issues_contents = get_issues_contents_after_time(auth, headers, host, date) + issues_contents = get_issues_contents_after_time(auth, headers, host, issue_filter) - generate_word_cloud(" ".join(issues_contents)) + generate_word_cloud(" ".join(issues_contents), issue_filter) if __name__ == "__main__": From 9d7e603283568a08fe188ea589d2a24894a32139 Mon Sep 17 00:00:00 2001 From: charlieDurnsfordHollands Date: Tue, 15 Aug 2023 15:56:12 +0100 Subject: [PATCH 03/11] added docstrings, tests and made changes to filters --- .../tests/test_word_cloud_generator.py | 191 ++++++++++++++++++ word_cloud_generator/word_cloud_generator.py | 72 +++++-- 2 files changed, 244 insertions(+), 19 deletions(-) create mode 100644 word_cloud_generator/tests/test_word_cloud_generator.py diff --git a/word_cloud_generator/tests/test_word_cloud_generator.py b/word_cloud_generator/tests/test_word_cloud_generator.py new file mode 100644 index 00000000..9639cc81 --- /dev/null +++ b/word_cloud_generator/tests/test_word_cloud_generator.py @@ -0,0 +1,191 @@ +from unittest import mock +from unittest.mock import MagicMock, patch +from parameterized import parameterized +from datetime import datetime + +import requests +import requests.auth +import word_cloud_generator +import unittest + + +class ChangingJson: + """ + Class to represent a json object which changes value when it's called. + """ + + def __init__(self, values): + """ + Constructs the attributes for the ChangingJson object + :param values: The values for the ChangingJson to change through (list) + """ + self.values = values + self.current_index = 0 + + def get(self, get_value): + """ + Function to emulate the Json "Get" function while cycling through the values + :param get_value: The value to requested (any) + :return: The next value currently stored in the list (any) + """ + return_value = self.values[self.current_index].get(get_value) + if get_value == "size": + self.current_index = (self.current_index + 1) % len(self.values) + return return_value + + +auth = requests.auth.HTTPBasicAuth("test_username", "test_password") +headers = { + "Accept": "application/json", +} +host = "https://test.com" + + +class WorldCloudGeneratorTests(unittest.TestCase): + """ + Class for the tests to be run against the functions from word_cloud_generator.py + """ + + @parameterized.expand( + [ + ("check found", "something-else", True), + ("check not found", b'{"status":"RUNNING"}', False), + ] + ) + def test_get_response_json(self, __, session_response_return_value, expected_out): + """ + Function to test the functionality of get_response_json by asserting that the function + calls a specific function or raises a Timeout error + :param __: The name of the parameter, which is thrown away (string) + :param session_response_return_value: The mocked return value for the + session response (string) + :param expected_out: The expected output of the function (bool) + """ + with mock.patch("word_cloud_generator.requests") and patch( + "word_cloud_generator.json" + ): + word_cloud_generator.requests.session = MagicMock() + word_cloud_generator.requests.session.return_value.get.return_value.content = ( + session_response_return_value + ) + + word_cloud_generator.json = MagicMock() + + if expected_out: + word_cloud_generator.get_response_json(auth, headers, host) + + word_cloud_generator.json.loads.assert_called_once() + else: + self.assertRaises( + requests.exceptions.Timeout, + word_cloud_generator.get_response_json, + auth, + headers, + host, + ) + + @parameterized.expand( + [ + ("dates valid", "2022-01-01", ["test1", "test2", "test3", "test4"]), + ("dates invalid", "2024-01-01", []), + ] + ) + def test_get_issues_contents_after_time(self, __, filter_date, expected_out): + """ + Function to test the functionality of get_issues_contents_after_time by asserting + that the value returned is expected + :param __: The name of the parameter, which is thrown away (string) + :param filter_date: The mocked date to filter after (list) + :param expected_out: The expected output of the function (bool) + """ + with mock.patch("word_cloud_generator.get_response_json"), mock.patch("word_cloud_generator.filter_issue"): + issue_filter = {"end_date": filter_date} + values = ChangingJson(( + { + "values": ( + {"fields": {"summary": "test1", "created": "2023-01-01T00:00:00"}}, + {"fields": {"summary": "test2", "created": "2023-01-01T00:00:00"}}, + ), + "size": 50 + }, + { + "values": ( + {"fields": {"summary": "test3", "created": "2023-01-01T00:00:00"}}, + {"fields": {"summary": "test4", "created": "2023-01-01T00:00:00"}}, + ), + "size": 32 + } + )) + word_cloud_generator.get_response_json.return_value = values + word_cloud_generator.filter_issue.return_value = True + self.assertEqual( + word_cloud_generator.get_issues_contents_after_time( + auth, + headers, + host, + issue_filter, + ), + expected_out) + + @parameterized.expand( + [ + ("dates valid", {"start_date": "2024-01-01", "assigned": "test"}, True), + ("dates invalid", {"start_date": "2022-01-01", "assigned": "test"}, False), + ("assigned valid", {"start_date": "2024-01-01", "assigned": "test"}, True), + ("assigned invalid", {"start_date": "2024-01-01", "assigned": "test failed"}, False), + ] + ) + def test_filter_issue(self, __, issue_filter, expected_out): + """ + Function to test the functionality of filter_issue by asserting + that the value returned is expected + :param __: The name of the parameter, which is thrown away (string) + :param issue_filter: The issue filter (dict) + :param expected_out: The expected output of the function (bool) + """ + issue = {"fields": {"assignee": {"displayName": "test"}}} + issue_date = datetime.strptime("2023-01-01", "%Y-%m-%d") + self.assertEqual( + word_cloud_generator.filter_issue( + issue, + issue_filter, + issue_date, + ), + expected_out) + + def test_generate_word_cloud(self): + """ + Function to test the functionality of generate_word_cloud by asserting that the function + is called with specific inputs + """ + with mock.patch("word_cloud_generator.filter_word_cloud"), \ + mock.patch("word_cloud_generator.WordCloud"): + issues_contents = "test data" + issue_filter = "" + word_cloud_output_location = "test" + word_cloud_generator.generate_word_cloud( + issues_contents, + issue_filter, + word_cloud_output_location, + ) + word_cloud_generator.WordCloud.return_value.generate.assert_called_with( + word_cloud_generator.filter_word_cloud.return_value + ) + word_cloud_generator.WordCloud.return_value.to_file.assert_called_with("test") + + def test_filter_word_cloud(self): + """ + Function to test the functionality of generate_word_cloud by asserting that the function + returns an expected value + """ + issue_filter = { + "filter_not": "delete|this", + "filter_for": "data|test|this|here|delete|not", + } + issues_contents = "test data delete this not here" + self.assertEqual(word_cloud_generator.filter_word_cloud(issue_filter, issues_contents), + "test data not here") + + +if __name__ == "__main__": + unittest.main() diff --git a/word_cloud_generator/word_cloud_generator.py b/word_cloud_generator/word_cloud_generator.py index a5e62af5..cf54b253 100644 --- a/word_cloud_generator/word_cloud_generator.py +++ b/word_cloud_generator/word_cloud_generator.py @@ -8,7 +8,6 @@ from wordcloud import WordCloud import json -import matplotlib.pyplot as pyplot import requests import re @@ -147,26 +146,35 @@ def get_issues_contents_after_time(auth, headers, host, issue_filter): def filter_issue(issue, issue_filter, issue_date): - issue_assigned = issue.get("fields").get("assignee") - if issue_date > datetime.strptime(issue_filter.get("start_date"), "%Y-%m-%d"): + """ + Function to check if an issue passes the set filters + :param issue: A dict of an issues contents (dict) + :param issue_filter: Dict of filters to check the issues against (dict) + :param issue_date: The date that the issue was created (string) + :returns: If the issue passes the filters + """ + if issue.get("fields").get("assignee"): + issue_assigned = issue.get("fields").get("assignee").get("displayName") + if issue_filter.get("assigned") and issue_assigned != issue_filter.get("assigned"): + return False + else: return False - if issue.get("assigned") and issue_assigned != issue_assigned.get("assigned"): + if issue_date > datetime.strptime(issue_filter.get("start_date"), "%Y-%m-%d"): return False return True -def generate_word_cloud(issues_contents, issue_filter): +def generate_word_cloud(issues_contents, issue_filter, word_cloud_output_location): + """ + Function to generate and save a word cloud + :param issues_contents: The summary of every valid issue (list) + :param issue_filter: Dict of filters to check the issues against (dict) + :param word_cloud_output_location: The output location for the word cloud to be saved to + """ matches = re.findall(r"((\w+([.'](?![ \n']))*[-_]*)+)", issues_contents) if matches: issues_contents = " ".join(list(list(zip(*matches))[0])) - if issue_filter.get("filter_not"): - issues_contents = re.sub(issue_filter.get("filter_not").lower(), "", issues_contents, flags=re.I) - if issue_filter.get("filter_for"): - issues_contents = " ".join(re.findall( - issue_filter.get("filter_for").lower(), - issues_contents, - flags=re.IGNORECASE - )) + issues_contents = filter_word_cloud(issue_filter, issues_contents) word_cloud = WordCloud( width=2000, height=1000, @@ -175,14 +183,37 @@ def generate_word_cloud(issues_contents, issue_filter): background_color="white", collocations=False, regexp=r"\w*\S*", - ).generate(issues_contents) + ) + + word_cloud.generate(issues_contents) - pyplot.imshow(word_cloud, interpolation="bilinear") - pyplot.axis("off") - pyplot.show() + word_cloud.to_file(word_cloud_output_location) + + +def filter_word_cloud(issue_filter, issues_contents): + """ + Function to filter the contents of the word cloud to or against certain strings + :param issues_contents: The summary of every valid issue (list) + :param issue_filter: Dict of filters to check the issues against (dict) + :returns: The filtered issues contents + """ + if issue_filter.get("filter_not"): + issues_contents = re.sub(issue_filter.get("filter_not").lower(), "", issues_contents, flags=re.I) + if issue_filter.get("filter_for"): + issues_contents = " ".join(re.findall( + issue_filter.get("filter_for").lower(), + issues_contents, + flags=re.IGNORECASE + )) + + return issues_contents def word_cloud_generator(): + """ + Function to take arguments, generate the output location and run the + functions to the data for the word cloud and generate it + """ args = parse_args(argv[1:]) host = "https://stfc.atlassian.net" username = args.username @@ -194,7 +225,10 @@ def word_cloud_generator(): Path(issue_filter.get("output")).mkdir(exist_ok=True) - word_cloud_output_location = path.join(issue_filter["output"], "JSM Metric Data.csv") + word_cloud_output_location = path.join( + issue_filter["output"], + f"word cloud - {datetime.now().strftime('%Y.%m.%d.%H.%M.%S')}.png" + ) auth = requests.auth.HTTPBasicAuth(username, password) headers = { @@ -203,7 +237,7 @@ def word_cloud_generator(): issues_contents = get_issues_contents_after_time(auth, headers, host, issue_filter) - generate_word_cloud(" ".join(issues_contents), issue_filter) + generate_word_cloud(" ".join(issues_contents), issue_filter, word_cloud_output_location) if __name__ == "__main__": From 4ca71095f0b3ab0a9a595307c8f0c0f73200b8fc Mon Sep 17 00:00:00 2001 From: charlieDurnsfordHollands Date: Tue, 15 Aug 2023 16:02:24 +0100 Subject: [PATCH 04/11] add workflows, readme and requirements --- .github/workflows/black.yaml | 5 ++++ .github/workflows/word_cloud_generator.yaml | 33 +++++++++++++++++++++ word_cloud_generator/readme.md | 29 ++++++++++++++++++ word_cloud_generator/requirements.txt | 4 +++ 4 files changed, 71 insertions(+) create mode 100644 .github/workflows/word_cloud_generator.yaml create mode 100644 word_cloud_generator/readme.md create mode 100644 word_cloud_generator/requirements.txt diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index 861e8f21..452b050f 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -32,6 +32,11 @@ jobs: with: src: "dns_entry_checker" + - name: Word Cloud Generator + uses: psf/black@stable + with: + src: "word_cloud_generator" + - name: Openstack-Rally-Tester uses: psf/black@stable with: diff --git a/.github/workflows/word_cloud_generator.yaml b/.github/workflows/word_cloud_generator.yaml new file mode 100644 index 00000000..98ad6954 --- /dev/null +++ b/.github/workflows/word_cloud_generator.yaml @@ -0,0 +1,33 @@ +name: Word Cloud Generator Unittest + +on: + push: + branches: + - master + pull_request: + paths: + - "word_cloud_generator/**" + - ".github/workflows/word_cloud_generator.yaml" + +jobs: + test_with_unit_test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r word_cloud_generator/requirements.txt + - name: Test with unittest + run: | + cd word_cloud_generator + python3 -m unittest discover -s ./test -p "test_*.py" \ No newline at end of file diff --git a/word_cloud_generator/readme.md b/word_cloud_generator/readme.md new file mode 100644 index 00000000..98252e9b --- /dev/null +++ b/word_cloud_generator/readme.md @@ -0,0 +1,29 @@ +# Word Cloud Generator + +## General Info + +This is a Python script that when run, creates a filter word cloud from the summary of tickets over a time period. + +The script takes ~10 seconds to complete on a month of tickets + +Unit tests exist which test the logic of the methods the script uses, and the tests should be run whenever changes are made to the code. + +## Requirements + +requests: 2.31.0 +parameterized: 0.9.0 +python-dateutil: 2.8.2 +wordcloud: 1.9.2 + +## Setup +Running the script: +``` +$ cd ../word_cloud_generator +$ pip install -r requirements.txt +$ python3 word_cloud_generator.py +``` + +Running the unit tests: +``` +$ python3 -m unittest discover -s ./test -p "test_*.py" +``` \ No newline at end of file diff --git a/word_cloud_generator/requirements.txt b/word_cloud_generator/requirements.txt new file mode 100644 index 00000000..106aaf6e --- /dev/null +++ b/word_cloud_generator/requirements.txt @@ -0,0 +1,4 @@ +requests~=2.31.0 +parameterized~=0.9.0 +python-dateutil~=2.8.2 +wordcloud~=1.9.2 \ No newline at end of file From 01cb14d9d5608b38ed6e23c71c5f755913b31d0a Mon Sep 17 00:00:00 2001 From: charlieDurnsfordHollands Date: Tue, 15 Aug 2023 16:14:20 +0100 Subject: [PATCH 05/11] make format changes --- .../tests/test_word_cloud_generator.py | 86 +++++++++++++------ word_cloud_generator/word_cloud_generator.py | 30 ++++--- 2 files changed, 80 insertions(+), 36 deletions(-) diff --git a/word_cloud_generator/tests/test_word_cloud_generator.py b/word_cloud_generator/tests/test_word_cloud_generator.py index 9639cc81..87b7b012 100644 --- a/word_cloud_generator/tests/test_word_cloud_generator.py +++ b/word_cloud_generator/tests/test_word_cloud_generator.py @@ -62,7 +62,7 @@ def test_get_response_json(self, __, session_response_return_value, expected_out :param expected_out: The expected output of the function (bool) """ with mock.patch("word_cloud_generator.requests") and patch( - "word_cloud_generator.json" + "word_cloud_generator.json" ): word_cloud_generator.requests.session = MagicMock() word_cloud_generator.requests.session.return_value.get.return_value.content = ( @@ -98,24 +98,48 @@ def test_get_issues_contents_after_time(self, __, filter_date, expected_out): :param filter_date: The mocked date to filter after (list) :param expected_out: The expected output of the function (bool) """ - with mock.patch("word_cloud_generator.get_response_json"), mock.patch("word_cloud_generator.filter_issue"): + with mock.patch("word_cloud_generator.get_response_json"), mock.patch( + "word_cloud_generator.filter_issue" + ): issue_filter = {"end_date": filter_date} - values = ChangingJson(( - { - "values": ( - {"fields": {"summary": "test1", "created": "2023-01-01T00:00:00"}}, - {"fields": {"summary": "test2", "created": "2023-01-01T00:00:00"}}, - ), - "size": 50 - }, - { - "values": ( - {"fields": {"summary": "test3", "created": "2023-01-01T00:00:00"}}, - {"fields": {"summary": "test4", "created": "2023-01-01T00:00:00"}}, - ), - "size": 32 - } - )) + values = ChangingJson( + ( + { + "values": ( + { + "fields": { + "summary": "test1", + "created": "2023-01-01T00:00:00", + } + }, + { + "fields": { + "summary": "test2", + "created": "2023-01-01T00:00:00", + } + }, + ), + "size": 50 + }, + { + "values": ( + { + "fields": { + "summary": "test3", + "created": "2023-01-01T00:00:00", + } + }, + { + "fields": { + "summary": "test4", + "created": "2023-01-01T00:00:00", + } + }, + ), + "size": 32 + }, + ) + ) word_cloud_generator.get_response_json.return_value = values word_cloud_generator.filter_issue.return_value = True self.assertEqual( @@ -125,14 +149,18 @@ def test_get_issues_contents_after_time(self, __, filter_date, expected_out): host, issue_filter, ), - expected_out) + expected_out, + ) @parameterized.expand( [ ("dates valid", {"start_date": "2024-01-01", "assigned": "test"}, True), ("dates invalid", {"start_date": "2022-01-01", "assigned": "test"}, False), ("assigned valid", {"start_date": "2024-01-01", "assigned": "test"}, True), - ("assigned invalid", {"start_date": "2024-01-01", "assigned": "test failed"}, False), + ("assigned invalid", + {"start_date": "2024-01-01", "assigned": "test failed"}, + False, + ), ] ) def test_filter_issue(self, __, issue_filter, expected_out): @@ -151,15 +179,17 @@ def test_filter_issue(self, __, issue_filter, expected_out): issue_filter, issue_date, ), - expected_out) + expected_out, + ) def test_generate_word_cloud(self): """ Function to test the functionality of generate_word_cloud by asserting that the function is called with specific inputs """ - with mock.patch("word_cloud_generator.filter_word_cloud"), \ - mock.patch("word_cloud_generator.WordCloud"): + with mock.patch("word_cloud_generator.filter_word_cloud"), mock.patch( + "word_cloud_generator.WordCloud" + ): issues_contents = "test data" issue_filter = "" word_cloud_output_location = "test" @@ -171,7 +201,9 @@ def test_generate_word_cloud(self): word_cloud_generator.WordCloud.return_value.generate.assert_called_with( word_cloud_generator.filter_word_cloud.return_value ) - word_cloud_generator.WordCloud.return_value.to_file.assert_called_with("test") + word_cloud_generator.WordCloud.return_value.to_file.assert_called_with( + "test" + ) def test_filter_word_cloud(self): """ @@ -183,8 +215,10 @@ def test_filter_word_cloud(self): "filter_for": "data|test|this|here|delete|not", } issues_contents = "test data delete this not here" - self.assertEqual(word_cloud_generator.filter_word_cloud(issue_filter, issues_contents), - "test data not here") + self.assertEqual( + word_cloud_generator.filter_word_cloud(issue_filter, issues_contents), + "test data not here", + ) if __name__ == "__main__": diff --git a/word_cloud_generator/word_cloud_generator.py b/word_cloud_generator/word_cloud_generator.py index cf54b253..26139a1a 100644 --- a/word_cloud_generator/word_cloud_generator.py +++ b/word_cloud_generator/word_cloud_generator.py @@ -131,7 +131,9 @@ def get_issues_contents_after_time(auth, headers, host, issue_filter): issues = json_load.get("values") for issue in issues: - issue_date = datetime.strptime(issue.get("fields").get("created")[:10], "%Y-%m-%d") + issue_date = datetime.strptime( + issue.get("fields").get("created")[:10], "%Y-%m-%d" + ) if issue_date < datetime.strptime(issue_filter.get("end_date"), "%Y-%m-%d"): return issues_contents if filter_issue(issue, issue_filter, issue_date): @@ -155,7 +157,9 @@ def filter_issue(issue, issue_filter, issue_date): """ if issue.get("fields").get("assignee"): issue_assigned = issue.get("fields").get("assignee").get("displayName") - if issue_filter.get("assigned") and issue_assigned != issue_filter.get("assigned"): + if issue_filter.get("assigned") and issue_assigned != issue_filter.get( + "assigned" + ): return False else: return False @@ -198,13 +202,17 @@ def filter_word_cloud(issue_filter, issues_contents): :returns: The filtered issues contents """ if issue_filter.get("filter_not"): - issues_contents = re.sub(issue_filter.get("filter_not").lower(), "", issues_contents, flags=re.I) + issues_contents = re.sub( + issue_filter.get("filter_not").lower(), "", issues_contents, flags=re.I + ) if issue_filter.get("filter_for"): - issues_contents = " ".join(re.findall( - issue_filter.get("filter_for").lower(), - issues_contents, - flags=re.IGNORECASE - )) + issues_contents = " ".join( + re.findall( + issue_filter.get("filter_for").lower(), + issues_contents, + flags=re.IGNORECASE + ) + ) return issues_contents @@ -227,7 +235,7 @@ def word_cloud_generator(): word_cloud_output_location = path.join( issue_filter["output"], - f"word cloud - {datetime.now().strftime('%Y.%m.%d.%H.%M.%S')}.png" + f"word cloud - {datetime.now().strftime('%Y.%m.%d.%H.%M.%S')}.png", ) auth = requests.auth.HTTPBasicAuth(username, password) @@ -237,7 +245,9 @@ def word_cloud_generator(): issues_contents = get_issues_contents_after_time(auth, headers, host, issue_filter) - generate_word_cloud(" ".join(issues_contents), issue_filter, word_cloud_output_location) + generate_word_cloud( + " ".join(issues_contents), issue_filter, word_cloud_output_location + ) if __name__ == "__main__": From 29a8de9a9097fee5760ee1c834f801f0193c669d Mon Sep 17 00:00:00 2001 From: charlieDurnsfordHollands Date: Tue, 15 Aug 2023 16:17:37 +0100 Subject: [PATCH 06/11] make more formatting changes --- .../test_word_cloud_generator.py | 21 ++++++++++--------- word_cloud_generator/word_cloud_generator.py | 8 +++---- 2 files changed, 15 insertions(+), 14 deletions(-) rename word_cloud_generator/{tests => test}/test_word_cloud_generator.py (94%) diff --git a/word_cloud_generator/tests/test_word_cloud_generator.py b/word_cloud_generator/test/test_word_cloud_generator.py similarity index 94% rename from word_cloud_generator/tests/test_word_cloud_generator.py rename to word_cloud_generator/test/test_word_cloud_generator.py index 87b7b012..43a434bd 100644 --- a/word_cloud_generator/tests/test_word_cloud_generator.py +++ b/word_cloud_generator/test/test_word_cloud_generator.py @@ -43,7 +43,7 @@ def get(self, get_value): class WorldCloudGeneratorTests(unittest.TestCase): """ - Class for the tests to be run against the functions from word_cloud_generator.py + Class for the test to be run against the functions from word_cloud_generator.py """ @parameterized.expand( @@ -62,7 +62,7 @@ def test_get_response_json(self, __, session_response_return_value, expected_out :param expected_out: The expected output of the function (bool) """ with mock.patch("word_cloud_generator.requests") and patch( - "word_cloud_generator.json" + "word_cloud_generator.json" ): word_cloud_generator.requests.session = MagicMock() word_cloud_generator.requests.session.return_value.get.return_value.content = ( @@ -99,7 +99,7 @@ def test_get_issues_contents_after_time(self, __, filter_date, expected_out): :param expected_out: The expected output of the function (bool) """ with mock.patch("word_cloud_generator.get_response_json"), mock.patch( - "word_cloud_generator.filter_issue" + "word_cloud_generator.filter_issue" ): issue_filter = {"end_date": filter_date} values = ChangingJson( @@ -119,7 +119,7 @@ def test_get_issues_contents_after_time(self, __, filter_date, expected_out): } }, ), - "size": 50 + "size": 50, }, { "values": ( @@ -136,7 +136,7 @@ def test_get_issues_contents_after_time(self, __, filter_date, expected_out): } }, ), - "size": 32 + "size": 32, }, ) ) @@ -157,10 +157,11 @@ def test_get_issues_contents_after_time(self, __, filter_date, expected_out): ("dates valid", {"start_date": "2024-01-01", "assigned": "test"}, True), ("dates invalid", {"start_date": "2022-01-01", "assigned": "test"}, False), ("assigned valid", {"start_date": "2024-01-01", "assigned": "test"}, True), - ("assigned invalid", - {"start_date": "2024-01-01", "assigned": "test failed"}, - False, - ), + ( + "assigned invalid", + {"start_date": "2024-01-01", "assigned": "test failed"}, + False, + ), ] ) def test_filter_issue(self, __, issue_filter, expected_out): @@ -188,7 +189,7 @@ def test_generate_word_cloud(self): is called with specific inputs """ with mock.patch("word_cloud_generator.filter_word_cloud"), mock.patch( - "word_cloud_generator.WordCloud" + "word_cloud_generator.WordCloud" ): issues_contents = "test data" issue_filter = "" diff --git a/word_cloud_generator/word_cloud_generator.py b/word_cloud_generator/word_cloud_generator.py index 26139a1a..212914bd 100644 --- a/word_cloud_generator/word_cloud_generator.py +++ b/word_cloud_generator/word_cloud_generator.py @@ -95,8 +95,8 @@ def get_response_json(auth, headers, url): while attempts > 0: response = session.get(url, timeout=5) if ( - response.content != b'{"status":"RUNNING"}' - and response.content != b'{"status":"ENQUEUED"}' + response.content != b'{"status":"RUNNING"}' + and response.content != b'{"status":"ENQUEUED"}' ): break else: @@ -158,7 +158,7 @@ def filter_issue(issue, issue_filter, issue_date): if issue.get("fields").get("assignee"): issue_assigned = issue.get("fields").get("assignee").get("displayName") if issue_filter.get("assigned") and issue_assigned != issue_filter.get( - "assigned" + "assigned" ): return False else: @@ -210,7 +210,7 @@ def filter_word_cloud(issue_filter, issues_contents): re.findall( issue_filter.get("filter_for").lower(), issues_contents, - flags=re.IGNORECASE + flags=re.IGNORECASE, ) ) From 06b72d1af573e8f44352854caf6a9a9d147e4efc Mon Sep 17 00:00:00 2001 From: charlieDurnsfordHollands Date: Tue, 15 Aug 2023 16:19:13 +0100 Subject: [PATCH 07/11] fix indentation issues --- word_cloud_generator/test/test_word_cloud_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/word_cloud_generator/test/test_word_cloud_generator.py b/word_cloud_generator/test/test_word_cloud_generator.py index 43a434bd..a91bcb1e 100644 --- a/word_cloud_generator/test/test_word_cloud_generator.py +++ b/word_cloud_generator/test/test_word_cloud_generator.py @@ -62,7 +62,7 @@ def test_get_response_json(self, __, session_response_return_value, expected_out :param expected_out: The expected output of the function (bool) """ with mock.patch("word_cloud_generator.requests") and patch( - "word_cloud_generator.json" + "word_cloud_generator.json" ): word_cloud_generator.requests.session = MagicMock() word_cloud_generator.requests.session.return_value.get.return_value.content = ( @@ -99,7 +99,7 @@ def test_get_issues_contents_after_time(self, __, filter_date, expected_out): :param expected_out: The expected output of the function (bool) """ with mock.patch("word_cloud_generator.get_response_json"), mock.patch( - "word_cloud_generator.filter_issue" + "word_cloud_generator.filter_issue" ): issue_filter = {"end_date": filter_date} values = ChangingJson( From 7e4f9f8c2a7dd06d5b3195b65f5b81278e7cf1c7 Mon Sep 17 00:00:00 2001 From: charlieDurnsfordHollands Date: Tue, 22 Aug 2023 12:36:29 +0100 Subject: [PATCH 08/11] use dataclass --- word_cloud_generator/requirements.txt | 3 +- word_cloud_generator/word_cloud_generator.py | 129 +++++++++++++------ 2 files changed, 92 insertions(+), 40 deletions(-) diff --git a/word_cloud_generator/requirements.txt b/word_cloud_generator/requirements.txt index 106aaf6e..ff5f428e 100644 --- a/word_cloud_generator/requirements.txt +++ b/word_cloud_generator/requirements.txt @@ -1,4 +1,5 @@ requests~=2.31.0 parameterized~=0.9.0 python-dateutil~=2.8.2 -wordcloud~=1.9.2 \ No newline at end of file +wordcloud~=1.9.2 +mashumaro~=3.9 \ No newline at end of file diff --git a/word_cloud_generator/word_cloud_generator.py b/word_cloud_generator/word_cloud_generator.py index 212914bd..c7e33568 100644 --- a/word_cloud_generator/word_cloud_generator.py +++ b/word_cloud_generator/word_cloud_generator.py @@ -1,3 +1,4 @@ +import re from argparse import ArgumentParser, RawDescriptionHelpFormatter from datetime import datetime from dateutil.relativedelta import relativedelta @@ -6,10 +7,32 @@ from time import sleep from os import path from wordcloud import WordCloud +from typing import Optional +from dataclasses import dataclass +from mashumaro import DataClassDictMixin import json import requests -import re + + +@dataclass +class IssuesFilter(DataClassDictMixin): + output: str + start_date: str + end_date: str + word_cloud: str + assigned: Optional[str] = None + filter_for: Optional[str] = None + filter_not: Optional[str] = None + + +def from_user_inputs(**kwargs): + """ + Take the inputs from a argparse and populate a IssuesFilter dataclass and return it + :param kwargs: a dictionary of argparse values + """ + + return IssuesFilter.from_dict(kwargs) def parse_args(inp_args): @@ -74,6 +97,13 @@ def parse_args(inp_args): metavar="FILTER_NOT", help="Strings to filter the word cloud to not have", ) + parser.add_argument( + "-w", + "--word_cloud", + metavar="WORD_CLOUD", + help="Parameters to create the word cloud with", + default="2000, 1000, 25, 10000" + ) args = parser.parse_args(inp_args) return args @@ -91,12 +121,13 @@ def get_response_json(auth, headers, url): session.auth = auth attempts = 5 + response = None while attempts > 0: response = session.get(url, timeout=5) if ( - response.content != b'{"status":"RUNNING"}' - and response.content != b'{"status":"ENQUEUED"}' + response.content != b'{"status":"RUNNING"}' + and response.content != b'{"status":"ENQUEUED"}' ): break else: @@ -120,30 +151,29 @@ def get_issues_contents_after_time(auth, headers, host, issue_filter): :param host: The host used to create the URL to send the request (string) :returns: A list with the contents of all valid issues """ + curr_marker = 0 + check_limit = 50 issues_contents = [] - return_amount = 50 - issues_amount = 0 - while return_amount == 50: - url = f"{host}/rest/servicedeskapi/servicedesk/6/queue/182/issue?start={issues_amount}" - + while True: + url = f"{host}/rest/servicedeskapi/servicedesk/6/queue/182/issue?start={curr_marker}" json_load = get_response_json(auth, headers, url) - issues = json_load.get("values") - - for issue in issues: + i = 0 + for i, issue in enumerate(issues, 1): issue_date = datetime.strptime( issue.get("fields").get("created")[:10], "%Y-%m-%d" ) - if issue_date < datetime.strptime(issue_filter.get("end_date"), "%Y-%m-%d"): + if issue_date < datetime.strptime(issue_filter.end_date, "%Y-%m-%d"): return issues_contents if filter_issue(issue, issue_filter, issue_date): issue_contents = issue.get("fields").get("summary") if issue_contents: issues_contents.append(issue_contents) - return_amount = json_load.get("size") - issues_amount = issues_amount + return_amount - + # break out of the loop if we reach the end of the issue list + if i < check_limit: + break + curr_marker += json_load.get("size") return issues_contents @@ -155,35 +185,47 @@ def filter_issue(issue, issue_filter, issue_date): :param issue_date: The date that the issue was created (string) :returns: If the issue passes the filters """ - if issue.get("fields").get("assignee"): - issue_assigned = issue.get("fields").get("assignee").get("displayName") - if issue_filter.get("assigned") and issue_assigned != issue_filter.get( - "assigned" - ): - return False - else: + fields = issue.get("fields", None) + if not fields: + return False + + assignee = fields.get("assignee", None) + if not assignee: return False - if issue_date > datetime.strptime(issue_filter.get("start_date"), "%Y-%m-%d"): + + issue_assigned = assignee.get("displayName", None) + assign_check = issue_filter.assigned + if (not issue_assigned or issue_assigned != assign_check) and assign_check: + return False + + if issue_date > datetime.strptime(issue_filter.start_date, "%Y-%m-%d"): return False return True -def generate_word_cloud(issues_contents, issue_filter, word_cloud_output_location): +def generate_word_cloud(issues_contents, issue_filter, word_cloud_output_location, kwargs): """ Function to generate and save a word cloud :param issues_contents: The summary of every valid issue (list) :param issue_filter: Dict of filters to check the issues against (dict) :param word_cloud_output_location: The output location for the word cloud to be saved to + :param kwargs: A set of kwargs to pass to WordCloud + - width + - height + - min_font_size + - max_words """ matches = re.findall(r"((\w+([.'](?![ \n']))*[-_]*)+)", issues_contents) + # Regex to find all words and include words joined with certain characters, while not + # allowing certain characters to exist at the start or end of the word, such as dots. if matches: issues_contents = " ".join(list(list(zip(*matches))[0])) issues_contents = filter_word_cloud(issue_filter, issues_contents) word_cloud = WordCloud( - width=2000, - height=1000, - min_font_size=25, - max_words=10000, + width=kwargs["width"], + height=kwargs["height"], + min_font_size=kwargs["min_font_size"], + max_words=kwargs["max_words"], background_color="white", collocations=False, regexp=r"\w*\S*", @@ -201,14 +243,14 @@ def filter_word_cloud(issue_filter, issues_contents): :param issue_filter: Dict of filters to check the issues against (dict) :returns: The filtered issues contents """ - if issue_filter.get("filter_not"): + if issue_filter.filter_not: issues_contents = re.sub( - issue_filter.get("filter_not").lower(), "", issues_contents, flags=re.I + issue_filter.filter_not.lower(), "", issues_contents, flags=re.I ) - if issue_filter.get("filter_for"): + if issue_filter.filter_for: issues_contents = " ".join( re.findall( - issue_filter.get("filter_for").lower(), + issue_filter.filter_for.lower(), issues_contents, flags=re.IGNORECASE, ) @@ -226,15 +268,21 @@ def word_cloud_generator(): host = "https://stfc.atlassian.net" username = args.username password = args.password - issue_filter = {} - for arg in args.__dict__: - if args.__dict__[arg] is not None and arg != "username" and arg != "password": - issue_filter.update({arg: args.__dict__[arg]}) - Path(issue_filter.get("output")).mkdir(exist_ok=True) + issue_filter = from_user_inputs(**args.__dict__) + + parameters_list = issue_filter.word_cloud.split(", ") + word_cloud_parameters = { + "width": int(parameters_list[0]), + "height": int(parameters_list[1]), + "min_font_size": int(parameters_list[2]), + "max_words": int(parameters_list[3]), + } + + Path(issue_filter.output).mkdir(exist_ok=True) word_cloud_output_location = path.join( - issue_filter["output"], + issue_filter.output, f"word cloud - {datetime.now().strftime('%Y.%m.%d.%H.%M.%S')}.png", ) @@ -246,7 +294,10 @@ def word_cloud_generator(): issues_contents = get_issues_contents_after_time(auth, headers, host, issue_filter) generate_word_cloud( - " ".join(issues_contents), issue_filter, word_cloud_output_location + " ".join(issues_contents), + issue_filter, + word_cloud_output_location, + word_cloud_parameters, ) From 4275c67acafaec480ce023e8f145877988e6232e Mon Sep 17 00:00:00 2001 From: charlieDurnsfordHollands Date: Tue, 22 Aug 2023 14:18:51 +0100 Subject: [PATCH 09/11] fix tests and make formatting changes --- .../test/test_word_cloud_generator.py | 68 +++++++++++++++---- word_cloud_generator/word_cloud_generator.py | 16 +++-- 2 files changed, 63 insertions(+), 21 deletions(-) diff --git a/word_cloud_generator/test/test_word_cloud_generator.py b/word_cloud_generator/test/test_word_cloud_generator.py index a91bcb1e..3b7b2a51 100644 --- a/word_cloud_generator/test/test_word_cloud_generator.py +++ b/word_cloud_generator/test/test_word_cloud_generator.py @@ -62,7 +62,7 @@ def test_get_response_json(self, __, session_response_return_value, expected_out :param expected_out: The expected output of the function (bool) """ with mock.patch("word_cloud_generator.requests") and patch( - "word_cloud_generator.json" + "word_cloud_generator.json" ): word_cloud_generator.requests.session = MagicMock() word_cloud_generator.requests.session.return_value.get.return_value.content = ( @@ -99,9 +99,14 @@ def test_get_issues_contents_after_time(self, __, filter_date, expected_out): :param expected_out: The expected output of the function (bool) """ with mock.patch("word_cloud_generator.get_response_json"), mock.patch( - "word_cloud_generator.filter_issue" + "word_cloud_generator.filter_issue" ): - issue_filter = {"end_date": filter_date} + issue_filter = word_cloud_generator.from_user_inputs(**{ + "output": None, + "start_date": None, + "end_date": filter_date, + "word_cloud": None, + }) values = ChangingJson( ( { @@ -154,14 +159,37 @@ def test_get_issues_contents_after_time(self, __, filter_date, expected_out): @parameterized.expand( [ - ("dates valid", {"start_date": "2024-01-01", "assigned": "test"}, True), - ("dates invalid", {"start_date": "2022-01-01", "assigned": "test"}, False), - ("assigned valid", {"start_date": "2024-01-01", "assigned": "test"}, True), - ( - "assigned invalid", - {"start_date": "2024-01-01", "assigned": "test failed"}, - False, - ), + ("dates valid", { + "output": None, + "end_date": None, + "word_cloud": None, + "start_date": "2024-01-01", + "assigned": "test", + }, True), + ("dates invalid", { + "output": None, + "end_date": None, + "word_cloud": None, + "start_date": "2022-01-01", + "assigned": "test", + }, False), + ("assigned valid", { + "output": None, + "end_date": None, + "word_cloud": None, + "start_date": "2024-01-01", + "assigned": "test", + }, True), + ("assigned invalid", + { + "output": None, + "end_date": None, + "word_cloud": None, + "start_date": "2024-01-01", + "assigned": "test failed", + }, + False, + ), ] ) def test_filter_issue(self, __, issue_filter, expected_out): @@ -174,6 +202,7 @@ def test_filter_issue(self, __, issue_filter, expected_out): """ issue = {"fields": {"assignee": {"displayName": "test"}}} issue_date = datetime.strptime("2023-01-01", "%Y-%m-%d") + issue_filter = word_cloud_generator.from_user_inputs(**issue_filter) self.assertEqual( word_cloud_generator.filter_issue( issue, @@ -189,15 +218,22 @@ def test_generate_word_cloud(self): is called with specific inputs """ with mock.patch("word_cloud_generator.filter_word_cloud"), mock.patch( - "word_cloud_generator.WordCloud" + "word_cloud_generator.WordCloud" ): issues_contents = "test data" issue_filter = "" word_cloud_output_location = "test" + word_cloud_parameters = { + "width": 2000, + "height": 1000, + "min_font_size": 25, + "max_words": 10000, + } word_cloud_generator.generate_word_cloud( issues_contents, issue_filter, word_cloud_output_location, + **word_cloud_parameters, ) word_cloud_generator.WordCloud.return_value.generate.assert_called_with( word_cloud_generator.filter_word_cloud.return_value @@ -211,10 +247,14 @@ def test_filter_word_cloud(self): Function to test the functionality of generate_word_cloud by asserting that the function returns an expected value """ - issue_filter = { + issue_filter = word_cloud_generator.from_user_inputs(**{ + "output": None, + "start_date": None, + "end_date": None, + "word_cloud": None, "filter_not": "delete|this", "filter_for": "data|test|this|here|delete|not", - } + }) issues_contents = "test data delete this not here" self.assertEqual( word_cloud_generator.filter_word_cloud(issue_filter, issues_contents), diff --git a/word_cloud_generator/word_cloud_generator.py b/word_cloud_generator/word_cloud_generator.py index c7e33568..fc5e6b61 100644 --- a/word_cloud_generator/word_cloud_generator.py +++ b/word_cloud_generator/word_cloud_generator.py @@ -126,8 +126,8 @@ def get_response_json(auth, headers, url): while attempts > 0: response = session.get(url, timeout=5) if ( - response.content != b'{"status":"RUNNING"}' - and response.content != b'{"status":"ENQUEUED"}' + response.content != b'{"status":"RUNNING"}' + and response.content != b'{"status":"ENQUEUED"}' ): break else: @@ -158,7 +158,7 @@ def get_issues_contents_after_time(auth, headers, host, issue_filter): url = f"{host}/rest/servicedeskapi/servicedesk/6/queue/182/issue?start={curr_marker}" json_load = get_response_json(auth, headers, url) issues = json_load.get("values") - i = 0 + issues_length = json_load.get("size") for i, issue in enumerate(issues, 1): issue_date = datetime.strptime( issue.get("fields").get("created")[:10], "%Y-%m-%d" @@ -171,9 +171,9 @@ def get_issues_contents_after_time(auth, headers, host, issue_filter): issues_contents.append(issue_contents) # break out of the loop if we reach the end of the issue list - if i < check_limit: + if issues_length < check_limit: break - curr_marker += json_load.get("size") + curr_marker += issues_length return issues_contents @@ -203,7 +203,9 @@ def filter_issue(issue, issue_filter, issue_date): return True -def generate_word_cloud(issues_contents, issue_filter, word_cloud_output_location, kwargs): +def generate_word_cloud( + issues_contents, issue_filter, word_cloud_output_location, **kwargs +): """ Function to generate and save a word cloud :param issues_contents: The summary of every valid issue (list) @@ -297,7 +299,7 @@ def word_cloud_generator(): " ".join(issues_contents), issue_filter, word_cloud_output_location, - word_cloud_parameters, + **word_cloud_parameters, ) From 47b076b55ff4aad6e6542e0e7a88c7ae6868cbbb Mon Sep 17 00:00:00 2001 From: charlieDurnsfordHollands Date: Tue, 22 Aug 2023 14:59:50 +0100 Subject: [PATCH 10/11] make formatting changes --- .../test/test_word_cloud_generator.py | 113 ++++++++++-------- word_cloud_generator/word_cloud_generator.py | 4 +- 2 files changed, 67 insertions(+), 50 deletions(-) diff --git a/word_cloud_generator/test/test_word_cloud_generator.py b/word_cloud_generator/test/test_word_cloud_generator.py index 3b7b2a51..130d5892 100644 --- a/word_cloud_generator/test/test_word_cloud_generator.py +++ b/word_cloud_generator/test/test_word_cloud_generator.py @@ -62,7 +62,7 @@ def test_get_response_json(self, __, session_response_return_value, expected_out :param expected_out: The expected output of the function (bool) """ with mock.patch("word_cloud_generator.requests") and patch( - "word_cloud_generator.json" + "word_cloud_generator.json" ): word_cloud_generator.requests.session = MagicMock() word_cloud_generator.requests.session.return_value.get.return_value.content = ( @@ -99,14 +99,16 @@ def test_get_issues_contents_after_time(self, __, filter_date, expected_out): :param expected_out: The expected output of the function (bool) """ with mock.patch("word_cloud_generator.get_response_json"), mock.patch( - "word_cloud_generator.filter_issue" + "word_cloud_generator.filter_issue" ): - issue_filter = word_cloud_generator.from_user_inputs(**{ - "output": None, - "start_date": None, - "end_date": filter_date, - "word_cloud": None, - }) + issue_filter = word_cloud_generator.from_user_inputs( + **{ + "output": None, + "start_date": None, + "end_date": filter_date, + "word_cloud": None, + } + ) values = ChangingJson( ( { @@ -159,37 +161,50 @@ def test_get_issues_contents_after_time(self, __, filter_date, expected_out): @parameterized.expand( [ - ("dates valid", { - "output": None, - "end_date": None, - "word_cloud": None, - "start_date": "2024-01-01", - "assigned": "test", - }, True), - ("dates invalid", { - "output": None, - "end_date": None, - "word_cloud": None, - "start_date": "2022-01-01", - "assigned": "test", - }, False), - ("assigned valid", { - "output": None, - "end_date": None, - "word_cloud": None, - "start_date": "2024-01-01", - "assigned": "test", - }, True), - ("assigned invalid", - { - "output": None, - "end_date": None, - "word_cloud": None, - "start_date": "2024-01-01", - "assigned": "test failed", - }, - False, - ), + ( + "dates valid", + { + "output": None, + "end_date": None, + "word_cloud": None, + "start_date": "2024-01-01", + "assigned": "test", + }, + True, + ), + ( + "dates invalid", + { + "output": None, + "end_date": None, + "word_cloud": None, + "start_date": "2022-01-01", + "assigned": "test", + }, + False, + ), + ( + "assigned valid", + { + "output": None, + "end_date": None, + "word_cloud": None, + "start_date": "2024-01-01", + "assigned": "test", + }, + True, + ), + ( + "assigned invalid", + { + "output": None, + "end_date": None, + "word_cloud": None, + "start_date": "2024-01-01", + "assigned": "test failed", + }, + False, + ), ] ) def test_filter_issue(self, __, issue_filter, expected_out): @@ -218,7 +233,7 @@ def test_generate_word_cloud(self): is called with specific inputs """ with mock.patch("word_cloud_generator.filter_word_cloud"), mock.patch( - "word_cloud_generator.WordCloud" + "word_cloud_generator.WordCloud" ): issues_contents = "test data" issue_filter = "" @@ -247,14 +262,16 @@ def test_filter_word_cloud(self): Function to test the functionality of generate_word_cloud by asserting that the function returns an expected value """ - issue_filter = word_cloud_generator.from_user_inputs(**{ - "output": None, - "start_date": None, - "end_date": None, - "word_cloud": None, - "filter_not": "delete|this", - "filter_for": "data|test|this|here|delete|not", - }) + issue_filter = word_cloud_generator.from_user_inputs( + **{ + "output": None, + "start_date": None, + "end_date": None, + "word_cloud": None, + "filter_not": "delete|this", + "filter_for": "data|test|this|here|delete|not", + } + ) issues_contents = "test data delete this not here" self.assertEqual( word_cloud_generator.filter_word_cloud(issue_filter, issues_contents), diff --git a/word_cloud_generator/word_cloud_generator.py b/word_cloud_generator/word_cloud_generator.py index fc5e6b61..d812f1ed 100644 --- a/word_cloud_generator/word_cloud_generator.py +++ b/word_cloud_generator/word_cloud_generator.py @@ -102,7 +102,7 @@ def parse_args(inp_args): "--word_cloud", metavar="WORD_CLOUD", help="Parameters to create the word cloud with", - default="2000, 1000, 25, 10000" + default="2000, 1000, 25, 10000", ) args = parser.parse_args(inp_args) return args @@ -204,7 +204,7 @@ def filter_issue(issue, issue_filter, issue_date): def generate_word_cloud( - issues_contents, issue_filter, word_cloud_output_location, **kwargs + issues_contents, issue_filter, word_cloud_output_location, **kwargs ): """ Function to generate and save a word cloud From 5ad462b734cfe5fd79d65e34ecfb2a39e276f843 Mon Sep 17 00:00:00 2001 From: Kalibh Halford Date: Wed, 6 Dec 2023 15:08:07 +0000 Subject: [PATCH 11/11] MAIN: Made changes from old PR comments to get the PR merged --- .github/workflows/word_cloud_generator.yaml | 2 +- word_cloud_generator/__init__.py | 0 word_cloud_generator/requirements.txt | 10 ++-- .../{test => }/test_word_cloud_generator.py | 0 word_cloud_generator/word_cloud_generator.py | 59 ++++++++++--------- 5 files changed, 38 insertions(+), 33 deletions(-) create mode 100644 word_cloud_generator/__init__.py rename word_cloud_generator/{test => }/test_word_cloud_generator.py (100%) diff --git a/.github/workflows/word_cloud_generator.yaml b/.github/workflows/word_cloud_generator.yaml index 98ad6954..64120189 100644 --- a/.github/workflows/word_cloud_generator.yaml +++ b/.github/workflows/word_cloud_generator.yaml @@ -30,4 +30,4 @@ jobs: - name: Test with unittest run: | cd word_cloud_generator - python3 -m unittest discover -s ./test -p "test_*.py" \ No newline at end of file + python3 -m unittest test_word_cloud_generator.py \ No newline at end of file diff --git a/word_cloud_generator/__init__.py b/word_cloud_generator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/word_cloud_generator/requirements.txt b/word_cloud_generator/requirements.txt index ff5f428e..693c8a1b 100644 --- a/word_cloud_generator/requirements.txt +++ b/word_cloud_generator/requirements.txt @@ -1,5 +1,5 @@ -requests~=2.31.0 -parameterized~=0.9.0 -python-dateutil~=2.8.2 -wordcloud~=1.9.2 -mashumaro~=3.9 \ No newline at end of file +requests +parameterized +python-dateutil +wordcloud +mashumaro \ No newline at end of file diff --git a/word_cloud_generator/test/test_word_cloud_generator.py b/word_cloud_generator/test_word_cloud_generator.py similarity index 100% rename from word_cloud_generator/test/test_word_cloud_generator.py rename to word_cloud_generator/test_word_cloud_generator.py diff --git a/word_cloud_generator/word_cloud_generator.py b/word_cloud_generator/word_cloud_generator.py index d812f1ed..5d8ce491 100644 --- a/word_cloud_generator/word_cloud_generator.py +++ b/word_cloud_generator/word_cloud_generator.py @@ -1,4 +1,3 @@ -import re from argparse import ArgumentParser, RawDescriptionHelpFormatter from datetime import datetime from dateutil.relativedelta import relativedelta @@ -7,10 +6,10 @@ from time import sleep from os import path from wordcloud import WordCloud -from typing import Optional +from typing import Optional, Dict, List from dataclasses import dataclass from mashumaro import DataClassDictMixin - +import re import json import requests @@ -28,17 +27,17 @@ class IssuesFilter(DataClassDictMixin): def from_user_inputs(**kwargs): """ - Take the inputs from a argparse and populate a IssuesFilter dataclass and return it + Take the inputs from an argparse and populate a IssuesFilter dataclass and return it :param kwargs: a dictionary of argparse values """ - return IssuesFilter.from_dict(kwargs) + return IssuesFilter(**kwargs) -def parse_args(inp_args): +def parse_args(inp_args: Dict) -> Dict: """ Function to parse commandline args - :param inp_args: a set of commandline args to parse (dict) + :param inp_args: a set of commandline args to parse :returns: A dictionary of parsed args """ # Get arguments passed to the script @@ -65,19 +64,23 @@ def parse_args(inp_args): help="Directory to create the output files in", default="output", ) + default_value_start_date = datetime.now().strftime("%Y-%m-%d") parser.add_argument( "-s", "--start_date", metavar="START_DATE", help="Date to get issues from", - default=datetime.now().strftime("%Y-%m-%d"), + default=default_value_start_date, + ) + default_value_end_date = (datetime.now() - relativedelta(months=1)).strftime( + "%Y-%m-%d" ) parser.add_argument( "-e", "--end_date", metavar="END_DATE", help="Date to get issues to", - default=(datetime.now() - relativedelta(months=1)).strftime("%Y-%m-%d"), + default=default_value_end_date, ) parser.add_argument( "-a", @@ -108,12 +111,12 @@ def parse_args(inp_args): return args -def get_response_json(auth, headers, url): +def get_response_json(auth, headers: Dict, url: str) -> Dict: """ Function to send a get request to a url and return the response as json :param auth: A HTTPBasicAuth object for authentication (HTTPBasicAuth) - :param headers: A request Header (dict) - :param url: The URL to send the request (string) + :param headers: A request Header + :param url: The URL to send the request :returns: A dictionary of JSON values """ session = requests.session() @@ -142,13 +145,15 @@ def get_response_json(auth, headers, url): return json.loads(response.text) -def get_issues_contents_after_time(auth, headers, host, issue_filter): +def get_issues_contents_after_time( + auth, headers: Dict, host: str, issue_filter: Dict +) -> List: """ Function to get the contents of through issues using a loop, as only 50 can be checked at a time - :param issue_filter: Dict of filters to check the issues against (dict) + :param issue_filter: Dict of filters to check the issues against :param auth: A HTTPBasicAuth object for authentication (HTTPBasicAuth) - :param headers: A request Header (dict) - :param host: The host used to create the URL to send the request (string) + :param headers: A request Header + :param host: The host used to create the URL to send the request :returns: A list with the contents of all valid issues """ curr_marker = 0 @@ -177,12 +182,12 @@ def get_issues_contents_after_time(auth, headers, host, issue_filter): return issues_contents -def filter_issue(issue, issue_filter, issue_date): +def filter_issue(issue: Dict, issue_filter: Dict, issue_date: str) -> bool: """ Function to check if an issue passes the set filters - :param issue: A dict of an issues contents (dict) - :param issue_filter: Dict of filters to check the issues against (dict) - :param issue_date: The date that the issue was created (string) + :param issue: A dict of an issues contents + :param issue_filter: Dict of filters to check the issues against + :param issue_date: The date that the issue was created :returns: If the issue passes the filters """ fields = issue.get("fields", None) @@ -204,12 +209,12 @@ def filter_issue(issue, issue_filter, issue_date): def generate_word_cloud( - issues_contents, issue_filter, word_cloud_output_location, **kwargs + issues_contents: List, issue_filter: Dict, word_cloud_output_location, **kwargs ): """ Function to generate and save a word cloud - :param issues_contents: The summary of every valid issue (list) - :param issue_filter: Dict of filters to check the issues against (dict) + :param issues_contents: The summary of every valid issue + :param issue_filter: Dict of filters to check the issues against :param word_cloud_output_location: The output location for the word cloud to be saved to :param kwargs: A set of kwargs to pass to WordCloud - width @@ -238,11 +243,11 @@ def generate_word_cloud( word_cloud.to_file(word_cloud_output_location) -def filter_word_cloud(issue_filter, issues_contents): +def filter_word_cloud(issue_filter: Dict, issues_contents: List): """ Function to filter the contents of the word cloud to or against certain strings - :param issues_contents: The summary of every valid issue (list) - :param issue_filter: Dict of filters to check the issues against (dict) + :param issues_contents: The summary of every valid issue + :param issue_filter: Dict of filters to check the issues against :returns: The filtered issues contents """ if issue_filter.filter_not: @@ -271,7 +276,7 @@ def word_cloud_generator(): username = args.username password = args.password - issue_filter = from_user_inputs(**args.__dict__) + issue_filter = from_user_inputs(vars(args)) parameters_list = issue_filter.word_cloud.split(", ") word_cloud_parameters = {