From 04042fdff450fdc69f3c3b0dbdf6944478f808ad Mon Sep 17 00:00:00 2001 From: Abdujabbar MIRKHALIKOV Date: Fri, 19 Apr 2024 17:11:34 +0500 Subject: [PATCH 1/3] Added try/except block for fetchdata process, added custom exceptions for 4xx and 5xx http error codes --- contributors/management/commands/fetchdata.py | 109 +++++++++--------- contributors/utils/github_lib.py | 11 ++ 2 files changed, 67 insertions(+), 53 deletions(-) diff --git a/contributors/management/commands/fetchdata.py b/contributors/management/commands/fetchdata.py index 56b44b7b..eb15fb17 100644 --- a/contributors/management/commands/fetchdata.py +++ b/contributors/management/commands/fetchdata.py @@ -143,61 +143,64 @@ def handle( # noqa: C901,WPS110,WPS213,WPS231,WPS210 ) for owner_data in data_of_owners_and_repos.values(): - table = ( - Contributor - if owner_data['details']['type'] == 'User' - else Organization - ) - owner, _ = misc.update_or_create_record( - table, owner_data['details'], - ) - logger.info(owner) - - repos_to_process = [ - repo for repo in owner_data['repos'] - if repo['name'] not in IGNORED_REPOSITORIES - ] - number_of_repos = len(repos_to_process) - for i, repo_data in enumerate(repos_to_process, start=1): # noqa: WPS111,E501 - repo, _ = misc.update_or_create_record(Repository, repo_data) - logger.info(f"{repo} ({i}/{number_of_repos})") - if repo_data['size'] == 0: - logger.info("Empty repository") - continue - - language = repo_data['language'] - if language: - label, _ = Label.objects.get_or_create(name=language) - repo.labels.add(label) - - logger.info("Processing issues and pull requests") - create_contributions( - repo, - github.get_repo_issues(owner, repo, session), - user_field='user', - id_field='id', - type_='iss', - ) - - logger.info("Processing commits") - create_contributions( - repo, - github.get_repo_commits_except_merges( - owner, repo, session=session, - ), - user_field='author', - id_field='sha', - type_='cit', + try: + table = ( + Contributor + if owner_data['details']['type'] == 'User' + else Organization ) - - logger.info("Processing comments") - create_contributions( - repo, - github.get_all_types_of_comments(owner, repo, session), - user_field='user', - id_field='id', - type_='cnt', + owner, _ = misc.update_or_create_record( + table, owner_data['details'], ) + logger.info(owner) + + repos_to_process = [ + repo for repo in owner_data['repos'] + if repo['name'] not in IGNORED_REPOSITORIES + ] + number_of_repos = len(repos_to_process) + for i, repo_data in enumerate(repos_to_process, start=1): # noqa: WPS111,E501 + repo, _ = misc.update_or_create_record(Repository, repo_data) + logger.info(f"{repo} ({i}/{number_of_repos})") + if repo_data['size'] == 0: + logger.info("Empty repository") + continue + + language = repo_data['language'] + if language: + label, _ = Label.objects.get_or_create(name=language) + repo.labels.add(label) + + logger.info("Processing issues and pull requests") + create_contributions( + repo, + github.get_repo_issues(owner, repo, session), + user_field='user', + id_field='id', + type_='iss', + ) + + logger.info("Processing commits") + create_contributions( + repo, + github.get_repo_commits_except_merges( + owner, repo, session=session, + ), + user_field='author', + id_field='sha', + type_='cit', + ) + + logger.info("Processing comments") + create_contributions( + repo, + github.get_all_types_of_comments(owner, repo, session), + user_field='user', + id_field='id', + type_='cnt', + ) + except Exception as ex: + logger.warning(self.style.WARNING(f"Unexpected behavior while sync data for owner"), owner_data, ex) session.close() diff --git a/contributors/utils/github_lib.py b/contributors/utils/github_lib.py index e3e1295e..b402ee80 100644 --- a/contributors/utils/github_lib.py +++ b/contributors/utils/github_lib.py @@ -29,6 +29,12 @@ class Accepted(GitHubError): class NoContent(GitHubError): """HTTP 204 Response.""" +class ClientErrorResponse(GitHubError): + """HTTP 4xx Response.""" + +class ServerErrorResponse(GitHubError): + """HTTP 5xx Response""" + class NoContributorsError(GitHubError): """A repository has no contributors.""" @@ -125,6 +131,11 @@ def get_whole_response_as_json(url, session=None): raise NoContent("204 No Content", response=response) elif response.status_code == requests.codes.accepted: raise Accepted("202 Accepted. No cached data. Retry.") + elif response.status_code >= 500: + raise ServerErrorResponse("Server error response", response=response) + elif response.status_code >= 400: + raise ClientErrorResponse("Client error response.", response=response) + return response.json() From b3a7202c0167e116562ccb23c286443e20a858c0 Mon Sep 17 00:00:00 2001 From: Abdujabbar MIRKHALIKOV Date: Fri, 19 Apr 2024 21:30:15 +0500 Subject: [PATCH 2/3] Fix lint errors --- contributors/management/commands/fetchdata.py | 83 +++++++++++-------- contributors/utils/github_lib.py | 37 +++++++-- 2 files changed, 78 insertions(+), 42 deletions(-) diff --git a/contributors/management/commands/fetchdata.py b/contributors/management/commands/fetchdata.py index eb15fb17..7f8e6697 100644 --- a/contributors/management/commands/fetchdata.py +++ b/contributors/management/commands/fetchdata.py @@ -143,35 +143,35 @@ def handle( # noqa: C901,WPS110,WPS213,WPS231,WPS210 ) for owner_data in data_of_owners_and_repos.values(): - try: - table = ( - Contributor - if owner_data['details']['type'] == 'User' - else Organization - ) - owner, _ = misc.update_or_create_record( - table, owner_data['details'], - ) - logger.info(owner) - - repos_to_process = [ - repo for repo in owner_data['repos'] - if repo['name'] not in IGNORED_REPOSITORIES - ] - number_of_repos = len(repos_to_process) - for i, repo_data in enumerate(repos_to_process, start=1): # noqa: WPS111,E501 - repo, _ = misc.update_or_create_record(Repository, repo_data) - logger.info(f"{repo} ({i}/{number_of_repos})") - if repo_data['size'] == 0: - logger.info("Empty repository") - continue - - language = repo_data['language'] - if language: - label, _ = Label.objects.get_or_create(name=language) - repo.labels.add(label) - - logger.info("Processing issues and pull requests") + table = ( + Contributor + if owner_data['details']['type'] == 'User' + else Organization + ) + owner, _ = misc.update_or_create_record( + table, owner_data['details'], + ) + logger.info(owner) + + repos_to_process = [ + repo for repo in owner_data['repos'] + if repo['name'] not in IGNORED_REPOSITORIES + ] + number_of_repos = len(repos_to_process) + for i, repo_data in enumerate(repos_to_process, start=1): # noqa: WPS111,E501 + repo, _ = misc.update_or_create_record(Repository, repo_data) + logger.info(f"{repo} ({i}/{number_of_repos})") + if repo_data['size'] == 0: + logger.info("Empty repository") + continue + + language = repo_data['language'] + if language: + label, _ = Label.objects.get_or_create(name=language) + repo.labels.add(label) + logger.info("Processing issues and pull requests") + + try: create_contributions( repo, github.get_repo_issues(owner, repo, session), @@ -179,8 +179,15 @@ def handle( # noqa: C901,WPS110,WPS213,WPS231,WPS210 id_field='id', type_='iss', ) + except Exception as processing_issues_exp: + logger.error( + msg="Failed processing issues and pull requests", + args=(repo, processing_issues_exp), + ) + continue - logger.info("Processing commits") + logger.info("Processing commits") + try: create_contributions( repo, github.get_repo_commits_except_merges( @@ -190,8 +197,15 @@ def handle( # noqa: C901,WPS110,WPS213,WPS231,WPS210 id_field='sha', type_='cit', ) + except Exception as processing_commits_ex: + logger.error( + msg="Failed processing commits", + args=(repo, processing_commits_ex), + ) + continue - logger.info("Processing comments") + logger.info("Processing comments") + try: create_contributions( repo, github.get_all_types_of_comments(owner, repo, session), @@ -199,8 +213,11 @@ def handle( # noqa: C901,WPS110,WPS213,WPS231,WPS210 id_field='id', type_='cnt', ) - except Exception as ex: - logger.warning(self.style.WARNING(f"Unexpected behavior while sync data for owner"), owner_data, ex) + except Exception as processing_comments_ex: + logger.error( + msg="Failed comments", + args=(repo, processing_comments_ex), + ) session.close() diff --git a/contributors/utils/github_lib.py b/contributors/utils/github_lib.py index b402ee80..1444f3ed 100644 --- a/contributors/utils/github_lib.py +++ b/contributors/utils/github_lib.py @@ -29,11 +29,13 @@ class Accepted(GitHubError): class NoContent(GitHubError): """HTTP 204 Response.""" + class ClientErrorResponse(GitHubError): """HTTP 4xx Response.""" + class ServerErrorResponse(GitHubError): - """HTTP 5xx Response""" + """HTTP 5xx Response.""" class NoContributorsError(GitHubError): @@ -127,14 +129,31 @@ def get_whole_response_as_json(url, session=None): req = session or requests response = req.get(url, headers=get_headers()) response.raise_for_status() - if response.status_code == requests.codes.no_content: - raise NoContent("204 No Content", response=response) - elif response.status_code == requests.codes.accepted: - raise Accepted("202 Accepted. No cached data. Retry.") - elif response.status_code >= 500: - raise ServerErrorResponse("Server error response", response=response) - elif response.status_code >= 400: - raise ClientErrorResponse("Client error response.", response=response) + status_exceptions = { + requests.codes.no_content: ( + NoContent("204 No Content", response=response) + ), + requests.codes.accepted: ( + Accepted("202 Accepted. No cached data. Retry.", response=response) + ), + **dict.fromkeys( + range( + requests.codes.internal_server_error, + requests.codes.internal_server_error + 100, + ), + ServerErrorResponse("Server error response", response=response), + ), + **dict.fromkeys( + range( + requests.codes.bad_request, + requests.codes.bad_request + 100, + ), + ClientErrorResponse("Client error response", response=response), + ), + } + + if response.status_code in status_exceptions: + raise status_exceptions.get(response.status_code) return response.json() From 0c8bef2702d5a28dbd7ff1a37aacc9fc786cb9af Mon Sep 17 00:00:00 2001 From: Abdujabbar MIRKHALIKOV Date: Fri, 19 Apr 2024 21:31:20 +0500 Subject: [PATCH 3/3] Additional info for logs on fetchdata.py --- contributors/management/commands/fetchdata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contributors/management/commands/fetchdata.py b/contributors/management/commands/fetchdata.py index 7f8e6697..dbb5cdec 100644 --- a/contributors/management/commands/fetchdata.py +++ b/contributors/management/commands/fetchdata.py @@ -182,7 +182,7 @@ def handle( # noqa: C901,WPS110,WPS213,WPS231,WPS210 except Exception as processing_issues_exp: logger.error( msg="Failed processing issues and pull requests", - args=(repo, processing_issues_exp), + args=(owner, repo, processing_issues_exp), ) continue @@ -200,7 +200,7 @@ def handle( # noqa: C901,WPS110,WPS213,WPS231,WPS210 except Exception as processing_commits_ex: logger.error( msg="Failed processing commits", - args=(repo, processing_commits_ex), + args=(owner, repo, processing_commits_ex), ) continue @@ -216,7 +216,7 @@ def handle( # noqa: C901,WPS110,WPS213,WPS231,WPS210 except Exception as processing_comments_ex: logger.error( msg="Failed comments", - args=(repo, processing_comments_ex), + args=(owner, repo, processing_comments_ex), ) session.close()