Merge pull request #404 from Abdujabbar/master

Fix to avoid fetchdata process, added try/except block and custom exceptions

contributors/management/commands/fetchdata.py

@@ -169,35 +169,55 @@ def handle(  # noqa: C901,WPS110,WPS213,WPS231,WPS210
                 if language:
                     label, _ = Label.objects.get_or_create(name=language)
                     repo.labels.add(label)
-
                 logger.info("Processing issues and pull requests")
-                create_contributions(
-                    repo,
-                    github.get_repo_issues(owner, repo, session),
-                    user_field='user',
-                    id_field='id',
-                    type_='iss',
-                )
+
+                try:
+                    create_contributions(
+                        repo,
+                        github.get_repo_issues(owner, repo, session),
+                        user_field='user',
+                        id_field='id',
+                        type_='iss',
+                    )
+                except Exception as processing_issues_exp:
+                    logger.error(
+                        msg="Failed processing issues and pull requests",
+                        args=(owner, repo, processing_issues_exp),
+                    )
+                    continue
 
                 logger.info("Processing commits")
-                create_contributions(
-                    repo,
-                    github.get_repo_commits_except_merges(
-                        owner, repo, session=session,
-                    ),
-                    user_field='author',
-                    id_field='sha',
-                    type_='cit',
-                )
+                try:
+                    create_contributions(
+                        repo,
+                        github.get_repo_commits_except_merges(
+                            owner, repo, session=session,
+                        ),
+                        user_field='author',
+                        id_field='sha',
+                        type_='cit',
+                    )
+                except Exception as processing_commits_ex:
+                    logger.error(
+                        msg="Failed processing commits",
+                        args=(owner, repo, processing_commits_ex),
+                    )
+                    continue
 
                 logger.info("Processing comments")
-                create_contributions(
-                    repo,
-                    github.get_all_types_of_comments(owner, repo, session),
-                    user_field='user',
-                    id_field='id',
-                    type_='cnt',
-                )
+                try:
+                    create_contributions(
+                        repo,
+                        github.get_all_types_of_comments(owner, repo, session),
+                        user_field='user',
+                        id_field='id',
+                        type_='cnt',
+                    )
+                except Exception as processing_comments_ex:
+                    logger.error(
+                        msg="Failed comments",
+                        args=(owner, repo, processing_comments_ex),
+                    )
 
         session.close()
 

contributors/utils/github_lib.py

@@ -30,6 +30,14 @@ class NoContent(GitHubError):
     """HTTP 204 Response."""
 
 
+class ClientErrorResponse(GitHubError):
+    """HTTP 4xx Response."""
+
+
+class ServerErrorResponse(GitHubError):
+    """HTTP 5xx Response."""
+
+
 class NoContributorsError(GitHubError):
     """A repository has no contributors."""
 
@@ -121,10 +129,32 @@ def get_whole_response_as_json(url, session=None):
     req = session or requests
     response = req.get(url, headers=get_headers())
     response.raise_for_status()
-    if response.status_code == requests.codes.no_content:
-        raise NoContent("204 No Content", response=response)
-    elif response.status_code == requests.codes.accepted:
-        raise Accepted("202 Accepted. No cached data. Retry.")
+    status_exceptions = {
+        requests.codes.no_content: (
+            NoContent("204 No Content", response=response)
+        ),
+        requests.codes.accepted: (
+            Accepted("202 Accepted. No cached data. Retry.", response=response)
+        ),
+        **dict.fromkeys(
+            range(
+                requests.codes.internal_server_error,
+                requests.codes.internal_server_error + 100,
+            ),
+            ServerErrorResponse("Server error response", response=response),
+        ),
+        **dict.fromkeys(
+            range(
+                requests.codes.bad_request,
+                requests.codes.bad_request + 100,
+            ),
+            ClientErrorResponse("Client error response", response=response),
+        ),
+    }
+
+    if response.status_code in status_exceptions:
+        raise status_exceptions.get(response.status_code)
+
     return response.json()