Merge branch 'pylint'

.github/workflows/black.yml

@@ -1,12 +1,12 @@
-name: Lint
+name: Format
 
 on: [push, pull_request]
 
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: psf/black@stable
         with:
           options: "--check --verbose"

.github/workflows/mypy.yml

@@ -7,9 +7,9 @@ jobs:
     runs-on: ubuntu-latest
     name: Mypy
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
     - name: Set up Python 3.9
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v5
       with:
         python-version: 3.9
     - name: Install Dependencies

.github/workflows/pylint.yml

@@ -0,0 +1,18 @@
+name: PyLint
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: PyLint
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.9
+    - name: Install Dependencies
+      run: pip install pylint
+    - name: Run PyLint
+      run: pylint ./src

.pylintrc

@@ -0,0 +1,2 @@
+[MASTER]
+disable=missing-class-docstring,missing-module-docstring,missing-function-docstring

src/near_lake_framework/__init__.py

@@ -5,6 +5,7 @@
 
 import logging
 from typing import Optional
+from dataclasses import dataclass
 
 from aiobotocore.session import get_session
 
@@ -51,7 +52,7 @@ class LakeConfig:
     start_block_height: near_primitives.BlockHeight
     blocks_preload_pool_size: int = 200
 
-    def __init__(
+    def __init__(  # pylint: disable=too-many-arguments
         self,
         network: Network,
         aws_access_key_id: str,

src/near_lake_framework/near_primitives.py

@@ -10,16 +10,17 @@
 
 
 class BlockHeightField(mm.fields.Integer):
-    """Block Height is unsigned 64-bit integer, so it needs to be serialized as a string and get deserialized
-    to an integer type in Python.
+    """
+    Block Height is unsigned 64-bit integer, so it needs to be serialized
+    as a string and get deserialized to an integer type in Python.
     """
 
     def __init__(self, *args, **kwargs):
-        return super().__init__(*args, **kwargs, as_string=True)
+        super().__init__(*args, **kwargs, as_string=True)
 
 
 @dataclass
-class BlockHeader(DataClassJsonMixin):
+class BlockHeader(DataClassJsonMixin):  # pylint: disable=too-many-instance-attributes
     epoch_id: CryptoHash
     next_epoch_id: CryptoHash
     hash: CryptoHash
@@ -63,7 +64,7 @@ class BlockHeader(DataClassJsonMixin):
 
 
 @dataclass
-class ChunkHeader(DataClassJsonMixin):
+class ChunkHeader(DataClassJsonMixin):  # pylint: disable=too-many-instance-attributes
     chunk_hash: CryptoHash
     prev_block_hash: CryptoHash
     outcome_root: CryptoHash

src/near_lake_framework/s3_fetchers.py

@@ -1,7 +1,7 @@
 import asyncio
 import logging
 import traceback
-from botocore.exceptions import ClientError
+from botocore.exceptions import ClientError, EndpointConnectionError
 
 from near_lake_framework import near_primitives
 from near_lake_framework.near_primitives import IndexerShard
@@ -17,7 +17,7 @@ async def list_blocks(
         Bucket=s3_bucket_name,
         Delimiter="/",
         MaxKeys=number_of_blocks_requested,
-        StartAfter="{:012d}".format(start_from_block_height),
+        StartAfter=f"{start_from_block_height:012d}",
         RequestPayer="requester",
     )
 
@@ -32,7 +32,7 @@ async def fetch_streamer_message(
 ) -> near_primitives.StreamerMessage:
     response = await s3_client.get_object(
         Bucket=s3_bucket_name,
-        Key="{:012d}/block.json".format(block_height),
+        Key=f"{block_height:012d}/block.json",
         RequestPayer="requester",
     )
 
@@ -57,7 +57,7 @@ async def fetch_shard_or_retry(
     shard_id: int,
 ) -> near_primitives.IndexerShard:
     while True:
-        shard_key = "{:012d}/shard_{}.json".format(block_height, shard_id)
+        shard_key = f"{block_height:012d}/shard_{shard_id}.json"
         try:
             response = await s3_client.get_object(
                 Bucket=s3_bucket_name,
@@ -71,10 +71,17 @@ async def fetch_shard_or_retry(
             return near_primitives.IndexerShard.from_json(body)
         except ClientError as e:
             if e.response["Error"]["Code"] == "NoSuchKey":
-                logging.warning(
-                    "Failed to fetch shard {} - does not exist".format(shard_key)
-                )
+                logging.warning("Failed to fetch shard %s - doesn't exist", shard_key)
             else:
                 traceback.print_exc()
-        except Exception:
+        except EndpointConnectionError as e:
+            logging.error(
+                "EndpointConnectionError while fetching shard %s: %s", shard_key, e
+            )
+            traceback.print_exc()
+        except asyncio.TimeoutError as e:
+            logging.error("TimeoutError while fetching shard %s: %s", shard_key, e)
+            traceback.print_exc()
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            logging.error("Unexpected error while fetching shard %s: %s", shard_key, e)
             traceback.print_exc()