Skip to content

Commit

Permalink
Fix scanner
Browse files Browse the repository at this point in the history
  • Loading branch information
@tvorogme
tvorogme committed Apr 11, 2024
1 parent 732fbd5 commit 43646e4
Show file tree
Hide file tree
Showing 2 changed files with 133 additions and 119 deletions.
2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def finalize_options(self):

setup(
name="tonpy" if not IS_DEV else "tonpy-dev",
version="0.0.0.1.2b0" if not IS_DEV else "0.0.0.4.1b1",
version="0.0.0.1.2b0" if not IS_DEV else "0.0.0.4.1c1",
author="Disintar LLP",
author_email="andrey@head-labs.com",
description="Types / API for TON blockchain",
Expand Down
250 changes: 132 additions & 118 deletions src/tonpy/blockscanner/blockscanner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,137 +227,147 @@ def load_process_shard(shards_chunk,
lcparams,
loglevel,
parse_txs_over_ls=False):
answer = []
try:
answer = []

lcparams = json.loads(lcparams)
lc = LiteClient(**lcparams)
lcparams = json.loads(lcparams)
lc = LiteClient(**lcparams)

def process_shard(x, prev_data=None, lc=None):
if prev_data is None:
prev_data = []

if isinstance(x, BlockId):
block_id = lc.lookup_block(x).blk_id
else:
block_id = x
x = x.id

current_full_block = lc.get_block(block_id)

# It's stored in blockExtra
block = Block().cell_unpack(current_full_block)
block_info = BlockInfo().cell_unpack(block.info, True)
block_extra = BlockExtra().cell_unpack(block.extra, False)

rand_seed = int(block_extra.rand_seed, 2)
prev_key_block_seqno = block_info.prev_key_block_seqno
right_shard = None
if block_info.after_merge:
left = block_info.prev_ref.prev1
left_shard = BlockIdExt(BlockId(x.workchain,
shard_child(x.shard, True),
left.seq_no),
root_hash=int(left.root_hash, 2), file_hash=int(left.file_hash, 2))

right = block_info.prev_ref.prev2
right_shard = BlockIdExt(BlockId(x.workchain,
shard_child(x.shard, False),
right.seq_no),
root_hash=int(right.root_hash, 2), file_hash=int(right.file_hash, 2))

if left_shard.id not in known_shards and block_id.id not in stop_shards:
prev_data = [*process_shard(left_shard, lc=lc, prev_data=[]), *prev_data]

if right_shard.id not in known_shards and block_id.id not in stop_shards:
prev_data = [*process_shard(right_shard, lc=lc, prev_data=[]), *prev_data]
def process_shard(x, prev_data=None, lc=None):
if prev_data is None:
prev_data = []

else:
prev = block_info.prev_ref.prev
if isinstance(x, BlockId):
block_id = lc.lookup_block(x).blk_id
else:
block_id = x
x = x.id

current_full_block = lc.get_block(block_id)

# It's stored in blockExtra
block = Block().cell_unpack(current_full_block)
block_info = BlockInfo().cell_unpack(block.info, True)
block_extra = BlockExtra().cell_unpack(block.extra, False)

rand_seed = int(block_extra.rand_seed, 2)
prev_key_block_seqno = block_info.prev_key_block_seqno
right_shard = None
if block_info.after_merge:
left = block_info.prev_ref.prev1
left_shard = BlockIdExt(BlockId(x.workchain,
shard_child(x.shard, True),
left.seq_no),
root_hash=int(left.root_hash, 2), file_hash=int(left.file_hash, 2))

right = block_info.prev_ref.prev2
right_shard = BlockIdExt(BlockId(x.workchain,
shard_child(x.shard, False),
right.seq_no),
root_hash=int(right.root_hash, 2), file_hash=int(right.file_hash, 2))

if left_shard.id not in known_shards and block_id.id not in stop_shards:
prev_data = [*process_shard(left_shard, lc=lc, prev_data=[]), *prev_data]

if right_shard.id not in known_shards and block_id.id not in stop_shards:
prev_data = [*process_shard(right_shard, lc=lc, prev_data=[]), *prev_data]

if block_info.after_split:
left_shard = BlockIdExt(BlockId(x.workchain, shard_parent(x.shard), prev.seq_no),
root_hash=int(prev.root_hash, 2), file_hash=int(prev.file_hash, 2))
else:
left_shard = BlockIdExt(BlockId(x.workchain, x.shard, prev.seq_no),
root_hash=int(prev.root_hash, 2), file_hash=int(prev.file_hash, 2))
prev = block_info.prev_ref.prev

if left_shard.id not in known_shards and block_id.id not in stop_shards:
prev_data = [*process_shard(left_shard, lc=lc, prev_data=[]), *prev_data]
if block_info.after_split:
left_shard = BlockIdExt(BlockId(x.workchain, shard_parent(x.shard), prev.seq_no),
root_hash=int(prev.root_hash, 2), file_hash=int(prev.file_hash, 2))
else:
left_shard = BlockIdExt(BlockId(x.workchain, x.shard, prev.seq_no),
root_hash=int(prev.root_hash, 2), file_hash=int(prev.file_hash, 2))

if left_shard.id not in known_shards and block_id.id not in stop_shards:
prev_data = [*process_shard(left_shard, lc=lc, prev_data=[]), *prev_data]

del lc
new_data = {
'block_id': block_id,
'rand_seed': rand_seed,
'gen_utime': block_info.gen_utime,
'prev_key_block_seqno': prev_key_block_seqno,
'prev_block_left': left_shard,
'prev_block_right': right_shard,
'master': block_info.master_ref.master.seq_no,
'account_blocks': convert_account_blocks_to_txs(
block_extra.account_blocks) if not parse_txs_over_ls else None
}

del lc
new_data = {
'block_id': block_id,
'rand_seed': rand_seed,
'gen_utime': block_info.gen_utime,
'prev_key_block_seqno': prev_key_block_seqno,
'prev_block_left': left_shard,
'prev_block_right': right_shard,
'master': block_info.master_ref.master.seq_no,
'account_blocks': convert_account_blocks_to_txs(
block_extra.account_blocks) if not parse_txs_over_ls else None
}

return [new_data, *prev_data]
return [new_data, *prev_data]

if loglevel > 1:
shards_chunk = tqdm(shards_chunk, desc="Load shards")
if loglevel > 1:
shards_chunk = tqdm(shards_chunk, desc="Load shards")

for shard in shards_chunk:
answer.extend(process_shard(shard, lc=lc))
for shard in shards_chunk:
answer.extend(process_shard(shard, lc=lc))

return answer
return answer
except Exception as e:
logger.error(f"{e}")
logger.error(f"{traceback.format_exc()}")
return None


@curry
def process_mc_blocks(seqnos, lcparams, loglevel, parse_txs_over_ls):
lcparams = json.loads(lcparams)
lc = LiteClient(**lcparams)

answer = []

if loglevel > 1:
seqnos = tqdm(seqnos, desc="Load MCs")

for i in seqnos:
num_errs = 0

while True:
try:
block_id = lc.lookup_block(BlockId(-1, 0x8000000000000000, i)).blk_id
current_full_block = lc.get_block(block_id)
block = Block().cell_unpack(current_full_block)
block_info = BlockInfo().cell_unpack(block.info, True)
block_extra = BlockExtra().cell_unpack(block.extra, False)

rand_seed = int(block_extra.rand_seed, 2)
prev_key_block_seqno = block_info.prev_key_block_seqno

# todo: add prev hash

answer.append({
'block_id': block_id,
'shards': lc.get_all_shards_info(block_id),
'rand_seed': rand_seed,
'gen_utime': block_info.gen_utime,
'prev_key_block_seqno': prev_key_block_seqno,
'gen_utime': block_info.gen_utime,
'account_blocks': convert_account_blocks_to_txs(
block_extra.account_blocks) if not parse_txs_over_ls else None
})

break
except Exception as e:
num_errs += 1
if num_errs > 600:
logger.error(
f"Error in process_mc_blocks, block: (-1, 0x8000000000000000, {i}): {e}, {tb.format_exc()}")

if num_errs > 3000:
raise e

sleep(0.1)
del lc
return answer
try:
lcparams = json.loads(lcparams)
lc = LiteClient(**lcparams)

answer = []

if loglevel > 1:
seqnos = tqdm(seqnos, desc="Load MCs")

for i in seqnos:
num_errs = 0

while True:
try:
block_id = lc.lookup_block(BlockId(-1, 0x8000000000000000, i)).blk_id
current_full_block = lc.get_block(block_id)
block = Block().cell_unpack(current_full_block)
block_info = BlockInfo().cell_unpack(block.info, True)
block_extra = BlockExtra().cell_unpack(block.extra, False)

rand_seed = int(block_extra.rand_seed, 2)
prev_key_block_seqno = block_info.prev_key_block_seqno

# todo: add prev hash

answer.append({
'block_id': block_id,
'shards': lc.get_all_shards_info(block_id),
'rand_seed': rand_seed,
'gen_utime': block_info.gen_utime,
'prev_key_block_seqno': prev_key_block_seqno,
'gen_utime': block_info.gen_utime,
'account_blocks': convert_account_blocks_to_txs(
block_extra.account_blocks) if not parse_txs_over_ls else None
})

break
except Exception as e:
num_errs += 1
if num_errs > 600:
logger.error(
f"Error in process_mc_blocks, block: (-1, 0x8000000000000000, {i}): {e}, {tb.format_exc()}")

if num_errs > 3000:
raise e

sleep(0.1)
del lc
return answer
except Exception as e:
logger.error(f"{e}")
logger.error(f"{traceback.format_exc()}")
return None


class BlockScanner(Thread):
Expand Down Expand Up @@ -477,6 +487,8 @@ def load_mcs(self, from_, to_):
results = tqdm(results, desc="Download MC blocks", total=len(mc_seqnos_chunks))

for result in results:
if result is None:
raise ValueError(f"Invalid result in MC blocks")
mc_data.extend(result)
# todo: check hashes
return mc_data
Expand All @@ -492,9 +504,11 @@ def load_process_shard(self, known_shards, stop_shards):
known_shards_chunks)

if self.loglevel > 1:
results = tqdm(results, desc="Download shards", total=len(known_shards_chunks))
results = tqdm(results, desc=f"Download shards / {p}", total=len(known_shards_chunks))

for result in results:
if result is None:
raise ValueError(f"Invalid result in shards process")
shards_data.extend(result)

return shards_data
Expand Down

0 comments on commit 43646e4

Please sign in to comment.