From 57842ad84247c7bd48010a9d009ddc8dba2c95c0 Mon Sep 17 00:00:00 2001
From: naterush <narush@wharton.upenn.edu>
Date: Fri, 9 Feb 2018 00:10:54 -0500
Subject: [PATCH 01/10] add v1 binary blockchain sharding

---
 Makefile                                      |   3 +
 casper/protocols/sharding/__init__.py         |   0
 casper/protocols/sharding/block.py            |  57 +++++++
 casper/protocols/sharding/forkchoice.py       |  88 +++++++++++
 .../protocols/sharding/sharding_plot_tool.py  | 106 +++++++++++++
 .../protocols/sharding/sharding_protocol.py   |  56 +++++++
 casper/protocols/sharding/sharding_view.py    | 139 ++++++++++++++++++
 simulations/utils.py                          |   5 +-
 8 files changed, 453 insertions(+), 1 deletion(-)
 create mode 100644 casper/protocols/sharding/__init__.py
 create mode 100644 casper/protocols/sharding/block.py
 create mode 100644 casper/protocols/sharding/forkchoice.py
 create mode 100644 casper/protocols/sharding/sharding_plot_tool.py
 create mode 100644 casper/protocols/sharding/sharding_protocol.py
 create mode 100644 casper/protocols/sharding/sharding_view.py

diff --git a/Makefile b/Makefile
index 5d6aa72..c5cff98 100644
--- a/Makefile
+++ b/Makefile
@@ -30,3 +30,6 @@ run-nofinal:
 
 run-binary:
 	venv/bin/python casper.py rand --protocol binary --report-interval 3
+
+run-sharding:
+	venv/bin/python casper.py rand --protocol sharding --validators 14 --report-interval 3
diff --git a/casper/protocols/sharding/__init__.py b/casper/protocols/sharding/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/casper/protocols/sharding/block.py b/casper/protocols/sharding/block.py
new file mode 100644
index 0000000..62ca217
--- /dev/null
+++ b/casper/protocols/sharding/block.py
@@ -0,0 +1,57 @@
+"""The block module implements the message data structure for a sharded blockchain"""
+from casper.message import Message
+
+
+class Block(Message):
+    """Message data structure for a sharded blockchain"""
+
+    @classmethod
+    def is_valid_estimate(cls, estimate):
+        if not isinstance(estimate, dict):
+            return False
+        if not isinstance(estimate['prev_blocks'], set):
+            return False
+        if not isinstance(estimate['shard_ids'], set):
+            return False
+        return True
+
+    def on_shard(self, shard_id):
+        return shard_id in self.estimate['shard_ids']
+
+    def prev_block(self, shard_id):
+        """Returns the previous block, which is on the shard_id
+        Throws a KeyError if there is no previous block"""
+        if shard_id not in self.estimate['shard_ids']:
+            raise KeyError("No previous block on that shard")
+
+        for block in self.estimate['prev_blocks']:
+            if block is None:
+                return None
+
+            if block.on_shard(shard_id):
+                return block
+
+        raise KeyError("Should have found previous block on shard!")
+
+    @property
+    def is_merge_block(self):
+        return len(self.estimate['shard_ids']) == 2
+
+    def conflicts_with(self, message):
+        """Returns true if self is not in the prev blocks of other_message"""
+        assert isinstance(message, Block), "...expected a block"
+
+        return not self.is_in_blockchain(message, '')
+
+    def is_in_blockchain(self, block, shard_id):
+        """Could be a zero generation ancestor!"""
+        if not block:
+            return False
+
+        if not block.on_shard(shard_id):
+            return False
+
+        if self == block:
+            return True
+
+        return self.is_in_blockchain(block.prev_block(shard_id), shard_id)
diff --git a/casper/protocols/sharding/forkchoice.py b/casper/protocols/sharding/forkchoice.py
new file mode 100644
index 0000000..237fa88
--- /dev/null
+++ b/casper/protocols/sharding/forkchoice.py
@@ -0,0 +1,88 @@
+"""The forkchoice module implements the estimator function a blockchain"""
+
+
+def get_max_weight_indexes(scores):
+    """Returns the keys that map to the max value in a dict.
+    The max value must be greater than zero."""
+
+    max_score = max(scores.values())
+
+    assert max_score != 0, "max_score of a block should never be zero"
+
+    max_weight_estimates = {e for e in scores if scores[e] == max_score}
+
+    return max_weight_estimates
+
+
+def get_scores(starting_block, latest_messages, shard_id):
+    """Returns a dict of block => weight"""
+    scores = dict()
+
+    for validator, current_block in latest_messages.items():
+        if not current_block.on_shard(shard_id):
+            continue
+
+        while current_block and current_block != starting_block:
+            scores[current_block] = scores.get(current_block, 0) + validator.weight
+            current_block = current_block.prev_block(shard_id)
+
+    return scores
+
+
+
+
+
+
+
+
+    # ah. So
+
+
+def get_shard_fork_choice(starting_block, children, latest_messages, shard_id):
+    """Get the forkchoice for a specific shard"""
+
+    scores = get_scores(starting_block, latest_messages, shard_id)
+
+    best_block = starting_block
+    while best_block in children:
+        curr_scores = dict()
+        max_score = 0
+        for child in children[best_block]:
+            if not child.on_shard(shard_id):
+                continue # we only select children on the same shard
+            curr_scores[child] = scores.get(child, 0)
+            max_score = max(curr_scores[child], max_score)
+
+        # If no child on shard, or 0 weight block, stop
+        if max_score == 0:
+            break
+
+        max_weight_children = get_max_weight_indexes(curr_scores)
+
+        assert len(max_weight_children) == 1, "... there should be no ties!"
+
+        best_block = max_weight_children.pop()
+
+    return best_block
+
+
+def get_all_shards_fork_choice(starting_blocks, children, latest_messages_on_shard):
+    """Returns a dict of shard_id -> forkchoice.
+    Starts from starting block for shard, and stops when it reaches tip"""
+
+    # for any shard we have latest messages on, we should have a starting block
+    for key in starting_blocks.keys():
+        assert key in latest_messages_on_shard
+    for key in latest_messages_on_shard.keys():
+        assert key in latest_messages_on_shard
+
+    shards_forkchoice = {
+        shard_id: get_shard_fork_choice(
+            starting_blocks[shard_id],
+            children,
+            latest_messages_on_shard[shard_id],
+            shard_id
+        ) for shard_id in starting_blocks
+        }
+
+    return shards_forkchoice
diff --git a/casper/protocols/sharding/sharding_plot_tool.py b/casper/protocols/sharding/sharding_plot_tool.py
new file mode 100644
index 0000000..df81646
--- /dev/null
+++ b/casper/protocols/sharding/sharding_plot_tool.py
@@ -0,0 +1,106 @@
+"""The blockchain plot tool implements functions for plotting blockchain data structures"""
+
+from casper.plot_tool import PlotTool
+from casper.safety_oracles.clique_oracle import CliqueOracle
+import casper.utils as utils
+
+
+class ShardingPlotTool(PlotTool):
+    """The module contains functions for plotting a blockchain data structure"""
+
+    def __init__(self, display, save, view, validator_set):
+        super().__init__(display, save, 's')
+        self.view = view
+        self.validator_set = validator_set
+        self.starting_blocks = self.view.starting_blocks
+        self.message_fault_tolerance = dict()
+
+        self.blockchain = []
+        self.communications = []
+
+        self.block_fault_tolerance = {}
+        self.message_labels = {}
+        self.justifications = {
+            validator: []
+            for validator in validator_set
+        }
+
+
+    def update(self, new_messages=None):
+        """Updates displayable items with new messages and paths"""
+        return
+
+        if new_messages is None:
+            new_messages = []
+
+        self._update_new_justifications(new_messages)
+        self._update_blockchain(new_messages)
+        self._update_block_fault_tolerance()
+        self._update_message_labels(new_messages)
+
+    def plot(self):
+        """Builds relevant edges to display and creates next viewgraph using them"""
+        return
+        best_chain_edge = self.get_best_chain()
+
+        validator_chain_edges = self.get_validator_chains()
+
+        edgelist = []
+        edgelist.append(utils.edge(self.blockchain, 2, 'grey', 'solid'))
+        edgelist.append(utils.edge(self.communications, 1, 'black', 'dotted'))
+        edgelist.append(best_chain_edge)
+        edgelist.extend(validator_chain_edges)
+
+        self.next_viewgraph(
+            self.view,
+            self.validator_set,
+            edges=edgelist,
+            message_colors=self.block_fault_tolerance,
+            message_labels=self.message_labels
+        )
+
+    def get_best_chain(self):
+        """Returns an edge made of the global forkchoice to genesis"""
+        best_message = self.view.estimate()
+        best_chain = utils.build_chain(best_message, None)[:-1]
+        return utils.edge(best_chain, 5, 'red', 'solid')
+
+    def get_validator_chains(self):
+        """Returns a list of edges main from validators current forkchoice to genesis"""
+        vals_chain_edges = []
+        for validator in self.validator_set:
+            chain = utils.build_chain(validator.my_latest_message(), None)[:-1]
+            vals_chain_edges.append(utils.edge(chain, 2, 'blue', 'solid'))
+
+        return vals_chain_edges
+
+    def _update_new_justifications(self, new_messages):
+        for message in new_messages:
+            sender = message.sender
+            for validator in message.justification:
+                last_message = self.view.justified_messages[message.justification[validator]]
+                # only show if new justification
+                if last_message not in self.justifications[sender]:
+                    self.communications.append([last_message, message])
+                    self.justifications[sender].append(last_message)
+
+    def _update_blockchain(self, new_messages):
+        for message in new_messages:
+            if message.estimate is not None:
+                self.blockchain.append([message, message.estimate])
+
+    def _update_message_labels(self, new_messages):
+        for message in new_messages:
+            self.message_labels[message] = message.sequence_number
+
+    def _update_block_fault_tolerance(self):
+        tip = self.view.estimate()
+
+        while tip and self.block_fault_tolerance.get(tip, 0) != len(self.validator_set) - 1:
+            oracle = CliqueOracle(tip, self.view, self.validator_set)
+            fault_tolerance, num_node_ft = oracle.check_estimate_safety()
+
+            if fault_tolerance > 0:
+                self.block_fault_tolerance[tip] = num_node_ft
+
+            tip = tip.estimate
diff --git a/casper/protocols/sharding/sharding_protocol.py b/casper/protocols/sharding/sharding_protocol.py
new file mode 100644
index 0000000..d499445
--- /dev/null
+++ b/casper/protocols/sharding/sharding_protocol.py
@@ -0,0 +1,56 @@
+from casper.protocols.sharding.sharding_view import ShardingView
+from casper.protocols.sharding.block import Block
+from casper.protocols.sharding.sharding_plot_tool import ShardingPlotTool
+from casper.protocol import Protocol
+
+
+class ShardingProtocol(Protocol):
+    View = ShardingView
+    Message = Block
+    PlotTool = ShardingPlotTool
+
+    shard_genesis_blocks = dict()
+
+
+    # Shard ID's look like this:
+    #       ''
+    #     /    \
+    #   '0'    '1'
+    #  /  \    /  \
+    #'00''01''10''11'
+    #
+
+    # Blocks can be merge mined between shards if
+    # there is an edge between shards
+    # That is, for ids shard_1 and shard_2, there can be a merge block if
+    # abs(len(shard_1) - len(shard_2)) = 1 AND
+    # for i in range(min(len(shard_1), len(shard_2))):
+    #    shard_1[i] = shard_2[i]
+
+    genesis_block = None
+    current_shard = 0
+    num_validators_assigned = 0 # for now, assign at least 3 validators!
+
+    @classmethod
+    def initial_message(cls, validator):
+        """Returns a dict from shard_id -> shard genesis block"""
+        # hard coded for now (3 is a arbitrary :-) )
+
+        if cls.num_validators_assigned == 0:
+            if '' not in cls.shard_genesis_blocks:
+                estimate = {'prev_blocks': set([None]), 'shard_ids': set([''])}
+                cls.shard_genesis_blocks[''] = Block(estimate, dict(), validator, -1, 0)
+            cls.num_validators_assigned = 1
+            return cls.shard_genesis_blocks['']
+        elif cls.num_validators_assigned == 1:
+            if '0' not in cls.shard_genesis_blocks:
+                estimate = {'prev_blocks': set([None]), 'shard_ids': set(['0'])}
+                cls.shard_genesis_blocks['0'] = Block(estimate, dict(), validator, -1, 0)
+            cls.num_validators_assigned = 2
+            return cls.shard_genesis_blocks['0']
+        else:
+            if '1' not in cls.shard_genesis_blocks:
+                estimate = {'prev_blocks': set([None]), 'shard_ids': set(['1'])}
+                cls.shard_genesis_blocks['1'] = Block(estimate, dict(), validator, -1, 0)
+            cls.num_validators_assigned = 0
+            return cls.shard_genesis_blocks['1']
diff --git a/casper/protocols/sharding/sharding_view.py b/casper/protocols/sharding/sharding_view.py
new file mode 100644
index 0000000..b1c83e2
--- /dev/null
+++ b/casper/protocols/sharding/sharding_view.py
@@ -0,0 +1,139 @@
+"""The blockchain view module extends a view for blockchain data structures """
+import random as r
+
+from casper.safety_oracles.clique_oracle import CliqueOracle
+from casper.abstract_view import AbstractView
+import casper.protocols.sharding.forkchoice as forkchoice
+
+
+class ShardingView(AbstractView):
+    """A view class that also keeps track of a last_finalized_block and children"""
+    def __init__(self, messages=None, shard_genesis_block=None):
+        self.children = dict()
+
+        self.starting_blocks = dict() # shard_id -> current starting block for the forkchoice
+        if shard_genesis_block:
+            assert not shard_genesis_block.is_merge_block
+            shard_id = r.sample(shard_genesis_block.estimate['shard_ids'], 1)[0]
+            self.starting_blocks[shard_id] = shard_genesis_block
+        # note: the shard_genesis_block should _not_ be merge mined!
+
+        self.latest_messages_on_shard = dict() # shard_id -> validator -> message
+
+        super().__init__(messages)
+
+    def estimate(self):
+        """Returns the current forkchoice in this view"""
+        print("\n\n Running Forkchoice")
+        self.update_starting_blocks()
+
+        shards_forkchoice = forkchoice.get_all_shards_fork_choice(
+            self.starting_blocks,
+            self.children,
+            self.latest_messages_on_shard
+        )
+
+        self.check_forkchoice_atomicity(shards_forkchoice)
+
+        # for now, randomly build somewhere!
+        shards_to_build_on = r.choice([{''}, {'0'}, {'1'}, {'', '0'}, {'', '1'}])
+        return {'prev_blocks': {shards_forkchoice[shard_id] for shard_id in shards_to_build_on},
+                'shard_ids': shards_to_build_on}
+
+    def check_forkchoice_atomicity(self, shards_forkchoice):
+        # assert the atomicity things about the shards_forkchoice
+        bast_chain_tip = self.forkchoice_on_shard('')
+        zero_chain_tip = self.forkchoice_on_shard('0')
+        one_chain_tip = self.forkchoice_on_shard('1')
+
+        zero_merge_block = self.previous_merge_block_on_shard(bast_chain_tip, '', '0')
+        one_merge_block = self.previous_merge_block_on_shard(bast_chain_tip, '', '1')
+
+        if zero_merge_block:
+            assert zero_merge_block.is_in_blockchain(zero_chain_tip, '0')
+            print("Checked shard '' and 0 atomicity: passed")
+        if one_merge_block:
+            assert one_merge_block.is_in_blockchain(one_chain_tip, '1')
+            print("Checked shard '' and 1 atomicity: passed")
+
+
+    def previous_merge_block_on_shard(self, starting_block, block_shard_id, merge_shard):
+        assert starting_block.on_shard(block_shard_id)
+        current_block = starting_block
+        while current_block:
+            if current_block.on_shard(merge_shard):
+                assert current_block.is_merge_block
+                return current_block
+            current_block = current_block.prev_block(block_shard_id)
+        return None
+
+
+    def update_starting_blocks(self):
+        base_chain_tip = self.forkchoice_on_shard('')
+
+        # get the most recent merge block on shard '0'
+        merge_block = self.previous_merge_block_on_shard(base_chain_tip, '', '0')
+        if merge_block:
+            self.starting_blocks['0'] = merge_block
+
+        # get the most recent merge block on shard '0'
+        merge_block = self.previous_merge_block_on_shard(base_chain_tip, '', '1')
+        if merge_block:
+            self.starting_blocks['1'] = merge_block
+
+    def forkchoice_on_shard(self, shard_id):
+        return forkchoice.get_shard_fork_choice(
+            self.starting_blocks[shard_id],
+            self.children,
+            self.latest_messages_on_shard[shard_id],
+            shard_id
+        )
+
+
+    def update_safe_estimates(self, validator_set):
+        """Checks safety on messages in views forkchoice, and updates last_finalized_block"""
+        pass
+
+        # check the safety of the top shard!
+        tip = self.forkchoice_on_shard('')
+
+        while tip and tip != self.starting_blocks['']:
+            oracle = CliqueOracle(tip, self, validator_set)
+            fault_tolerance, _ = oracle.check_estimate_safety()
+
+            if fault_tolerance > 0:
+                self.starting_blocks[''] = tip
+                return tip
+
+            tip = tip.prev_block('')
+
+    def _update_protocol_specific_view(self, message):
+        """Given a now justified message, updates children and when_recieved"""
+        assert message.hash in self.justified_messages, "...should not have seen message!"
+
+        # set starting messages! (should only happen initially)
+        if None in message.estimate['prev_blocks']:
+            assert not message.is_merge_block
+            shard_id = r.sample(message.estimate['shard_ids'], 1)[0]
+            print("Adding starting block for {}".format(shard_id))
+            self.starting_blocks[shard_id] = message
+
+        # update the latest_messages
+        for shard_id in message.estimate['shard_ids']:
+            if shard_id not in self.latest_messages_on_shard:
+                self.latest_messages_on_shard[shard_id] = dict()
+            latest_messages = self.latest_messages_on_shard[shard_id]
+            if message.sender not in latest_messages:
+                latest_messages[message.sender] = message
+            elif latest_messages[message.sender].sequence_number < message.sequence_number:
+                latest_messages[message.sender] = message
+
+            assert message.sender in self.latest_messages_on_shard[shard_id]
+
+        # update children dictonary
+        for parent in message.estimate['prev_blocks']:
+            if parent not in self.children:
+                self.children[parent] = set()
+            self.children[parent].add(message)
+
+        # preferably, would update starting blocks here.
diff --git a/simulations/utils.py b/simulations/utils.py
index 7f6bc52..8105458 100644
--- a/simulations/utils.py
+++ b/simulations/utils.py
@@ -13,11 +13,12 @@
 from casper.protocols.integer.integer_protocol import IntegerProtocol
 from casper.protocols.order.order_protocol import OrderProtocol
 from casper.protocols.concurrent.concurrent_protocol import ConcurrentProtocol
+from casper.protocols.sharding.sharding_protocol import ShardingProtocol
 from casper.validator_set import ValidatorSet
 
 MESSAGE_MODES = ['rand', 'rrob', 'full', 'nofinal']
 NETWORKS = ['no-delay', 'step', 'constant', 'linear', 'gaussian']
-PROTOCOLS = ['blockchain', 'binary', 'integer', 'order', 'concurrent']
+PROTOCOLS = ['blockchain', 'binary', 'integer', 'order', 'concurrent', 'sharding']
 
 
 def select_network(network):
@@ -44,6 +45,8 @@ def select_protocol(protocol):
         return IntegerProtocol
     if protocol == 'concurrent':
         return ConcurrentProtocol
+    if protocol == 'sharding':
+        return ShardingProtocol
 
 
 def message_maker(mode):

From 6c5163a2618793743b8e76b9bf9a477951d84813 Mon Sep 17 00:00:00 2001
From: naterush <narush@wharton.upenn.edu>
Date: Fri, 9 Feb 2018 12:39:25 -0500
Subject: [PATCH 02/10] minor lints

---
 casper/protocols/sharding/block.py             | 2 +-
 casper/protocols/sharding/forkchoice.py        | 9 ---------
 casper/protocols/sharding/sharding_protocol.py | 1 -
 casper/protocols/sharding/sharding_view.py     | 2 +-
 4 files changed, 2 insertions(+), 12 deletions(-)

diff --git a/casper/protocols/sharding/block.py b/casper/protocols/sharding/block.py
index 62ca217..afae3aa 100644
--- a/casper/protocols/sharding/block.py
+++ b/casper/protocols/sharding/block.py
@@ -19,7 +19,7 @@ def on_shard(self, shard_id):
         return shard_id in self.estimate['shard_ids']
 
     def prev_block(self, shard_id):
-        """Returns the previous block, which is on the shard_id
+        """Returns the previous block on the shard: shard_id
         Throws a KeyError if there is no previous block"""
         if shard_id not in self.estimate['shard_ids']:
             raise KeyError("No previous block on that shard")
diff --git a/casper/protocols/sharding/forkchoice.py b/casper/protocols/sharding/forkchoice.py
index 237fa88..95bb57d 100644
--- a/casper/protocols/sharding/forkchoice.py
+++ b/casper/protocols/sharding/forkchoice.py
@@ -29,15 +29,6 @@ def get_scores(starting_block, latest_messages, shard_id):
     return scores
 
 
-
-
-
-
-
-
-    # ah. So
-
-
 def get_shard_fork_choice(starting_block, children, latest_messages, shard_id):
     """Get the forkchoice for a specific shard"""
 
diff --git a/casper/protocols/sharding/sharding_protocol.py b/casper/protocols/sharding/sharding_protocol.py
index d499445..49c9c5f 100644
--- a/casper/protocols/sharding/sharding_protocol.py
+++ b/casper/protocols/sharding/sharding_protocol.py
@@ -35,7 +35,6 @@ class ShardingProtocol(Protocol):
     def initial_message(cls, validator):
         """Returns a dict from shard_id -> shard genesis block"""
         # hard coded for now (3 is a arbitrary :-) )
-
         if cls.num_validators_assigned == 0:
             if '' not in cls.shard_genesis_blocks:
                 estimate = {'prev_blocks': set([None]), 'shard_ids': set([''])}
diff --git a/casper/protocols/sharding/sharding_view.py b/casper/protocols/sharding/sharding_view.py
index b1c83e2..f3d328e 100644
--- a/casper/protocols/sharding/sharding_view.py
+++ b/casper/protocols/sharding/sharding_view.py
@@ -92,9 +92,9 @@ def forkchoice_on_shard(self, shard_id):
 
     def update_safe_estimates(self, validator_set):
         """Checks safety on messages in views forkchoice, and updates last_finalized_block"""
+        # check the safety of the top shard!
         pass
 
-        # check the safety of the top shard!
         tip = self.forkchoice_on_shard('')
 
         while tip and tip != self.starting_blocks['']:

From 26dfff98d5e98d037618186e39cc48d176262bcf Mon Sep 17 00:00:00 2001
From: naterush <narush@wharton.upenn.edu>
Date: Tue, 13 Feb 2018 11:43:33 -0500
Subject: [PATCH 03/10] update forkchoice to have unlimited number of shards

---
 .../protocols/sharding/sharding_protocol.py   | 49 +++++------
 casper/protocols/sharding/sharding_view.py    | 85 +++++++++++--------
 2 files changed, 75 insertions(+), 59 deletions(-)

diff --git a/casper/protocols/sharding/sharding_protocol.py b/casper/protocols/sharding/sharding_protocol.py
index 49c9c5f..9439f21 100644
--- a/casper/protocols/sharding/sharding_protocol.py
+++ b/casper/protocols/sharding/sharding_protocol.py
@@ -10,7 +10,8 @@ class ShardingProtocol(Protocol):
     PlotTool = ShardingPlotTool
 
     shard_genesis_blocks = dict()
-
+    curr_shard_idx = 0
+    curr_shard_ids = ['']
 
     # Shard ID's look like this:
     #       ''
@@ -27,29 +28,29 @@ class ShardingProtocol(Protocol):
     # for i in range(min(len(shard_1), len(shard_2))):
     #    shard_1[i] = shard_2[i]
 
-    genesis_block = None
-    current_shard = 0
-    num_validators_assigned = 0 # for now, assign at least 3 validators!
-
     @classmethod
     def initial_message(cls, validator):
         """Returns a dict from shard_id -> shard genesis block"""
-        # hard coded for now (3 is a arbitrary :-) )
-        if cls.num_validators_assigned == 0:
-            if '' not in cls.shard_genesis_blocks:
-                estimate = {'prev_blocks': set([None]), 'shard_ids': set([''])}
-                cls.shard_genesis_blocks[''] = Block(estimate, dict(), validator, -1, 0)
-            cls.num_validators_assigned = 1
-            return cls.shard_genesis_blocks['']
-        elif cls.num_validators_assigned == 1:
-            if '0' not in cls.shard_genesis_blocks:
-                estimate = {'prev_blocks': set([None]), 'shard_ids': set(['0'])}
-                cls.shard_genesis_blocks['0'] = Block(estimate, dict(), validator, -1, 0)
-            cls.num_validators_assigned = 2
-            return cls.shard_genesis_blocks['0']
-        else:
-            if '1' not in cls.shard_genesis_blocks:
-                estimate = {'prev_blocks': set([None]), 'shard_ids': set(['1'])}
-                cls.shard_genesis_blocks['1'] = Block(estimate, dict(), validator, -1, 0)
-            cls.num_validators_assigned = 0
-            return cls.shard_genesis_blocks['1']
+        shard_id = cls.get_new_shard_id()
+
+        estimate = {'prev_blocks': set([None]), 'shard_ids': set([shard_id])}
+        cls.shard_genesis_blocks[''] = Block(estimate, dict(), validator, -1, 0)
+
+        return cls.shard_genesis_blocks['']
+
+
+    @classmethod
+    def get_new_shard_id(cls):
+        new_id = cls.curr_shard_ids[cls.curr_shard_idx]
+        cls.curr_shard_idx += 1
+
+        if cls.curr_shard_idx == len(cls.curr_shard_ids):
+            new_ids = []
+            for shard_id in cls.curr_shard_ids:
+                new_ids.append(shard_id + '0')
+                new_ids.append(shard_id + '1')
+
+            cls.curr_shard_idx = 0
+            cls.curr_shard_ids = new_ids
+
+        return new_id
diff --git a/casper/protocols/sharding/sharding_view.py b/casper/protocols/sharding/sharding_view.py
index f3d328e..030a258 100644
--- a/casper/protocols/sharding/sharding_view.py
+++ b/casper/protocols/sharding/sharding_view.py
@@ -6,6 +6,7 @@
 import casper.protocols.sharding.forkchoice as forkchoice
 
 
+
 class ShardingView(AbstractView):
     """A view class that also keeps track of a last_finalized_block and children"""
     def __init__(self, messages=None, shard_genesis_block=None):
@@ -25,36 +26,63 @@ def __init__(self, messages=None, shard_genesis_block=None):
     def estimate(self):
         """Returns the current forkchoice in this view"""
         print("\n\n Running Forkchoice")
-        self.update_starting_blocks()
-
-        shards_forkchoice = forkchoice.get_all_shards_fork_choice(
-            self.starting_blocks,
-            self.children,
-            self.latest_messages_on_shard
-        )
+        shards_forkchoice = dict()
+
+        for shard_id in sorted(self.starting_blocks):
+            tip = forkchoice.get_shard_fork_choice(
+                self.starting_blocks[shard_id],
+                self.children,
+                self.latest_messages_on_shard[shard_id],
+                shard_id
+            )
+            shards_forkchoice[shard_id] = tip
+
+            left_child_shard = shard_id + '0'
+            if left_child_shard in self.starting_blocks:
+                left_merge_block = self.previous_merge_block_on_shard(tip, shard_id, left_child_shard)
+                if left_merge_block:
+                    self.starting_blocks[left_child_shard] = left_merge_block
+
+            right_child_shard = shard_id + '1'
+            if right_child_shard in self.starting_blocks:
+                right_merge_block = self.previous_merge_block_on_shard(tip, shard_id, right_child_shard)
+                if right_merge_block:
+                    self.starting_blocks[right_child_shard] = right_merge_block
 
         self.check_forkchoice_atomicity(shards_forkchoice)
 
-        # for now, randomly build somewhere!
-        shards_to_build_on = r.choice([{''}, {'0'}, {'1'}, {'', '0'}, {'', '1'}])
+        # pay no attenting to the code behind the curtain ::))
+        shards_to_build_on = [r.choice([key for key in self.starting_blocks.keys()])]
+        if (r.randint(0, 1) == 1):
+            child = str(r.randint(0, 1))
+            if shards_to_build_on[0] + child in self.starting_blocks:
+                shards_to_build_on.append(shards_to_build_on[0] + child)
+
+        shards_to_build_on = set(shards_to_build_on)
+
         return {'prev_blocks': {shards_forkchoice[shard_id] for shard_id in shards_to_build_on},
                 'shard_ids': shards_to_build_on}
 
     def check_forkchoice_atomicity(self, shards_forkchoice):
         # assert the atomicity things about the shards_forkchoice
-        bast_chain_tip = self.forkchoice_on_shard('')
-        zero_chain_tip = self.forkchoice_on_shard('0')
-        one_chain_tip = self.forkchoice_on_shard('1')
-
-        zero_merge_block = self.previous_merge_block_on_shard(bast_chain_tip, '', '0')
-        one_merge_block = self.previous_merge_block_on_shard(bast_chain_tip, '', '1')
-
-        if zero_merge_block:
-            assert zero_merge_block.is_in_blockchain(zero_chain_tip, '0')
-            print("Checked shard '' and 0 atomicity: passed")
-        if one_merge_block:
-            assert one_merge_block.is_in_blockchain(one_chain_tip, '1')
-            print("Checked shard '' and 1 atomicity: passed")
+        for shard_id in sorted(shards_forkchoice):
+            tip = shards_forkchoice[shard_id]
+
+            left_child_shard = shard_id + '0'
+            if left_child_shard in shards_forkchoice:
+                left_merge_block = self.previous_merge_block_on_shard(tip, shard_id, left_child_shard)
+                if left_merge_block:
+                    print("Checking shard {} and {} atomicity:".format(shard_id, left_child_shard))
+                    assert left_merge_block.is_in_blockchain(shards_forkchoice[left_child_shard], left_child_shard), str(left_merge_block.estimate)
+                    print("Passed")
+
+            right_child_shard = shard_id + '1'
+            if right_child_shard in shards_forkchoice:
+                right_merge_block = self.previous_merge_block_on_shard(tip, shard_id, right_child_shard)
+                if right_merge_block:
+                    print("Checking shard {} and {} atomicity:".format(shard_id, right_child_shard))
+                    assert right_merge_block.is_in_blockchain(shards_forkchoice[right_child_shard], right_child_shard), str(right_merge_block.estimate)
+                    print("Passed")
 
 
     def previous_merge_block_on_shard(self, starting_block, block_shard_id, merge_shard):
@@ -68,19 +96,6 @@ def previous_merge_block_on_shard(self, starting_block, block_shard_id, merge_sh
         return None
 
 
-    def update_starting_blocks(self):
-        base_chain_tip = self.forkchoice_on_shard('')
-
-        # get the most recent merge block on shard '0'
-        merge_block = self.previous_merge_block_on_shard(base_chain_tip, '', '0')
-        if merge_block:
-            self.starting_blocks['0'] = merge_block
-
-        # get the most recent merge block on shard '0'
-        merge_block = self.previous_merge_block_on_shard(base_chain_tip, '', '1')
-        if merge_block:
-            self.starting_blocks['1'] = merge_block
-
     def forkchoice_on_shard(self, shard_id):
         return forkchoice.get_shard_fork_choice(
             self.starting_blocks[shard_id],

From c0a13e97a22b8abc96eebc9a5a16fd401e5f4877 Mon Sep 17 00:00:00 2001
From: naterush <narush@wharton.upenn.edu>
Date: Tue, 13 Feb 2018 20:14:54 -0500
Subject: [PATCH 04/10] cleanup sharding view

---
 .../protocols/sharding/sharding_plot_tool.py  |   1 -
 casper/protocols/sharding/sharding_view.py    | 118 +++++++++---------
 2 files changed, 56 insertions(+), 63 deletions(-)

diff --git a/casper/protocols/sharding/sharding_plot_tool.py b/casper/protocols/sharding/sharding_plot_tool.py
index df81646..9599926 100644
--- a/casper/protocols/sharding/sharding_plot_tool.py
+++ b/casper/protocols/sharding/sharding_plot_tool.py
@@ -25,7 +25,6 @@ def __init__(self, display, save, view, validator_set):
             for validator in validator_set
         }
 
-
     def update(self, new_messages=None):
         """Updates displayable items with new messages and paths"""
         return
diff --git a/casper/protocols/sharding/sharding_view.py b/casper/protocols/sharding/sharding_view.py
index 030a258..b96dcdc 100644
--- a/casper/protocols/sharding/sharding_view.py
+++ b/casper/protocols/sharding/sharding_view.py
@@ -1,4 +1,4 @@
-"""The blockchain view module extends a view for blockchain data structures """
+"""The sharding view module extends a view for sharded blockchain data structure"""
 import random as r
 
 from casper.safety_oracles.clique_oracle import CliqueOracle
@@ -6,86 +6,96 @@
 import casper.protocols.sharding.forkchoice as forkchoice
 
 
-
 class ShardingView(AbstractView):
-    """A view class that also keeps track of a last_finalized_block and children"""
+    """A view class that keeps track of children, latest_messages_on_shard and starting_blocks"""
     def __init__(self, messages=None, shard_genesis_block=None):
         self.children = dict()
 
-        self.starting_blocks = dict() # shard_id -> current starting block for the forkchoice
-        if shard_genesis_block:
-            assert not shard_genesis_block.is_merge_block
-            shard_id = r.sample(shard_genesis_block.estimate['shard_ids'], 1)[0]
-            self.starting_blocks[shard_id] = shard_genesis_block
-        # note: the shard_genesis_block should _not_ be merge mined!
-
+        self.starting_blocks = dict() # shard_id -> starting block for forkchoice
         self.latest_messages_on_shard = dict() # shard_id -> validator -> message
 
+        self.select_shards = self.select_random_shards
+
+        if shard_genesis_block:
+            for shard_id in shard_genesis_block.estimate['shard_ids']:
+                self.starting_blocks[shard_id] = shard_genesis_block
+
         super().__init__(messages)
 
     def estimate(self):
         """Returns the current forkchoice in this view"""
-        print("\n\n Running Forkchoice")
         shards_forkchoice = dict()
 
         for shard_id in sorted(self.starting_blocks):
-            tip = forkchoice.get_shard_fork_choice(
+            shard_tip = forkchoice.get_shard_fork_choice(
                 self.starting_blocks[shard_id],
                 self.children,
                 self.latest_messages_on_shard[shard_id],
                 shard_id
             )
-            shards_forkchoice[shard_id] = tip
+            shards_forkchoice[shard_id] = shard_tip
 
             left_child_shard = shard_id + '0'
-            if left_child_shard in self.starting_blocks:
-                left_merge_block = self.previous_merge_block_on_shard(tip, shard_id, left_child_shard)
-                if left_merge_block:
-                    self.starting_blocks[left_child_shard] = left_merge_block
-
             right_child_shard = shard_id + '1'
+            if left_child_shard in self.starting_blocks:
+                self.set_child_starting_block(shard_tip, shard_id, left_child_shard)
             if right_child_shard in self.starting_blocks:
-                right_merge_block = self.previous_merge_block_on_shard(tip, shard_id, right_child_shard)
-                if right_merge_block:
-                    self.starting_blocks[right_child_shard] = right_merge_block
+                self.set_child_starting_block(shard_tip, shard_id, right_child_shard)
 
         self.check_forkchoice_atomicity(shards_forkchoice)
 
-        # pay no attenting to the code behind the curtain ::))
+        shards_to_build_on = self.select_shards(shards_forkchoice)
+        return {'prev_blocks': {shards_forkchoice[shard_id] for shard_id in shards_to_build_on},
+                'shard_ids': shards_to_build_on}
+
+    def set_child_starting_block(self, tip_block, parent_id, child_id):
+        """Changes the starting block for the forkchoice of a shard"""
+        child_merge_block = self.previous_merge_block_on_shard(
+            tip_block,
+            parent_id,
+            child_id
+        )
+
+        if child_merge_block:
+            self.starting_blocks[child_id] = child_merge_block
+
+    def select_random_shards(self, shards_forkchoice):
+        """Randomly selects a shard to build on, and sometimes selects another child shard"""
         shards_to_build_on = [r.choice([key for key in self.starting_blocks.keys()])]
         if (r.randint(0, 1) == 1):
             child = str(r.randint(0, 1))
             if shards_to_build_on[0] + child in self.starting_blocks:
                 shards_to_build_on.append(shards_to_build_on[0] + child)
 
-        shards_to_build_on = set(shards_to_build_on)
+        return set(shards_to_build_on)
 
-        return {'prev_blocks': {shards_forkchoice[shard_id] for shard_id in shards_to_build_on},
-                'shard_ids': shards_to_build_on}
 
     def check_forkchoice_atomicity(self, shards_forkchoice):
-        # assert the atomicity things about the shards_forkchoice
+        """Asserts that if a merge block is in the forkchoice for a parent chain
+        then it is in the forkchoice for a child chain"""
+        print("Checking merge block atomicity")
         for shard_id in sorted(shards_forkchoice):
             tip = shards_forkchoice[shard_id]
 
-            left_child_shard = shard_id + '0'
-            if left_child_shard in shards_forkchoice:
-                left_merge_block = self.previous_merge_block_on_shard(tip, shard_id, left_child_shard)
-                if left_merge_block:
-                    print("Checking shard {} and {} atomicity:".format(shard_id, left_child_shard))
-                    assert left_merge_block.is_in_blockchain(shards_forkchoice[left_child_shard], left_child_shard), str(left_merge_block.estimate)
-                    print("Passed")
-
-            right_child_shard = shard_id + '1'
-            if right_child_shard in shards_forkchoice:
-                right_merge_block = self.previous_merge_block_on_shard(tip, shard_id, right_child_shard)
-                if right_merge_block:
-                    print("Checking shard {} and {} atomicity:".format(shard_id, right_child_shard))
-                    assert right_merge_block.is_in_blockchain(shards_forkchoice[right_child_shard], right_child_shard), str(right_merge_block.estimate)
-                    print("Passed")
-
+            for child_shard_id in [shard_id + '0', shard_id + '1']:
+                if child_shard_id not in shards_forkchoice:
+                    continue
+
+                merge_block = self.previous_merge_block_on_shard(
+                    tip,
+                    shard_id,
+                    child_shard_id
+                )
+                if merge_block:
+                    assert merge_block.is_in_blockchain(
+                        shards_forkchoice[child_shard_id],
+                        child_shard_id
+                    )
+        print("Passed")
 
     def previous_merge_block_on_shard(self, starting_block, block_shard_id, merge_shard):
+        """Get the most recent merge block between block_shard_id and merge_shard
+        Starts from starting_block"""
         assert starting_block.on_shard(block_shard_id)
         current_block = starting_block
         while current_block:
@@ -95,22 +105,12 @@ def previous_merge_block_on_shard(self, starting_block, block_shard_id, merge_sh
             current_block = current_block.prev_block(block_shard_id)
         return None
 
-
-    def forkchoice_on_shard(self, shard_id):
-        return forkchoice.get_shard_fork_choice(
-            self.starting_blocks[shard_id],
-            self.children,
-            self.latest_messages_on_shard[shard_id],
-            shard_id
-        )
-
-
     def update_safe_estimates(self, validator_set):
         """Checks safety on messages in views forkchoice, and updates last_finalized_block"""
         # check the safety of the top shard!
         pass
 
-        tip = self.forkchoice_on_shard('')
+        tip = None
 
         while tip and tip != self.starting_blocks['']:
             oracle = CliqueOracle(tip, self, validator_set)
@@ -126,12 +126,10 @@ def _update_protocol_specific_view(self, message):
         """Given a now justified message, updates children and when_recieved"""
         assert message.hash in self.justified_messages, "...should not have seen message!"
 
-        # set starting messages! (should only happen initially)
+        # set starting messages! ::))
         if None in message.estimate['prev_blocks']:
-            assert not message.is_merge_block
-            shard_id = r.sample(message.estimate['shard_ids'], 1)[0]
-            print("Adding starting block for {}".format(shard_id))
-            self.starting_blocks[shard_id] = message
+            for shard_id in message.estimate['shard_ids']:
+                self.starting_blocks[shard_id] = message
 
         # update the latest_messages
         for shard_id in message.estimate['shard_ids']:
@@ -143,12 +141,8 @@ def _update_protocol_specific_view(self, message):
             elif latest_messages[message.sender].sequence_number < message.sequence_number:
                 latest_messages[message.sender] = message
 
-            assert message.sender in self.latest_messages_on_shard[shard_id]
-
         # update children dictonary
         for parent in message.estimate['prev_blocks']:
             if parent not in self.children:
                 self.children[parent] = set()
             self.children[parent].add(message)
-
-        # preferably, would update starting blocks here.

From 3c057cbf1d32682f2b9b904feb02e0d481d6cfed Mon Sep 17 00:00:00 2001
From: naterush <narush@wharton.upenn.edu>
Date: Tue, 13 Feb 2018 21:05:38 -0500
Subject: [PATCH 05/10] minor lints

---
 casper/protocols/sharding/forkchoice.py       |  4 +--
 .../protocols/sharding/sharding_protocol.py   | 30 +++++++++----------
 casper/protocols/sharding/sharding_view.py    |  5 ++--
 3 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/casper/protocols/sharding/forkchoice.py b/casper/protocols/sharding/forkchoice.py
index 95bb57d..134a4cc 100644
--- a/casper/protocols/sharding/forkchoice.py
+++ b/casper/protocols/sharding/forkchoice.py
@@ -40,7 +40,7 @@ def get_shard_fork_choice(starting_block, children, latest_messages, shard_id):
         max_score = 0
         for child in children[best_block]:
             if not child.on_shard(shard_id):
-                continue # we only select children on the same shard
+                continue  # we only select children on the same shard
             curr_scores[child] = scores.get(child, 0)
             max_score = max(curr_scores[child], max_score)
 
@@ -74,6 +74,6 @@ def get_all_shards_fork_choice(starting_blocks, children, latest_messages_on_sha
             latest_messages_on_shard[shard_id],
             shard_id
         ) for shard_id in starting_blocks
-        }
+    }
 
     return shards_forkchoice
diff --git a/casper/protocols/sharding/sharding_protocol.py b/casper/protocols/sharding/sharding_protocol.py
index 9439f21..1a1526c 100644
--- a/casper/protocols/sharding/sharding_protocol.py
+++ b/casper/protocols/sharding/sharding_protocol.py
@@ -13,20 +13,21 @@ class ShardingProtocol(Protocol):
     curr_shard_idx = 0
     curr_shard_ids = ['']
 
-    # Shard ID's look like this:
-    #       ''
-    #     /    \
-    #   '0'    '1'
-    #  /  \    /  \
-    #'00''01''10''11'
-    #
-
-    # Blocks can be merge mined between shards if
-    # there is an edge between shards
-    # That is, for ids shard_1 and shard_2, there can be a merge block if
-    # abs(len(shard_1) - len(shard_2)) = 1 AND
-    # for i in range(min(len(shard_1), len(shard_2))):
-    #    shard_1[i] = shard_2[i]
+    """Shard ID's look like this:
+           ''
+         /    \
+       '0'    '1'
+      /  \    /  \
+    '00''01''10''11'
+
+
+     Blocks can be merge mined between shards if
+     there is an edge between shards
+     That is, for ids shard_1 and shard_2, there can be a merge block if
+     abs(len(shard_1) - len(shard_2)) = 1 AND
+     for i in range(min(len(shard_1), len(shard_2))):
+        shard_1[i] = shard_2[i]
+    """
 
     @classmethod
     def initial_message(cls, validator):
@@ -38,7 +39,6 @@ def initial_message(cls, validator):
 
         return cls.shard_genesis_blocks['']
 
-
     @classmethod
     def get_new_shard_id(cls):
         new_id = cls.curr_shard_ids[cls.curr_shard_idx]
diff --git a/casper/protocols/sharding/sharding_view.py b/casper/protocols/sharding/sharding_view.py
index b96dcdc..a1542b0 100644
--- a/casper/protocols/sharding/sharding_view.py
+++ b/casper/protocols/sharding/sharding_view.py
@@ -11,8 +11,8 @@ class ShardingView(AbstractView):
     def __init__(self, messages=None, shard_genesis_block=None):
         self.children = dict()
 
-        self.starting_blocks = dict() # shard_id -> starting block for forkchoice
-        self.latest_messages_on_shard = dict() # shard_id -> validator -> message
+        self.starting_blocks = dict()  # shard_id -> starting block for forkchoice
+        self.latest_messages_on_shard = dict()  # shard_id -> validator -> message
 
         self.select_shards = self.select_random_shards
 
@@ -69,7 +69,6 @@ def select_random_shards(self, shards_forkchoice):
 
         return set(shards_to_build_on)
 
-
     def check_forkchoice_atomicity(self, shards_forkchoice):
         """Asserts that if a merge block is in the forkchoice for a parent chain
         then it is in the forkchoice for a child chain"""

From 94006e83485c74475aad1c3d8493632beeaabfa4 Mon Sep 17 00:00:00 2001
From: naterush <narush@wharton.upenn.edu>
Date: Wed, 14 Feb 2018 21:57:58 -0500
Subject: [PATCH 06/10] fix bug in sharding forkchoice

---
 casper/protocols/sharding/sharding_view.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/casper/protocols/sharding/sharding_view.py b/casper/protocols/sharding/sharding_view.py
index a1542b0..7f04a4c 100644
--- a/casper/protocols/sharding/sharding_view.py
+++ b/casper/protocols/sharding/sharding_view.py
@@ -11,6 +11,7 @@ class ShardingView(AbstractView):
     def __init__(self, messages=None, shard_genesis_block=None):
         self.children = dict()
 
+        self.shard_genesis_blocks = dict()  # shard_id -> genesis for shard
         self.starting_blocks = dict()  # shard_id -> starting block for forkchoice
         self.latest_messages_on_shard = dict()  # shard_id -> validator -> message
 
@@ -18,6 +19,7 @@ def __init__(self, messages=None, shard_genesis_block=None):
 
         if shard_genesis_block:
             for shard_id in shard_genesis_block.estimate['shard_ids']:
+                self.shard_genesis_blocks[shard_id] = shard_genesis_block
                 self.starting_blocks[shard_id] = shard_genesis_block
 
         super().__init__(messages)
@@ -58,6 +60,8 @@ def set_child_starting_block(self, tip_block, parent_id, child_id):
 
         if child_merge_block:
             self.starting_blocks[child_id] = child_merge_block
+        else:
+            self.starting_blocks[child_id] = self.shard_genesis_blocks[child_id]
 
     def select_random_shards(self, shards_forkchoice):
         """Randomly selects a shard to build on, and sometimes selects another child shard"""
@@ -128,6 +132,7 @@ def _update_protocol_specific_view(self, message):
         # set starting messages! ::))
         if None in message.estimate['prev_blocks']:
             for shard_id in message.estimate['shard_ids']:
+                self.shard_genesis_blocks[shard_id] = message
                 self.starting_blocks[shard_id] = message
 
         # update the latest_messages

From af6ffbd78118e298433fc0e80c7e0d2320061173 Mon Sep 17 00:00:00 2001
From: naterush <narush@wharton.upenn.edu>
Date: Thu, 15 Feb 2018 21:55:44 -0500
Subject: [PATCH 07/10] minor updates to sharding protocol

---
 casper/protocols/sharding/block.py            | 22 ++++++++++++-------
 casper/protocols/sharding/forkchoice.py       |  2 +-
 .../protocols/sharding/sharding_plot_tool.py  |  2 +-
 .../protocols/sharding/sharding_protocol.py   | 20 ++++++++---------
 casper/protocols/sharding/sharding_view.py    |  2 +-
 5 files changed, 27 insertions(+), 21 deletions(-)

diff --git a/casper/protocols/sharding/block.py b/casper/protocols/sharding/block.py
index afae3aa..d1e3c39 100644
--- a/casper/protocols/sharding/block.py
+++ b/casper/protocols/sharding/block.py
@@ -2,17 +2,17 @@
 from casper.message import Message
 
 
+NUM_MERGE_SHARDS = 2
 class Block(Message):
     """Message data structure for a sharded blockchain"""
 
     @classmethod
     def is_valid_estimate(cls, estimate):
-        if not isinstance(estimate, dict):
-            return False
-        if not isinstance(estimate['prev_blocks'], set):
-            return False
-        if not isinstance(estimate['shard_ids'], set):
-            return False
+        for key in ['prev_blocks', 'shard_ids']:
+            if key not in estimate:
+                return False
+            if not isinstance(estimate[key], set):
+                return False
         return True
 
     def on_shard(self, shard_id):
@@ -25,17 +25,23 @@ def prev_block(self, shard_id):
             raise KeyError("No previous block on that shard")
 
         for block in self.estimate['prev_blocks']:
+            # if this block is the genesis, previous is None
             if block is None:
                 return None
 
+            # otherwise, return the previous block on that shard
             if block.on_shard(shard_id):
                 return block
 
-        raise KeyError("Should have found previous block on shard!")
+        raise KeyError("Block on {}, but has no previous block on that shard!".format(shard_id))
 
     @property
     def is_merge_block(self):
-        return len(self.estimate['shard_ids']) == 2
+        return len(self.estimate['shard_ids']) == NUM_MERGE_SHARDS
+
+    @property
+    def is_genesis_block(self):
+        return None in self.estimate['prev_blocks']
 
     def conflicts_with(self, message):
         """Returns true if self is not in the prev blocks of other_message"""
diff --git a/casper/protocols/sharding/forkchoice.py b/casper/protocols/sharding/forkchoice.py
index 134a4cc..104eded 100644
--- a/casper/protocols/sharding/forkchoice.py
+++ b/casper/protocols/sharding/forkchoice.py
@@ -1,4 +1,4 @@
-"""The forkchoice module implements the estimator function a blockchain"""
+"""The forkchoice module implements the estimator function a sharded blockchain"""
 
 
 def get_max_weight_indexes(scores):
diff --git a/casper/protocols/sharding/sharding_plot_tool.py b/casper/protocols/sharding/sharding_plot_tool.py
index 9599926..a30c7de 100644
--- a/casper/protocols/sharding/sharding_plot_tool.py
+++ b/casper/protocols/sharding/sharding_plot_tool.py
@@ -1,4 +1,4 @@
-"""The blockchain plot tool implements functions for plotting blockchain data structures"""
+"""The blockchain plot tool implements functions for plotting sharded blockchain data structures"""
 
 from casper.plot_tool import PlotTool
 from casper.safety_oracles.clique_oracle import CliqueOracle
diff --git a/casper/protocols/sharding/sharding_protocol.py b/casper/protocols/sharding/sharding_protocol.py
index 1a1526c..5749116 100644
--- a/casper/protocols/sharding/sharding_protocol.py
+++ b/casper/protocols/sharding/sharding_protocol.py
@@ -31,26 +31,26 @@ class ShardingProtocol(Protocol):
 
     @classmethod
     def initial_message(cls, validator):
-        """Returns a dict from shard_id -> shard genesis block"""
-        shard_id = cls.get_new_shard_id()
+        """Returns a starting block for a shard"""
+        shard_id = cls.get_next_shard_id()
 
         estimate = {'prev_blocks': set([None]), 'shard_ids': set([shard_id])}
-        cls.shard_genesis_blocks[''] = Block(estimate, dict(), validator, -1, 0)
+        cls.shard_genesis_blocks[shard_id] = Block(estimate, dict(), validator, -1, 0)
 
         return cls.shard_genesis_blocks['']
 
     @classmethod
-    def get_new_shard_id(cls):
-        new_id = cls.curr_shard_ids[cls.curr_shard_idx]
+    def get_next_shard_id(cls):
+        next_id = cls.curr_shard_ids[cls.curr_shard_idx]
         cls.curr_shard_idx += 1
 
         if cls.curr_shard_idx == len(cls.curr_shard_ids):
-            new_ids = []
+            next_ids = []
             for shard_id in cls.curr_shard_ids:
-                new_ids.append(shard_id + '0')
-                new_ids.append(shard_id + '1')
+                next_ids.append(shard_id + '0')
+                next_ids.append(shard_id + '1')
 
             cls.curr_shard_idx = 0
-            cls.curr_shard_ids = new_ids
+            cls.curr_shard_ids = next_ids
 
-        return new_id
+        return next_id
diff --git a/casper/protocols/sharding/sharding_view.py b/casper/protocols/sharding/sharding_view.py
index 7f04a4c..76249f5 100644
--- a/casper/protocols/sharding/sharding_view.py
+++ b/casper/protocols/sharding/sharding_view.py
@@ -130,7 +130,7 @@ def _update_protocol_specific_view(self, message):
         assert message.hash in self.justified_messages, "...should not have seen message!"
 
         # set starting messages! ::))
-        if None in message.estimate['prev_blocks']:
+        if message.is_genesis_block:
             for shard_id in message.estimate['shard_ids']:
                 self.shard_genesis_blocks[shard_id] = message
                 self.starting_blocks[shard_id] = message

From 677a728efe976a92fb0121333943e34e1ec41581 Mon Sep 17 00:00:00 2001
From: naterush <narush@wharton.upenn.edu>
Date: Thu, 15 Feb 2018 22:20:31 -0500
Subject: [PATCH 08/10] minor lint

---
 casper/protocols/sharding/block.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/casper/protocols/sharding/block.py b/casper/protocols/sharding/block.py
index d1e3c39..1845d9b 100644
--- a/casper/protocols/sharding/block.py
+++ b/casper/protocols/sharding/block.py
@@ -1,8 +1,8 @@
 """The block module implements the message data structure for a sharded blockchain"""
 from casper.message import Message
+NUM_MERGE_SHARDS = 2
 
 
-NUM_MERGE_SHARDS = 2
 class Block(Message):
     """Message data structure for a sharded blockchain"""
 

From d9e58c275b250babf308cdc9ec464c9cc74758a9 Mon Sep 17 00:00:00 2001
From: naterush <narush@wharton.upenn.edu>
Date: Fri, 16 Feb 2018 20:28:08 -0500
Subject: [PATCH 09/10] fix bug in forkchoice

---
 casper/protocols/sharding/forkchoice.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/casper/protocols/sharding/forkchoice.py b/casper/protocols/sharding/forkchoice.py
index 104eded..46ece8f 100644
--- a/casper/protocols/sharding/forkchoice.py
+++ b/casper/protocols/sharding/forkchoice.py
@@ -41,6 +41,15 @@ def get_shard_fork_choice(starting_block, children, latest_messages, shard_id):
         for child in children[best_block]:
             if not child.on_shard(shard_id):
                 continue  # we only select children on the same shard
+            # can't pick a child that a merge block with a higher shard
+            if child.is_merge_block:
+                not_in_forkchoice = False
+                for shard in child.estimate['shard_ids']:
+                    if len(shard) < len(shard_id):
+                        not_in_forkchoice = True
+                        break
+                if not_in_forkchoice:
+                    continue
             curr_scores[child] = scores.get(child, 0)
             max_score = max(curr_scores[child], max_score)
 

From 307e7d4aa74313f31f39f53094d63c3d99bc07b1 Mon Sep 17 00:00:00 2001
From: naterush <narush@wharton.upenn.edu>
Date: Sat, 24 Feb 2018 13:52:59 -0500
Subject: [PATCH 10/10] minor lints and cleanup to sharding

---
 casper/protocols/sharding/forkchoice.py    |  2 +-
 casper/protocols/sharding/sharding_view.py | 23 +++++-----------------
 2 files changed, 6 insertions(+), 19 deletions(-)

diff --git a/casper/protocols/sharding/forkchoice.py b/casper/protocols/sharding/forkchoice.py
index 46ece8f..f8124ae 100644
--- a/casper/protocols/sharding/forkchoice.py
+++ b/casper/protocols/sharding/forkchoice.py
@@ -7,7 +7,7 @@ def get_max_weight_indexes(scores):
 
     max_score = max(scores.values())
 
-    assert max_score != 0, "max_score of a block should never be zero"
+    assert max_score > 0, "max_score should be greater than zero"
 
     max_weight_estimates = {e for e in scores if scores[e] == max_score}
 
diff --git a/casper/protocols/sharding/sharding_view.py b/casper/protocols/sharding/sharding_view.py
index 76249f5..dada971 100644
--- a/casper/protocols/sharding/sharding_view.py
+++ b/casper/protocols/sharding/sharding_view.py
@@ -1,7 +1,6 @@
 """The sharding view module extends a view for sharded blockchain data structure"""
 import random as r
 
-from casper.safety_oracles.clique_oracle import CliqueOracle
 from casper.abstract_view import AbstractView
 import casper.protocols.sharding.forkchoice as forkchoice
 
@@ -65,11 +64,11 @@ def set_child_starting_block(self, tip_block, parent_id, child_id):
 
     def select_random_shards(self, shards_forkchoice):
         """Randomly selects a shard to build on, and sometimes selects another child shard"""
-        shards_to_build_on = [r.choice([key for key in self.starting_blocks.keys()])]
-        if (r.randint(0, 1) == 1):
-            child = str(r.randint(0, 1))
-            if shards_to_build_on[0] + child in self.starting_blocks:
-                shards_to_build_on.append(shards_to_build_on[0] + child)
+        shards_to_build_on = [r.choice(list(self.starting_blocks.keys()))]
+        if r.choice([True, False]):
+            child_shard_id = shards_to_build_on[0] + str(r.randint(0, 1))
+            if child_shard_id in self.starting_blocks:
+                shards_to_build_on.append(child_shard_id)
 
         return set(shards_to_build_on)
 
@@ -113,18 +112,6 @@ def update_safe_estimates(self, validator_set):
         # check the safety of the top shard!
         pass
 
-        tip = None
-
-        while tip and tip != self.starting_blocks['']:
-            oracle = CliqueOracle(tip, self, validator_set)
-            fault_tolerance, _ = oracle.check_estimate_safety()
-
-            if fault_tolerance > 0:
-                self.starting_blocks[''] = tip
-                return tip
-
-            tip = tip.prev_block('')
-
     def _update_protocol_specific_view(self, message):
         """Given a now justified message, updates children and when_recieved"""
         assert message.hash in self.justified_messages, "...should not have seen message!"