From e964e1606db8b13d248d20f041847313285d14de Mon Sep 17 00:00:00 2001 From: klassen9 Date: Sat, 6 Jul 2024 17:52:43 +0200 Subject: [PATCH 1/3] Add support for the Resize operator --- .../fpgadataflow/convert_to_hw_layers.py | 26 ++++++- src/finn/transformation/streamline/reorder.py | 40 ++++++++-- .../streamline/test_scale_resize_nhwc.py | 76 ++++++++++++++++++- 3 files changed, 132 insertions(+), 10 deletions(-) diff --git a/src/finn/transformation/fpgadataflow/convert_to_hw_layers.py b/src/finn/transformation/fpgadataflow/convert_to_hw_layers.py index e14181b140..9017b171dd 100644 --- a/src/finn/transformation/fpgadataflow/convert_to_hw_layers.py +++ b/src/finn/transformation/fpgadataflow/convert_to_hw_layers.py @@ -289,7 +289,31 @@ def apply(self, model): if n.op_type == "Upsample": scales = model.get_initializer(n.input[1]) else: - scales = model.get_initializer(n.input[2]) + if len(n.input) == 2: + # Resize version 10 + scales = model.get_initializer(n.input[1]) + elif len(n.input) == 3: + # Resize version 11 and up (no size input) + scales = model.get_initializer(n.input[2]) + elif len(n.input) == 4: + # Resize version 11 and up + scales_exists = (model.get_initializer(n.input[2]) is not None) and (len(model.get_initializer(n.input[2])) != 0) + sizes_exists = (model.get_initializer(n.input[3]) is not None) and (len(model.get_initializer(n.input[3])) != 0) + assert (scales_exists ^ sizes_exists), ( + "%s: Either scales or the target output size must " + "be specified. Specifying both is prohibited." % n.name + ) + assert (model.get_initializer(n.input[1]) is None), ( + "%s: Defining the ROI is not supported" % n.name + ) + if (scales_exists): + # Scales input + scales = model.get_initializer(n.input[2]) + else: + # Convert sizes to scales + sizes = model.get_initializer(n.input[3]) + data_input_size = model.get_tensor_shape(n.input[0]) + scales = sizes/data_input_size in_shape = model.get_tensor_shape(n.input[0]) dt = model.get_tensor_datatype(n.input[0]) diff --git a/src/finn/transformation/streamline/reorder.py b/src/finn/transformation/streamline/reorder.py index 8ac2d7dad6..797c554297 100644 --- a/src/finn/transformation/streamline/reorder.py +++ b/src/finn/transformation/streamline/reorder.py @@ -758,18 +758,42 @@ def apply(self, model): consumer = model.find_consumer(n.output[0]) producer = model.find_producer(n.input[0]) if n.op_type == "Upsample": - scales_ind = 1 + transformation_ind = 1 + d_type = "float32" else: - scales_ind = 2 + if len(n.input) == 2: + # Resize version 10 + transformation_ind = 1 + d_type = "float32" + elif len(n.input) == 3: + # Resize version 11 and up (no size input) + transformation_ind = 2 + d_type = "float32" + elif len(n.input) == 4: + # Resize version 11 and up + scales_exists = (model.get_initializer(n.input[2]) is not None) and (len(model.get_initializer(n.input[2])) != 0) + sizes_exists = (model.get_initializer(n.input[3]) is not None) and (len(model.get_initializer(n.input[3])) != 0) + assert (scales_exists ^ sizes_exists), ( + "%s: Either scales or the target output size must " + "be specified. Specifying both is prohibited." % n.name + ) + if (scales_exists): + # Scales input + transformation_ind = 2 + d_type = "float32" + else: + # Sizes input + transformation_ind = 3 + d_type = "int64" if producer is not None and producer.op_type == "Transpose": perms = list(get_by_name(producer.attribute, "perm").ints) if perms == [0, 3, 1, 2]: - old_value = model.get_initializer(n.input[scales_ind]) + old_value = model.get_initializer(n.input[transformation_ind]) new_value = np.array( [old_value[idx] for idx in (0, 2, 3, 1)], - dtype=np.dtype("float32"), + dtype=np.dtype(d_type), ) - model.set_initializer(n.input[scales_ind], new_value) + model.set_initializer(n.input[transformation_ind], new_value) start_name = producer.input[0] mid_name = n.input[0] end_name = n.output[0] @@ -786,12 +810,12 @@ def apply(self, model): elif consumer is not None and consumer.op_type == "Transpose": perms = list(get_by_name(consumer.attribute, "perm").ints) if perms == [0, 2, 3, 1]: - old_value = model.get_initializer(n.input[scales_ind]) + old_value = model.get_initializer(n.input[transformation_ind]) new_value = np.array( [old_value[idx] for idx in (0, 2, 3, 1)], - dtype=np.dtype("float32"), + dtype=np.dtype(d_type), ) - model.set_initializer(n.input[scales_ind], new_value) + model.set_initializer(n.input[transformation_ind], new_value) start_name = n.input[0] mid_name = consumer.input[0] end_name = consumer.output[0] diff --git a/tests/transformation/streamline/test_scale_resize_nhwc.py b/tests/transformation/streamline/test_scale_resize_nhwc.py index 350f5b3133..47a34e4601 100644 --- a/tests/transformation/streamline/test_scale_resize_nhwc.py +++ b/tests/transformation/streamline/test_scale_resize_nhwc.py @@ -179,6 +179,58 @@ def create_transpose_resize_transpose(ifm_dim, ifm_ch, scales, mode, idt): return model +def create_resize_transpose_sizes(ifm_dim, ifm_ch, sizes, mode, idt): + ofm_dim_h = sizes[2] + ofm_dim_w = sizes[3] + inp = oh.make_tensor_value_info("inp", TensorProto.FLOAT, [1, ifm_ch, ifm_dim[0], ifm_dim[1]]) + + # Empty scales + scales = oh.make_tensor_value_info("scales", TensorProto.FLOAT, []) + + # Not actually used, only needed for compliance with the Resize node interface + roi = oh.make_tensor_value_info("roi", TensorProto.FLOAT, [4]) + + param = oh.make_tensor_value_info("sizes", TensorProto.INT64, [4]) + + outp_up = oh.make_tensor_value_info( + "outp_up", TensorProto.FLOAT, [1, ifm_ch, ofm_dim_h, ofm_dim_w] + ) + outp = oh.make_tensor_value_info("outp", TensorProto.FLOAT, [1, ofm_dim_h, ofm_dim_w, ifm_ch]) + + resize_node = oh.make_node( + "Resize", + inputs=["inp", "roi", "scales", "sizes"], + outputs=["outp_up"], + name="Resize1", + mode=mode, + ) + + transpose_node = onnx.helper.make_node( + "Transpose", + inputs=["outp_up"], + outputs=["outp"], + name="Transpose1", + perm=[0, 2, 3, 1], + ) + + graph = oh.make_graph( + nodes=[resize_node, transpose_node], + name="resize_graph", + inputs=[inp], + outputs=[outp], + value_info=[outp_up, roi, scales, param], + ) + + model = qonnx_make_model(graph, producer_name="resize_model4") + model = ModelWrapper(model) + model.set_tensor_datatype("inp", idt) + model.set_tensor_datatype("outp", idt) + + model.set_tensor_layout("inp", DataLayout.NCHW) + model = model.transform(InferShapes()) + model = model.transform(InferDataLayouts()) + + return model def check_transform(model): graph = model.graph @@ -198,20 +250,25 @@ def check_transform(model): @pytest.mark.parametrize("ifm_ch", [3]) # scales @pytest.mark.parametrize("scales", [[1, 1, i, j] for i in range(2, 5) for j in range(2, 5)]) +# sizes +@pytest.mark.parametrize("sizes", [[1, 3, 2**i, 2**j] for i in range(6, 7) for j in range(6, 7)]) # mode @pytest.mark.parametrize("mode", ["nearest"]) # input datatype @pytest.mark.parametrize("idt", [DataType["INT4"]]) -def test_scale_resize_nhwc(ifm_dim, ifm_ch, scales, mode, idt): +def test_scale_resize_nhwc(ifm_dim, ifm_ch, sizes, scales, mode, idt): # create models resize_model1 = create_resize_transpose(ifm_dim, ifm_ch, scales, mode, idt) resize_model2 = create_transpose_resize(ifm_dim, ifm_ch, scales, mode, idt) resize_model3 = create_transpose_resize_transpose(ifm_dim, ifm_ch, scales, mode, idt) + resize_model4 = create_resize_transpose_sizes(ifm_dim, ifm_ch, sizes, mode, idt) # set initializers resize_model1.set_initializer("scales", np.array(scales, dtype=np.float32)) resize_model2.set_initializer("scales", np.array(scales, dtype=np.float32)) resize_model3.set_initializer("scales", np.array(scales, dtype=np.float32)) + resize_model4.set_initializer("sizes", np.array(sizes, dtype=np.int64)) + resize_model4.set_initializer("scales", np.array([], dtype=np.float32)) # generate input tensor for testing input_tensor_nchw = gen_finn_dt_tensor(idt, [1, ifm_ch, ifm_dim[0], ifm_dim[1]]) @@ -269,3 +326,20 @@ def test_scale_resize_nhwc(ifm_dim, ifm_ch, scales, mode, idt): # compare outputs assert (expected3 == output3).all() assert check_transform(resize_model3) + + # execute fourth model + output_dict4 = oxe.execute_onnx(resize_model4, input_dict_nchw) + expected4 = output_dict4["outp"] + + # transform Resize into ResizeNHWC + resize_model4 = resize_model4.transform(MakeScaleResizeNHWC()) + resize_model4 = resize_model4.transform(InferDataLayouts()) + + # execute transformed model + output_node_name4 = resize_model4.graph.output[0].name + output_dict4 = oxe.execute_onnx(resize_model4, input_dict_nchw, return_full_exec_context=False) + output4 = output_dict4[output_node_name4] + + # compare outputs + assert (expected4 == output4).all() + assert check_transform(resize_model4) From e21d54384e982c4bda7db0891a4d3617916d7ff9 Mon Sep 17 00:00:00 2001 From: klassen9 Date: Sun, 7 Jul 2024 12:00:25 +0200 Subject: [PATCH 2/3] Apply pre-commit to the changes regarding resize --- .../fpgadataflow/convert_to_hw_layers.py | 28 +++++++++++-------- src/finn/transformation/streamline/reorder.py | 20 +++++++------ .../streamline/test_scale_resize_nhwc.py | 8 ++++-- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/src/finn/transformation/fpgadataflow/convert_to_hw_layers.py b/src/finn/transformation/fpgadataflow/convert_to_hw_layers.py index 9017b171dd..edea1a9d19 100644 --- a/src/finn/transformation/fpgadataflow/convert_to_hw_layers.py +++ b/src/finn/transformation/fpgadataflow/convert_to_hw_layers.py @@ -294,26 +294,30 @@ def apply(self, model): scales = model.get_initializer(n.input[1]) elif len(n.input) == 3: # Resize version 11 and up (no size input) - scales = model.get_initializer(n.input[2]) + scales = model.get_initializer(n.input[2]) elif len(n.input) == 4: # Resize version 11 and up - scales_exists = (model.get_initializer(n.input[2]) is not None) and (len(model.get_initializer(n.input[2])) != 0) - sizes_exists = (model.get_initializer(n.input[3]) is not None) and (len(model.get_initializer(n.input[3])) != 0) - assert (scales_exists ^ sizes_exists), ( - "%s: Either scales or the target output size must " + scales_exists = (model.get_initializer(n.input[2]) is not None) and ( + len(model.get_initializer(n.input[2])) != 0 + ) + sizes_exists = (model.get_initializer(n.input[3]) is not None) and ( + len(model.get_initializer(n.input[3])) != 0 + ) + assert scales_exists ^ sizes_exists, ( + "%s: Either scales or the target output size must " "be specified. Specifying both is prohibited." % n.name - ) - assert (model.get_initializer(n.input[1]) is None), ( - "%s: Defining the ROI is not supported" % n.name - ) - if (scales_exists): + ) + assert model.get_initializer(n.input[1]) is None, ( + "%s: Defining the ROI is not supported" % n.name + ) + if scales_exists: # Scales input scales = model.get_initializer(n.input[2]) else: # Convert sizes to scales sizes = model.get_initializer(n.input[3]) - data_input_size = model.get_tensor_shape(n.input[0]) - scales = sizes/data_input_size + data_input_size = model.get_tensor_shape(n.input[0]) + scales = sizes / data_input_size in_shape = model.get_tensor_shape(n.input[0]) dt = model.get_tensor_datatype(n.input[0]) diff --git a/src/finn/transformation/streamline/reorder.py b/src/finn/transformation/streamline/reorder.py index 797c554297..9d7dc0b7ce 100644 --- a/src/finn/transformation/streamline/reorder.py +++ b/src/finn/transformation/streamline/reorder.py @@ -763,21 +763,25 @@ def apply(self, model): else: if len(n.input) == 2: # Resize version 10 - transformation_ind = 1 + transformation_ind = 1 d_type = "float32" elif len(n.input) == 3: # Resize version 11 and up (no size input) - transformation_ind = 2 + transformation_ind = 2 d_type = "float32" elif len(n.input) == 4: # Resize version 11 and up - scales_exists = (model.get_initializer(n.input[2]) is not None) and (len(model.get_initializer(n.input[2])) != 0) - sizes_exists = (model.get_initializer(n.input[3]) is not None) and (len(model.get_initializer(n.input[3])) != 0) - assert (scales_exists ^ sizes_exists), ( - "%s: Either scales or the target output size must " + scales_exists = (model.get_initializer(n.input[2]) is not None) and ( + len(model.get_initializer(n.input[2])) != 0 + ) + sizes_exists = (model.get_initializer(n.input[3]) is not None) and ( + len(model.get_initializer(n.input[3])) != 0 + ) + assert scales_exists ^ sizes_exists, ( + "%s: Either scales or the target output size must " "be specified. Specifying both is prohibited." % n.name - ) - if (scales_exists): + ) + if scales_exists: # Scales input transformation_ind = 2 d_type = "float32" diff --git a/tests/transformation/streamline/test_scale_resize_nhwc.py b/tests/transformation/streamline/test_scale_resize_nhwc.py index 47a34e4601..dfa2e7433a 100644 --- a/tests/transformation/streamline/test_scale_resize_nhwc.py +++ b/tests/transformation/streamline/test_scale_resize_nhwc.py @@ -179,6 +179,7 @@ def create_transpose_resize_transpose(ifm_dim, ifm_ch, scales, mode, idt): return model + def create_resize_transpose_sizes(ifm_dim, ifm_ch, sizes, mode, idt): ofm_dim_h = sizes[2] ofm_dim_w = sizes[3] @@ -189,7 +190,7 @@ def create_resize_transpose_sizes(ifm_dim, ifm_ch, sizes, mode, idt): # Not actually used, only needed for compliance with the Resize node interface roi = oh.make_tensor_value_info("roi", TensorProto.FLOAT, [4]) - + param = oh.make_tensor_value_info("sizes", TensorProto.INT64, [4]) outp_up = oh.make_tensor_value_info( @@ -232,6 +233,7 @@ def create_resize_transpose_sizes(ifm_dim, ifm_ch, sizes, mode, idt): return model + def check_transform(model): graph = model.graph node_ind = 0 @@ -251,7 +253,9 @@ def check_transform(model): # scales @pytest.mark.parametrize("scales", [[1, 1, i, j] for i in range(2, 5) for j in range(2, 5)]) # sizes -@pytest.mark.parametrize("sizes", [[1, 3, 2**i, 2**j] for i in range(6, 7) for j in range(6, 7)]) +@pytest.mark.parametrize( + "sizes", [[1, 3, 2**i, 2**j] for i in range(6, 7) for j in range(6, 7)] +) # mode @pytest.mark.parametrize("mode", ["nearest"]) # input datatype From d2ed4bedc4e1855342e9dc65c27ffc1506313c64 Mon Sep 17 00:00:00 2001 From: klassen9 Date: Tue, 30 Jul 2024 17:13:54 +0200 Subject: [PATCH 3/3] Remove ROI check for Resize --- src/finn/transformation/fpgadataflow/convert_to_hw_layers.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/finn/transformation/fpgadataflow/convert_to_hw_layers.py b/src/finn/transformation/fpgadataflow/convert_to_hw_layers.py index edea1a9d19..0fd736ffc9 100644 --- a/src/finn/transformation/fpgadataflow/convert_to_hw_layers.py +++ b/src/finn/transformation/fpgadataflow/convert_to_hw_layers.py @@ -307,9 +307,6 @@ def apply(self, model): "%s: Either scales or the target output size must " "be specified. Specifying both is prohibited." % n.name ) - assert model.get_initializer(n.input[1]) is None, ( - "%s: Defining the ROI is not supported" % n.name - ) if scales_exists: # Scales input scales = model.get_initializer(n.input[2])