Skip to content

Commit

Permalink
#7525: [skip ci] Unskip WH demo models for bert large 11, as they now…
Browse files Browse the repository at this point in the history
… run in ttnn nightly (#18646)

### Ticket

#7525 

### Problem description

LOL this thing was hung for almost a year. And we never did anything
about it.

Perhaps this is an opportunity to reflect on team dynamics, and how
blaming one person doesn't actually get us anywhere.

### What's changed
Describe the approach used to solve the problem.
Summarize the changes made and its impact.

### Checklist
- [ ] [All post
commit](https://github.com/tenstorrent/tt-metal/actions/workflows/all-post-commit-workflows.yaml)
CI passes
- [ ] [Blackhole Post
commit](https://github.com/tenstorrent/tt-metal/actions/workflows/blackhole-post-commit.yaml)
CI passes (if applicable)
- [ ] [Model
regression](https://github.com/tenstorrent/tt-metal/actions/workflows/perf-models.yaml)
CI passes (if applicable)
- [ ] [Device performance
regression](https://github.com/tenstorrent/tt-metal/actions/workflows/perf-device-models.yaml)
CI passes (if applicable)
- [ ] **(For models and ops writers)** Full [new models
tests](https://github.com/tenstorrent/tt-metal/actions/workflows/full-new-models-suite.yaml)
CI passes (if applicable)
- [ ] New/Existing tests provide coverage for changes

---------

Co-authored-by: Raymond Kim <raymond.kim243@outlook.com>
  • Loading branch information
tt-rkim and rayraykay authored Mar 6, 2025
1 parent d41f968 commit 0c0def2
Show file tree
Hide file tree
Showing 3 changed files with 8 additions and 8 deletions.
12 changes: 6 additions & 6 deletions models/demos/metal_BERT_large_11/tests/test_demo.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
(("models/demos/metal_BERT_large_11/demo/input_data.json"),),
ids=["default_input"],
)
@pytest.mark.skipif(is_wormhole_b0() or is_blackhole(), reason="#7525: hangs on wh b0")
@pytest.mark.skipif(is_blackhole(), reason="Not functional on BH yet")
def test_demo_batch_7(batch, input_path, model_location_generator, device, use_program_cache):
if is_e75(device):
pytest.skip(f"Bert large 11 is not supported on E75")
Expand Down Expand Up @@ -51,7 +51,7 @@ def test_demo_batch_7(batch, input_path, model_location_generator, device, use_p
(("models/demos/metal_BERT_large_11/demo/input_data.json"),),
ids=["default_input"],
)
@pytest.mark.skipif(is_wormhole_b0() or is_blackhole(), reason="#7525: hangs on wh b0")
@pytest.mark.skipif(is_wormhole_b0() or is_blackhole(), reason="#7525: only runs GS")
def test_demo_batch_12(batch, input_path, model_location_generator, device, use_program_cache):
if is_e75(device):
pytest.skip(f"Bert large 11 is not supported on E75")
Expand Down Expand Up @@ -81,14 +81,14 @@ def test_demo_batch_12(batch, input_path, model_location_generator, device, use_


@skip_for_grayskull()
@pytest.mark.skipif(is_wormhole_b0() or is_blackhole(), reason="#7525: hangs on wh b0")
@pytest.mark.skipif(is_blackhole(), reason="#7525: Not functional on BH yet")
@pytest.mark.parametrize(
"batch, exact, f1",
(
(
7,
78.57,
84.37,
77.14,
84.21,
),
),
ids=["batch_7"],
Expand All @@ -112,7 +112,7 @@ def test_demo_squadv2_batch_7(batch, exact, f1, model_location_generator, device
),
ids=["batch_12"],
)
@pytest.mark.skipif(is_wormhole_b0() or is_blackhole(), reason="#7525: hangs on wh b0")
@pytest.mark.skipif(is_wormhole_b0() or is_blackhole(), reason="#7525: only runs GS")
def test_demo_squadv2_batch_12(batch, exact, f1, model_location_generator, device, use_program_cache):
loop_count = 10
evals = demo_squadv2(model_location_generator, device, use_program_cache, batch, loop_count)
Expand Down
2 changes: 1 addition & 1 deletion models/demos/metal_BERT_large_11/tests/test_perf_bert11.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ def run_perf_bert11(
logger.info(f"bert11 compile time: {compile_time}")


@pytest.mark.skipif(is_wormhole_b0() or is_blackhole(), reason="Didn't test on WH yet")
@pytest.mark.skipif(is_wormhole_b0() or is_blackhole(), reason="May hang, need to check functional tests")
@run_for_wormhole_b0(reason_str="WH specific batch size")
@pytest.mark.models_performance_bare_metal
@pytest.mark.parametrize(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def test_perf_device_bare_metal(batch_size, test, expected_perf):
run_bert_perf(batch_size, test, expected_perf)


@pytest.mark.skip("#7525: Hangs non-deterministically on device perf")
@pytest.mark.skip("#7525: Hangs non-deterministically on device perf, likely same issue as demo model")
@skip_for_grayskull("Incorrect device metrics for grayskull")
@pytest.mark.models_device_performance_bare_metal
@pytest.mark.parametrize(
Expand Down

0 comments on commit 0c0def2

Please sign in to comment.