test_mistral_decoder_inference RuntimeError: TT_FATAL @ /work/ttnn/cpp/ttnn/operations/normalization/layernorm/device/multi_core/layernorm_op_multi_core.cpp:278: !use_row_major_kernel or !three_pass_needed info: ROW_MAJOR layout not supported for tensors this large backtrace: --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0xe36ad8) [0x7f301f685ad8] --- ttnn::operations::normalization::layernorm_multi_core(tt::tt_metal::Tensor const&, std::__1::optional<tt::tt_metal::Tensor const> const&, std::__1::optional<tt::tt_metal::Tensor const> const&, std::__1::optional<tt::tt_metal::Tensor const> const&, tt::tt_metal::Tensor&, ttnn::operations::normalization::LayerNormType, float, std::__1::variant<ttnn::GrayskullComputeKernelConfig, ttnn::WormholeComputeKernelConfig>) --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0x1a884d4) [0x7f30202d74d4] --- ttnn::operations::normalization::LayerNorm::create_program(std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>> const&, std::__1::vector<std::__1::optional<tt::tt_metal::Tensor const>, std::__1::allocator<std::__1::optional<tt::tt_metal::Tensor const>>> const&, std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>&) const --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0x1a2ddd9) [0x7f302027cdd9] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0x2f867ad) [0x7f30217d57ad] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0x2f8648e) [0x7f30217d548e] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0x2f862c5) [0x7f30217d52c5] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(_ZN4ttnn16device_operation6detail23launch_on_worker_threadIN2tt8tt_metal9operation23OldInfraDeviceOperationINSt3__16vectorINS4_6TensorENS7_9allocatorIS9_EEEEEENS3_3stl10StrongTypeIhNS_10QueueIdTagEEElNS5_15DeviceOperationISC_EENSD_13tensor_args_tESC_PNS4_2v07IDeviceEEEvT0_T1_RKT2_RKT3_RT4_RT5_+0x270) [0x7f30217d4ea0] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(_ZN4ttnn16device_operation6detail23launch_on_single_deviceIN2tt8tt_metal9operation23OldInfraDeviceOperationINSt3__16vectorINS4_6TensorENS7_9allocatorIS9_EEEEEEEENT_21tensor_return_value_tENS3_3stl10StrongTypeIhNS_10QueueIdTagEEERKNSE_22operation_attributes_tERKNSE_13tensor_args_tE+0x83) [0x7f30217d4b83] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0x2f85a7a) [0x7f30217d4a7a] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(_ZN4ttnn16device_operation6detail6invokeIN2tt8tt_metal9operation23OldInfraDeviceOperationINSt3__16vectorINS4_6TensorENS7_9allocatorIS9_EEEEEEEENT_21tensor_return_value_tENS3_3stl10StrongTypeIhNS_10QueueIdTagEEERKNSE_22operation_attributes_tERKNSE_13tensor_args_tE+0x197) [0x7f30217d4437] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0x2f84deb) [0x7f30217d3deb] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0x2f84acb) [0x7f30217d3acb] --- std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>> tt::tt_metal::operation::run<std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>>(tt::tt_metal::operation::DeviceOperation<std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>>&&, std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>> const&, std::__1::vector<std::__1::optional<tt::tt_metal::Tensor const>, std::__1::allocator<std::__1::optional<tt::tt_metal::Tensor const>>> const&, std::__1::vector<std::__1::optional<tt::tt_metal::Tensor>, std::__1::allocator<std::__1::optional<tt::tt_metal::Tensor>>> const&, tt::stl::StrongType<unsigned char, ttnn::QueueIdTag>) --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0x19fcfe5) [0x7f302024bfe5] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(_ZN4

test_mistral_decoder_inference[128] RuntimeError: TT_FATAL @ /work/ttnn/cpp/ttnn/operations/normalization/layernorm/device/multi_core/layernorm_op_multi_core.cpp:278: !use_row_major_kernel or !three_pass_needed info: ROW_MAJOR layout not supported for tensors this large backtrace: --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0xe36ad8) [0x7f1caa01dad8] --- ttnn::operations::normalization::layernorm_multi_core(tt::tt_metal::Tensor const&, std::__1::optional<tt::tt_metal::Tensor const> const&, std::__1::optional<tt::tt_metal::Tensor const> const&, std::__1::optional<tt::tt_metal::Tensor const> const&, tt::tt_metal::Tensor&, ttnn::operations::normalization::LayerNormType, float, std::__1::variant<ttnn::GrayskullComputeKernelConfig, ttnn::WormholeComputeKernelConfig>) --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0x1a884d4) [0x7f1caac6f4d4] --- ttnn::operations::normalization::LayerNorm::create_program(std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>> const&, std::__1::vector<std::__1::optional<tt::tt_metal::Tensor const>, std::__1::allocator<std::__1::optional<tt::tt_metal::Tensor const>>> const&, std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>&) const --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0x1a2ddd9) [0x7f1caac14dd9] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0x2f867ad) [0x7f1cac16d7ad] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0x2f8648e) [0x7f1cac16d48e] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0x2f862c5) [0x7f1cac16d2c5] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(_ZN4ttnn16device_operation6detail23launch_on_worker_threadIN2tt8tt_metal9operation23OldInfraDeviceOperationINSt3__16vectorINS4_6TensorENS7_9allocatorIS9_EEEEEENS3_3stl10StrongTypeIhNS_10QueueIdTagEEElNS5_15DeviceOperationISC_EENSD_13tensor_args_tESC_PNS4_2v07IDeviceEEEvT0_T1_RKT2_RKT3_RT4_RT5_+0x270) [0x7f1cac16cea0] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(_ZN4ttnn16device_operation6detail23launch_on_single_deviceIN2tt8tt_metal9operation23OldInfraDeviceOperationINSt3__16vectorINS4_6TensorENS7_9allocatorIS9_EEEEEEEENT_21tensor_return_value_tENS3_3stl10StrongTypeIhNS_10QueueIdTagEEERKNSE_22operation_attributes_tERKNSE_13tensor_args_tE+0x83) [0x7f1cac16cb83] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0x2f85a7a) [0x7f1cac16ca7a] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(_ZN4ttnn16device_operation6detail6invokeIN2tt8tt_metal9operation23OldInfraDeviceOperationINSt3__16vectorINS4_6TensorENS7_9allocatorIS9_EEEEEEEENT_21tensor_return_value_tENS3_3stl10StrongTypeIhNS_10QueueIdTagEEERKNSE_22operation_attributes_tERKNSE_13tensor_args_tE+0x197) [0x7f1cac16c437] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0x2f84deb) [0x7f1cac16bdeb] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0x2f84acb) [0x7f1cac16bacb] --- std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>> tt::tt_metal::operation::run<std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>>(tt::tt_metal::operation::DeviceOperation<std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>>&&, std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>> const&, std::__1::vector<std::__1::optional<tt::tt_metal::Tensor const>, std::__1::allocator<std::__1::optional<tt::tt_metal::Tensor const>>> const&, std::__1::vector<std::__1::optional<tt::tt_metal::Tensor>, std::__1::allocator<std::__1::optional<tt::tt_metal::Tensor>>> const&, tt::stl::StrongType<unsigned char, ttnn::QueueIdTag>) --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so(+0x19fcfe5) [0x7f1caabe3fe5] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/ttnn/ttnn/_ttnn.so

Process completed with exit code 1.

Hugepages service found. Command returned with exit code 3. Restarting it so we can ensure hugepages are available

Disk usage is 76 %

Touching and printing out SMI info

tt-smi reset was successful

Hugepages service found. Command returned with exit code 3. Restarting it so we can ensure hugepages are available