From 1ba3e07e86f89b5fb203db0ca0330c67ddf553f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Fri, 17 Jan 2025 14:11:57 +0100 Subject: [PATCH 01/12] =?UTF-8?q?chore(beep=20boop=20=F0=9F=A4=96):=20Bump?= =?UTF-8?q?=20`MCORE=5FTAG=3D004fbcb...`=20(2025-01-16)=20(#11864)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: oliver könig Co-authored-by: pablo-garay <7166088+pablo-garay@users.noreply.github.com> --- Dockerfile.ci | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile.ci b/Dockerfile.ci index f7e637442158..2901f2e30f84 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -49,6 +49,7 @@ RUN pip install nemo_run@git+https://github.com/NVIDIA/NeMo-Run.git@${NEMO_RUN_T # Install NeMo requirements ARG TE_TAG=7d576ed25266a17a7b651f2c12e8498f67e0baea ARG MODELOPT_VERSION=0.21.0 +ARG MCORE_TAG=004fbcb30355d6d5336989fcdd6683fa5cd4d1c0 ARG APEX_TAG=810ffae374a2b9cb4b5c5e28eaeca7d7998fca0c RUN \ --mount=type=bind,source=requirements,target=requirements \ From d4a75d06caa5820383621970199adc0d8831f84e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Sat, 18 Jan 2025 18:36:56 +0100 Subject: [PATCH 02/12] =?UTF-8?q?chore(beep=20boop=20=F0=9F=A4=96):=20Bump?= =?UTF-8?q?=20`MCORE=5FTAG=3De5793c0...`=20(2025-01-17)=20(#11879)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(beep boop 🤖): Bump `MCORE_TAG=e5793c0...` (2025-01-17) Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix Signed-off-by: oliver könig * fix MCoreSelfAttentionMixin Signed-off-by: oliver könig * FalconTransformerLayer Signed-off-by: oliver könig * fixes Signed-off-by: oliver könig --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: oliver könig Co-authored-by: pablo-garay <7166088+pablo-garay@users.noreply.github.com> --- Dockerfile.ci | 2 +- .../language_modeling/megatron/falcon/falcon_decoder_layer.py | 1 + .../megatron/gpt_full_te_layer_autocast_spec.py | 1 + .../nlp/modules/common/megatron/adapters/mcore_mixins.py | 2 ++ nemo/collections/vlm/mllama/model/vision.py | 2 +- 5 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 2901f2e30f84..7145cb0ba147 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -68,7 +68,7 @@ pip install --no-cache-dir --no-build-isolation --extra-index-url https://pypi.n ".[all]" EOF -ARG MCORE_TAG=4dc8977167d71f86bdec47a60a98e85c4cfa0031 +ARG MCORE_TAG=e5793c07227d346d6c644779005f179be12f6895 RUN <<"EOF" bash -ex # Megatron-LM installation git clone https://github.com/NVIDIA/Megatron-LM.git diff --git a/nemo/collections/nlp/models/language_modeling/megatron/falcon/falcon_decoder_layer.py b/nemo/collections/nlp/models/language_modeling/megatron/falcon/falcon_decoder_layer.py index 7c3f3c194f14..bc4d103285a7 100644 --- a/nemo/collections/nlp/models/language_modeling/megatron/falcon/falcon_decoder_layer.py +++ b/nemo/collections/nlp/models/language_modeling/megatron/falcon/falcon_decoder_layer.py @@ -111,6 +111,7 @@ def forward( attention_bias=None, inference_params=None, packed_seq_params=None, + sequence_len_offset=None, ): # hidden_states: [s, b, h] diff --git a/nemo/collections/nlp/models/language_modeling/megatron/gpt_full_te_layer_autocast_spec.py b/nemo/collections/nlp/models/language_modeling/megatron/gpt_full_te_layer_autocast_spec.py index 1def214113ee..955bc659eb42 100644 --- a/nemo/collections/nlp/models/language_modeling/megatron/gpt_full_te_layer_autocast_spec.py +++ b/nemo/collections/nlp/models/language_modeling/megatron/gpt_full_te_layer_autocast_spec.py @@ -255,6 +255,7 @@ def forward( attention_bias=None, inference_params=None, packed_seq_params=None, # TODO: handle this + sequence_len_offset=None, # TODO: handle this ): # Use is_first_microbatch argument during CUDA graph capture. Use self.is_first_microbatch otherwise. hidden_states = super().forward( diff --git a/nemo/collections/nlp/modules/common/megatron/adapters/mcore_mixins.py b/nemo/collections/nlp/modules/common/megatron/adapters/mcore_mixins.py index bba990750adb..e426212bcca6 100644 --- a/nemo/collections/nlp/modules/common/megatron/adapters/mcore_mixins.py +++ b/nemo/collections/nlp/modules/common/megatron/adapters/mcore_mixins.py @@ -85,6 +85,7 @@ def forward( attention_bias: Tensor = None, inference_params: InferenceParams = None, packed_seq_params: PackedSeqParams = None, + sequence_len_offset=None, ): hidden_states = super().forward( hidden_states=hidden_states, @@ -234,6 +235,7 @@ def forward( rotary_pos_cos=None, rotary_pos_sin=None, attention_bias=None, + sequence_len_offset=None, ): # hidden_states: [sq, b, h] diff --git a/nemo/collections/vlm/mllama/model/vision.py b/nemo/collections/vlm/mllama/model/vision.py index bb58ad093cd6..6ecd51ecf29d 100644 --- a/nemo/collections/vlm/mllama/model/vision.py +++ b/nemo/collections/vlm/mllama/model/vision.py @@ -23,7 +23,6 @@ import torch.nn.functional as F from megatron.core import InferenceParams, parallel_state, tensor_parallel from megatron.core.fusions.fused_bias_dropout import get_bias_dropout_add - from megatron.core.packed_seq_params import PackedSeqParams from megatron.core.transformer.attention import SelfAttention, SelfAttentionSubmodules from megatron.core.transformer.custom_layers.transformer_engine import ( @@ -494,6 +493,7 @@ def forward( attention_bias=None, inference_params=None, packed_seq_params=None, + sequence_len_offset=None, ): """Forward.""" # hidden_states: [s, b, h] From 0f83d3193742ffb2d8506fc96747cf2dc0cb2a9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Sat, 18 Jan 2025 18:46:30 +0100 Subject: [PATCH 03/12] allow running on weekly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: oliver könig --- .github/workflows/cicd-main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index 3f7c5e8c3933..1ff1d9ee7320 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -17,6 +17,7 @@ on: branches: - 'main' - 'r**' + - 'weekly-*' types: [ labeled ] workflow_dispatch: From 8ba8a5a40fc4e315a121f944f84a8527fb4e8837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Mon, 20 Jan 2025 11:06:48 +0100 Subject: [PATCH 04/12] fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: oliver könig --- .github/workflows/cicd-main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index 1ff1d9ee7320..aedbb52d07af 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -17,7 +17,7 @@ on: branches: - 'main' - 'r**' - - 'weekly-*' + - 'weekly-bump' types: [ labeled ] workflow_dispatch: From 1e4b7d92353fca203ae014a4c433a60ed285ced3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Tue, 21 Jan 2025 09:30:59 +0100 Subject: [PATCH 05/12] =?UTF-8?q?chore(beep=20boop=20=F0=9F=A4=96):=20Bump?= =?UTF-8?q?=20`MCORE=5FTAG=3D7dd2658...`=20(2025-01-21)=20(#11905)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: pablo-garay <7166088+pablo-garay@users.noreply.github.com> --- Dockerfile.ci | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 7145cb0ba147..872396cfb86e 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -49,7 +49,7 @@ RUN pip install nemo_run@git+https://github.com/NVIDIA/NeMo-Run.git@${NEMO_RUN_T # Install NeMo requirements ARG TE_TAG=7d576ed25266a17a7b651f2c12e8498f67e0baea ARG MODELOPT_VERSION=0.21.0 -ARG MCORE_TAG=004fbcb30355d6d5336989fcdd6683fa5cd4d1c0 +ARG MCORE_TAG=7dd2658cf241098ad9e7b9b1c8526ef520ff6dc6 ARG APEX_TAG=810ffae374a2b9cb4b5c5e28eaeca7d7998fca0c RUN \ --mount=type=bind,source=requirements,target=requirements \ @@ -68,7 +68,7 @@ pip install --no-cache-dir --no-build-isolation --extra-index-url https://pypi.n ".[all]" EOF -ARG MCORE_TAG=e5793c07227d346d6c644779005f179be12f6895 +ARG MCORE_TAG=7dd2658cf241098ad9e7b9b1c8526ef520ff6dc6 RUN <<"EOF" bash -ex # Megatron-LM installation git clone https://github.com/NVIDIA/Megatron-LM.git From e98d7308141989e1d976ff4f8954196a40418c78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Tue, 21 Jan 2025 12:19:19 +0100 Subject: [PATCH 06/12] =?UTF-8?q?chore(beep=20boop=20=F0=9F=A4=96):=20Bump?= =?UTF-8?q?=20`MCORE=5FTAG=3Ddf70c00...`=20(2025-01-21)=20(#11906)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- Dockerfile.ci | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 872396cfb86e..600d285b3777 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -49,7 +49,7 @@ RUN pip install nemo_run@git+https://github.com/NVIDIA/NeMo-Run.git@${NEMO_RUN_T # Install NeMo requirements ARG TE_TAG=7d576ed25266a17a7b651f2c12e8498f67e0baea ARG MODELOPT_VERSION=0.21.0 -ARG MCORE_TAG=7dd2658cf241098ad9e7b9b1c8526ef520ff6dc6 +ARG MCORE_TAG=df70c003f4f544b63305c607fe2e68cba0ac2002 ARG APEX_TAG=810ffae374a2b9cb4b5c5e28eaeca7d7998fca0c RUN \ --mount=type=bind,source=requirements,target=requirements \ @@ -68,7 +68,7 @@ pip install --no-cache-dir --no-build-isolation --extra-index-url https://pypi.n ".[all]" EOF -ARG MCORE_TAG=7dd2658cf241098ad9e7b9b1c8526ef520ff6dc6 +ARG MCORE_TAG=df70c003f4f544b63305c607fe2e68cba0ac2002 RUN <<"EOF" bash -ex # Megatron-LM installation git clone https://github.com/NVIDIA/Megatron-LM.git From 9d0a69e869c5de7ae712d5be7f4531cea1f1ff99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Wed, 22 Jan 2025 22:23:09 +0100 Subject: [PATCH 07/12] =?UTF-8?q?chore(beep=20boop=20=F0=9F=A4=96):=20Bump?= =?UTF-8?q?=20`MCORE=5FTAG=3D5c12382...`=20(2025-01-22)=20(#11925)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- reinstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reinstall.sh b/reinstall.sh index 2c59dd7eb9e3..9f5b969cfe11 100755 --- a/reinstall.sh +++ b/reinstall.sh @@ -24,7 +24,7 @@ export NEMO_RUN_TAG=34259bd3e752fef94045a9a019e4aaf62bd11ce2 export APEX_TAG=810ffae374a2b9cb4b5c5e28eaeca7d7998fca0c export CAUSAL_CONV_TAG=v1.2.2.post1 export MAMBA_TAG=v2.2.0 -export MCORE_TAG=4dc8977167d71f86bdec47a60a98e85c4cfa0031 +export MCORE_TAG=5c1238256853b202c73768a64988f6a7ba411961 export NV_RESILIENCY_EXT_TAG=97aad77609d2e25ed38ac5c99f0c13f93c48464e if [ -n "${NVIDIA_PYTORCH_VERSION}" ]; then From 3cc134e8d925f85f6cd2897f4509005f3499454d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Fri, 24 Jan 2025 08:53:54 +0100 Subject: [PATCH 08/12] =?UTF-8?q?chore(beep=20boop=20=F0=9F=A4=96):=20Bump?= =?UTF-8?q?=20`MCORE=5FTAG=3D0d59157...`=20(2025-01-23)=20(#11932)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(beep boop 🤖): Bump `MCORE_TAG=0d59157...` (2025-01-23) Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * constraint triton Signed-off-by: oliver könig * f Signed-off-by: oliver könig * fix Signed-off-by: oliver könig --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: oliver könig Co-authored-by: pablo-garay <7166088+pablo-garay@users.noreply.github.com> --- reinstall.sh | 3 ++- tests/collections/llm/megatron_mixtral_pretraining.py | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/reinstall.sh b/reinstall.sh index 9f5b969cfe11..7875aa6a215f 100755 --- a/reinstall.sh +++ b/reinstall.sh @@ -24,7 +24,7 @@ export NEMO_RUN_TAG=34259bd3e752fef94045a9a019e4aaf62bd11ce2 export APEX_TAG=810ffae374a2b9cb4b5c5e28eaeca7d7998fca0c export CAUSAL_CONV_TAG=v1.2.2.post1 export MAMBA_TAG=v2.2.0 -export MCORE_TAG=5c1238256853b202c73768a64988f6a7ba411961 +export MCORE_TAG=0d591579d0c55446c4a7a6e20d9c37c7e7263470 export NV_RESILIENCY_EXT_TAG=97aad77609d2e25ed38ac5c99f0c13f93c48464e if [ -n "${NVIDIA_PYTORCH_VERSION}" ]; then @@ -65,6 +65,7 @@ if [[ "$HEAVY_DEPS" == "TRUE" ]]; then "unstructured==0.14.9" "git+https://github.com/Dao-AILab/causal-conv1d.git@${CAUSAL_CONV_TAG}" "git+https://github.com/state-spaces/mamba.git@${MAMBA_TAG}" + "triton==3.1.0" ) pip install --no-cache-dir -r tools/ctc_segmentation/requirements.txt diff --git a/tests/collections/llm/megatron_mixtral_pretraining.py b/tests/collections/llm/megatron_mixtral_pretraining.py index 2a7b1fdfdad6..ee506474ab00 100644 --- a/tests/collections/llm/megatron_mixtral_pretraining.py +++ b/tests/collections/llm/megatron_mixtral_pretraining.py @@ -337,7 +337,15 @@ def main(args): for key, (shape, dtype, device) in expected_ckpt.items(): assert key in ckpt, f"Expected {key} to be in ckpt" assert isinstance(ckpt[key], torch.Tensor), f"Expected {key} to be a tensor" - assert ckpt[key].shape == shape, f"Expected {key} shapes to match {ckpt[key].shape} & {shape}" + + if len(shape) == 1 and key.startswith('optimizer.state'): + assert ckpt[key].shape == ( + 1, + shape[0], + ), f"Expected {key} shapes to match {ckpt[key].shape} & (1, {shape[0]})" + else: + assert ckpt[key].shape == shape, f"Expected {key} shapes to match {ckpt[key].shape} & {shape}" + assert ckpt[key].dtype == dtype, f"Expected {key} dtype to match {ckpt[key].dtype} & {dtype}" assert str(ckpt[key].device) == device, f"Expected {key} device to match {ckpt[key].device} & {device}" From fa8fd5a0aeebc0d1e317b60339b3a849235e2ea1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Fri, 24 Jan 2025 17:19:19 +0100 Subject: [PATCH 09/12] =?UTF-8?q?chore(beep=20boop=20=F0=9F=A4=96):=20Bump?= =?UTF-8?q?=20`MCORE=5FTAG=3D2167226...`=20(2025-01-24)=20(#11947)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: oliver könig Co-authored-by: pablo-garay <7166088+pablo-garay@users.noreply.github.com> --- reinstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reinstall.sh b/reinstall.sh index 7875aa6a215f..015f2f0cfd75 100755 --- a/reinstall.sh +++ b/reinstall.sh @@ -24,7 +24,7 @@ export NEMO_RUN_TAG=34259bd3e752fef94045a9a019e4aaf62bd11ce2 export APEX_TAG=810ffae374a2b9cb4b5c5e28eaeca7d7998fca0c export CAUSAL_CONV_TAG=v1.2.2.post1 export MAMBA_TAG=v2.2.0 -export MCORE_TAG=0d591579d0c55446c4a7a6e20d9c37c7e7263470 +export MCORE_TAG=2167226f9d0426a66b03a1beaabda3015fd44ed2 export NV_RESILIENCY_EXT_TAG=97aad77609d2e25ed38ac5c99f0c13f93c48464e if [ -n "${NVIDIA_PYTORCH_VERSION}" ]; then From 0cc43a2b6d6b2ea88070b63a2e6874e978fbc24c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Mon, 27 Jan 2025 09:37:16 +0100 Subject: [PATCH 10/12] =?UTF-8?q?chore(beep=20boop=20=F0=9F=A4=96):=20Bump?= =?UTF-8?q?=20`MCORE=5FTAG=3Df960d4d...`=20(2025-01-26)=20(#11958)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: pablo-garay <7166088+pablo-garay@users.noreply.github.com> --- reinstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reinstall.sh b/reinstall.sh index 015f2f0cfd75..b91adaf852ec 100755 --- a/reinstall.sh +++ b/reinstall.sh @@ -24,7 +24,7 @@ export NEMO_RUN_TAG=34259bd3e752fef94045a9a019e4aaf62bd11ce2 export APEX_TAG=810ffae374a2b9cb4b5c5e28eaeca7d7998fca0c export CAUSAL_CONV_TAG=v1.2.2.post1 export MAMBA_TAG=v2.2.0 -export MCORE_TAG=2167226f9d0426a66b03a1beaabda3015fd44ed2 +export MCORE_TAG=f960d4decc1fe38b0e013b9837515204bd3d9e54 export NV_RESILIENCY_EXT_TAG=97aad77609d2e25ed38ac5c99f0c13f93c48464e if [ -n "${NVIDIA_PYTORCH_VERSION}" ]; then From 1329f3e224da9005706936368b2c71f67a707f3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Tue, 28 Jan 2025 09:59:20 +0100 Subject: [PATCH 11/12] =?UTF-8?q?chore(beep=20boop=20=F0=9F=A4=96):=20Bump?= =?UTF-8?q?=20`MCORE=5FTAG=3D0e85db5...`=20(2025-01-28)=20(#11967)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: pablo-garay <7166088+pablo-garay@users.noreply.github.com> --- reinstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reinstall.sh b/reinstall.sh index b91adaf852ec..51b0749f6353 100755 --- a/reinstall.sh +++ b/reinstall.sh @@ -24,7 +24,7 @@ export NEMO_RUN_TAG=34259bd3e752fef94045a9a019e4aaf62bd11ce2 export APEX_TAG=810ffae374a2b9cb4b5c5e28eaeca7d7998fca0c export CAUSAL_CONV_TAG=v1.2.2.post1 export MAMBA_TAG=v2.2.0 -export MCORE_TAG=f960d4decc1fe38b0e013b9837515204bd3d9e54 +export MCORE_TAG=0e85db539cf16816ffced6e7dac644d91ffadc04 export NV_RESILIENCY_EXT_TAG=97aad77609d2e25ed38ac5c99f0c13f93c48464e if [ -n "${NVIDIA_PYTORCH_VERSION}" ]; then From 0e9757d4dbf4b772a0b2dcf43a2bb942b58e90fe Mon Sep 17 00:00:00 2001 From: pablo-garay <7166088+pablo-garay@users.noreply.github.com> Date: Thu, 30 Jan 2025 00:21:48 +0000 Subject: [PATCH 12/12] =?UTF-8?q?chore(beep=20boop=20=F0=9F=A4=96):=20Bump?= =?UTF-8?q?=20`MCORE=5FTAG=3Dfb591c7...`=20(2025-01-30)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- reinstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reinstall.sh b/reinstall.sh index 51b0749f6353..dfd5d5c68f5c 100755 --- a/reinstall.sh +++ b/reinstall.sh @@ -24,7 +24,7 @@ export NEMO_RUN_TAG=34259bd3e752fef94045a9a019e4aaf62bd11ce2 export APEX_TAG=810ffae374a2b9cb4b5c5e28eaeca7d7998fca0c export CAUSAL_CONV_TAG=v1.2.2.post1 export MAMBA_TAG=v2.2.0 -export MCORE_TAG=0e85db539cf16816ffced6e7dac644d91ffadc04 +export MCORE_TAG=fb591c757c2d7b8624f893af962bcdb964db8121 export NV_RESILIENCY_EXT_TAG=97aad77609d2e25ed38ac5c99f0c13f93c48464e if [ -n "${NVIDIA_PYTORCH_VERSION}" ]; then