Moved sharktank runner to ossci cluster

.github/workflows/ci-sharktank.yml

@@ -93,15 +93,16 @@ jobs:
     strategy:
       matrix:
         python-version: [3.11]
-        runs-on: [llama-mi300x-3]
+        runs-on: [linux-mi300-1gpu-ossci]
       fail-fast: false
     runs-on: ${{matrix.runs-on}}
     defaults:
       run:
         shell: bash
     env:
       VENV_DIR: ${{ github.workspace }}/.venv
-      HF_HOME: "/data/huggingface"
+      HF_HOME: "/shark-cache/data/huggingface"
+      HF_TOKEN: ${{ secrets.HF_FLUX_TOKEN }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
@@ -149,7 +150,7 @@ jobs:
             sharktank/tests/models/vae/vae_test.py \
             sharktank/tests/models/llama/quark_parity_test.py \
             --durations=0 \
-            --timeout=800
+            --timeout=10000
 # TODO: add back
 #             --with-t5-data \
 # when #888 is resolved
@@ -193,7 +194,7 @@ jobs:
         run: |
           pytest -v sharktank/ -m punet_quick \
             --durations=0 \
-            --timeout=600
+            --timeout=900
 
   # Depends on other jobs to provide an aggregate job status.
   # TODO(#584): move test_with_data and test_integration to a pkgci integration test workflow?

sharktank/conftest.py

@@ -191,25 +191,25 @@ def pytest_addoption(parser):
     parser.addoption(
         "--google-t5-v1-1-small-f32-model-path",
         type=Path,
-        default="/data/t5/small/google__t5-v1_1-small_f32.gguf",
+        default="/shark-dev/data/t5/small/google__t5-v1_1-small_f32.gguf",
         help="Google T5 v1.1 small float32 model path",
     )
     parser.addoption(
         "--google-t5-v1-1-small-bf16-model-path",
         type=Path,
-        default="/data/t5/small/google__t5-v1_1-small_bf16.gguf",
+        default="/shark-dev/data/t5/small/google__t5-v1_1-small_bf16.gguf",
         help="Google T5 v1.1 small bfloat16 model path",
     )
     parser.addoption(
         "--google-t5-v1-1-xxl-f32-model-path",
         type=Path,
-        default="/data/t5/xxl/google__t5-v1_1-xxl_f32.gguf",
+        default="/shark-dev/data/t5/xxl/google__t5-v1_1-xxl_f32.gguf",
         help="Google T5 v1.1 XXL float32 model path",
     )
     parser.addoption(
         "--google-t5-v1-1-xxl-bf16-model-path",
         type=Path,
-        default="/data/t5/xxl/google__t5-v1_1-xxl_bf16.gguf",
+        default="/shark-dev/data/t5/xxl/google__t5-v1_1-xxl_bf16.gguf",
         help="Google T5 v1.1 XXL bfloat16 model path",
     )
 

sharktank/tests/models/llama/quark_parity_test.py

@@ -18,7 +18,7 @@
 class QuarkParityTest(unittest.TestCase):
     def setUp(self):
         super().setUp()
-        self.path_prefix = Path("/shark-dev/quark_test")
+        self.path_prefix = Path("/shark-cache/quark_test")
 
     @with_quark_data
     def test_compare_against_quark(self):
@@ -54,7 +54,7 @@ def test_compare_against_quark(self):
             "sharktank.examples.paged_llm_v1",
             "The capitol of Texas is",
             f"--irpa-file={self.path_prefix}/fp8_bf16_weight.irpa",
-            f"--tokenizer-config-json=/data/llama3.1/8b/tokenizer.json",
+            f"--tokenizer-config-json=/shark-dev/data/llama3.1/weights/8b/tokenizer.json",
             "--fake-quant",
             "--attention-kernel=torch",
             "--activation-dtype=bfloat16",
@@ -69,6 +69,10 @@ def test_compare_against_quark(self):
             command, shell=True, capture_output=True, cwd=sharktank_dir
         )
 
+        f_ = open("/shark-cache/quark_test/test0.txt", "w+")
+        f_.write(str(proc))
+        f_.close()
+
         ours = dict()
         with safe_open(our_path, "pytorch") as st:
             for key in st.keys():