change: Create core integ tests folder and setup

setup.py

@@ -85,8 +85,8 @@ def read_requirements(filename):
 extras["all"] = [item for group in extras.values() for item in group]
 # Tests specific dependencies (do not need to be included in 'all')
 test_dependencies = read_requirements("requirements/extras/test_requirements.txt")
-# remove torch and torchvision if python version is not 3.10
-if sys.version_info.minor != 10:
+# remove torch and torchvision if python version is not 3.10 or 3.11
+if sys.version_info.minor != 10 or sys.version_info.minor != 11:
     test_dependencies = [
         module
         for module in test_dependencies
@@ -124,6 +124,7 @@ def read_requirements(filename):
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
     ],
     install_requires=required_packages,
     extras_require=extras,

tests/integ/core/__init__.py

@@ -0,0 +1,13 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from __future__ import absolute_import

tests/integ/core/conftest.py

@@ -0,0 +1,27 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from __future__ import absolute_import
+
+import pytest
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "core: subset of integ tests that must run for every pull request"
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    for item in items:
+        if "tests/integ/core" in str(item.fspath):
+            item.add_marker(pytest.mark.core)

tests/integ/test_tf.py → tests/integ/core/test_tf.py

@@ -32,7 +32,7 @@
 
 ROLE = "SageMakerRole"
 
-RESOURCE_PATH = os.path.join(os.path.dirname(__file__), "..", "data")
+RESOURCE_PATH = os.path.join(os.path.dirname(__file__), "..", "..", "data")
 MNIST_RESOURCE_PATH = os.path.join(RESOURCE_PATH, "tensorflow_mnist")
 TFS_RESOURCE_PATH = os.path.join(RESOURCE_PATH, "tfs", "tfs-test-entrypoint-with-handler")
 

tests/integ/test_pytorchddp.py

@@ -18,7 +18,7 @@
 import tests.integ as integ
 from sagemaker.pytorch import PyTorch
 from tests.integ import timeout
-from tests.integ.test_pytorch import _upload_training_data
+from tests.integ.core.test_pytorch import _upload_training_data
 
 pytorchddp_dir = os.path.join(os.path.dirname(__file__), "..", "data", "pytorch_ddp")
 

tests/integ/test_smdataparallel_pt.py

@@ -21,7 +21,7 @@
 
 from sagemaker.pytorch import PyTorch
 from tests.integ import timeout
-from tests.integ.test_pytorch import _upload_training_data
+from tests.integ.core.test_pytorch import _upload_training_data
 
 smdataparallel_dir = os.path.join(
     os.path.dirname(__file__), "..", "data", "smdistributed_dataparallel"

tests/integ/test_torch_distributed.py

@@ -18,7 +18,7 @@
 import tests.integ as integ
 from sagemaker.pytorch import PyTorch
 from tests.integ import timeout
-from tests.integ.test_pytorch import _upload_training_data
+from tests.integ.core.test_pytorch import _upload_training_data
 
 torch_distributed_dir = os.path.join(os.path.dirname(__file__), "..", "data", "torch_distributed")
 

tox.ini

@@ -85,6 +85,8 @@ commands =
     pip install 'torch==2.0.1+cpu' -f 'https://download.pytorch.org/whl/torch_stable.html'
     pip install 'torchvision==0.15.2+cpu' -f 'https://download.pytorch.org/whl/torch_stable.html'
     pip install 'dill>=0.3.8'
+    # present in test_requirements but install accelerate specifically to fix Module not found error 
+    pip install 'accelerate>=0.24.1,<=0.27.0'
 
     pytest {posargs}
 deps = .[test]