Update Windows containers (#436)

New packages, new compilers. Changes how environment is prepared on the container. Skips environment initialization by embedding the msvc vars into the global env.
rapidsai · Jan 27, 2025 · 41c90b7 · 41c90b7
1 parent 011fcae
commit 41c90b7
Show file tree

Hide file tree

Showing 16 changed files with 138 additions and 70 deletions.
diff --git a/matrix.yml b/matrix.yml
@@ -160,9 +160,9 @@ include:
   - { features: [{ <<: *cuda_curr_max, <<: *cccl_cuda_opts }, { name: "cl", version: "14.27" }] } # MSVC 2019
   - { features: [{ <<: *cuda_curr_max, <<: *cccl_cuda_opts }, { name: "cl", version: "14.28" }] } # MSVC 2019
   - { features: [{ <<: *cuda_curr_max, <<: *cccl_cuda_opts }, { name: "cl", version: "14.29" }] } # MSVC 2019
-  - { features: [{ <<: *cuda_curr_max, <<: *cccl_cuda_opts }, { name: "cl", version: "14.37" }] } # MSVC 2022
-  - { features: [{ <<: *cuda_curr_max, <<: *cccl_cuda_opts }, { name: "cl", version: "14.38" }] } # MSVC 2022
   - { features: [{ <<: *cuda_curr_max, <<: *cccl_cuda_opts }, { name: "cl", version: "14.39" }] } # MSVC 2022
+  - { features: [{ <<: *cuda_curr_max, <<: *cccl_cuda_opts }, { name: "cl", version: "14.41" }] } # MSVC 2022
+  - { features: [{ <<: *cuda_curr_max, <<: *cccl_cuda_opts }, { name: "cl", version: "14.42" }] } # MSVC 2022
 
 # RAPIDS devcontainers
 

diff --git a/windows/image/install-compiler.ps1 b/windows/image/install-compiler.ps1
@@ -12,7 +12,6 @@ $ErrorActionPreference='Stop'
 
 Push-location "$PSScriptRoot"
 
-## Install older vs cl
 ./installers/install-vs.ps1 -msvcVersion $msvcVersion -clVersion $clVersion
 ./installers/clear-temp.ps1
 

diff --git a/windows/image/install-tools.ps1 b/windows/image/install-tools.ps1
@@ -20,18 +20,6 @@ Push-location "$PSScriptRoot"
 ./installers/install-sccache.ps1
 ./installers/install-tbb.ps1
 
-## Save the current environment without MSVC plugged in
-New-Item -ItemType Directory -Path "$HOME" -Name "build-env"
-
-# Filter these non-portable exported environment variables
-$envFilter = `
-    "COMPUTERNAME","TEMP","TMP","SystemDrive","SystemRoot","USERNAME","USERPROFILE",`
-    "APPDATA","LOCALAPPDATA","NUMBER_OF_PROCESSORS","PROCESSOR_ARCHITECTURE",`
-    "PROCESSOR_IDENTIFIER","PROCESSOR_LEVEL","PROCESSOR_REVISION","OS"
-
-$ENV:INSTALLED_MSVC_VERSION=$msvcVersion
-Get-ChildItem ENV: | Where-Object { $_.Name -notin $envFilter } | Export-CliXml "$HOME\build-env\env-var.clixml"
-
 ./installers/clear-temp.ps1
 
 Pop-Location
diff --git a/windows/image/installers/build-env.psm1 → windows/image/installers/build-env.ps1 b/windows/image/installers/build-env.psm1 → windows/image/installers/build-env.ps1
@@ -1,6 +1,9 @@
-$EnvVarBackup = (Resolve-Path -path "$HOME\build-env\env-var.clixml" | % {$_ -replace '\\','/'})
-# Import the pre-configured environment
-Import-CliXml $EnvVarBackup | % { Set-Item -force -path "env:$($_.Name)" $_.Value }
+param(
+    [string]
+    $vcver="latest"
+)
+
+. "$PSScriptRoot/envvars.ps1"
 
 $MSBuildPath = "C:\msbuild"
 
@@ -35,42 +38,46 @@ $MSBuildPathMap = @{
     "14.37"="$MSBuildPath\17\VC\Auxiliary\Build"
     "14.38"="$MSBuildPath\17\VC\Auxiliary\Build"
     "14.39"="$MSBuildPath\17\VC\Auxiliary\Build"
+    "14.40"="$MSBuildPath\17\VC\Auxiliary\Build"
+    "14.41"="$MSBuildPath\17\VC\Auxiliary\Build"
+    "14.42"="$MSBuildPath\17\VC\Auxiliary\Build"
     "latest"="$MSBuildPath\$ENV:INSTALLED_MSVC_VERSION\VC\Auxiliary\Build"
 }
 
-function Get-VSDevPrompt {
-    param(
-        [string]
-        $vcver="latest"
-    )
+$BuildPath = $MSBuildPathMap[$vcver]
+Write-Output "Loading VC from: $BuildPath"
 
-    if (Test-Path -Path $EnvVarBackup) {
-        Import-CliXml $EnvVarBackup | % { Set-Item -force -path "env:$($_.Name)" $_.Value }
-    }
+# Filter these non-portable exported environment variables
+$envFilter = `
+    "COMPUTERNAME","TEMP","TMP","SystemDrive","SystemRoot","USERNAME","USERPROFILE",`
+    "APPDATA","LOCALAPPDATA","NUMBER_OF_PROCESSORS","PROCESSOR_ARCHITECTURE",`
+    "PROCESSOR_IDENTIFIER","PROCESSOR_LEVEL","PROCESSOR_REVISION","OS","Platform"
 
-    $BuildPath = $MSBuildPathMap[$vcver]
-    Write-Output "Loading VC from: $BuildPath"
+# If a specific version has been requested provide that rather than grab default
+Push-Location "$BuildPath"
+$cmd="vcvars64.bat & set"
+if ($vcver -ne "latest") {
+    $cmd="vcvars64.bat -vcvars_ver=$vcver & set"
+}
 
-    # If a specific version has been requested provide that rather than grab default
-    Push-Location "$BuildPath"
-    $cmd="vcvars64.bat & set"
-    if ($vcver -ne "latest") {
-        $cmd="vcvars64.bat -vcvars_ver=$vcver & set"
-    }
+Set-MachineEnvironmentVariable -Variable "INSTALLED_MSVC_VERSION" -Value "$msvcVersion"
 
-    cmd /c $cmd |
-    foreach {
-        if ($_ -match "=") {
-            $v = $_.split("="); set-item -force -path "ENV:\$($v[0])"  -value "$($v[1])"
+cmd /c $cmd | foreach {
+    if ($_ -match "=") {
+        $v = $_.split("=");
+        if ($v[0] -notin $envFilter) {
+            Set-MachineEnvironmentVariable -Append -Variable "$($v[0])" -Value "$($v[1])"
         }
     }
-    Pop-Location
+}
 
-    # Stupid, but can make CMake happy if it is needed
-    $global:CC_FP = $(get-command cl).Source.Replace("\","/")
+Pop-Location
 
-    Write-Host "`nVisual Studio Command Prompt variables set." -ForegroundColor Yellow
-    Write-Host "Use 'cl' or `$CC_FP as shortcut for Cmake: $CC_FP" -ForegroundColor Yellow
-}
+# Update PATH from machine env
+Write-MachineEnvironmentVariable -Variable "PATH"
+
+# Stupid, but can make CMake happy if it is needed
+Set-MachineEnvironmentVariable -Variable "CC_FP" -Value "$($(get-command cl).Source.Replace("\","/"))"
 
-Export-ModuleMember -Function Get-VSDevPrompt
+Write-Host "`nVisual Studio Command Prompt variables set." -ForegroundColor Yellow
+Write-Host "Use 'cl' or `$CC_FP as shortcut for CMake: $CC_FP" -ForegroundColor Yellow
diff --git a/windows/image/installers/envvars.ps1 b/windows/image/installers/envvars.ps1
@@ -0,0 +1,56 @@
+function Set-MachineEnvironmentVariable {
+    param(
+        [switch]
+        $Append,
+        [string]
+        [parameter(Mandatory=$true)]
+        $Variable,
+        [string]
+        [parameter(Mandatory=$true)]
+        $Value
+    )
+
+    $ProgressPreference = "SilentlyContinue"
+    $ErrorActionPreference = "Stop"
+
+    if ($Append) {
+        $old = [Environment]::GetEnvironmentVariable("$Variable", [EnvironmentVariableTarget]::Machine)
+        if ($old -And $old.Split(';') -icontains "$Value") {
+            Write-Warning "Environment variable already configured"
+            return
+        }
+        ## If $old is null because it is empty, it will fallthrough to non-append
+        elseif ($old) {
+            $Value = "${Value};${old}"
+        }
+    }
+
+    [Environment]::SetEnvironmentVariable("${Variable}", "${Value}", [EnvironmentVariableTarget]::Machine)
+
+    $check = [Environment]::GetEnvironmentVariable("${Variable}", [EnvironmentVariableTarget]::Machine)
+    if ($check -And $check -icontains "${Value}") {
+        Write-Warning "Succesfully set ${Variable} = '${Value}'"
+        return
+    }
+    else {
+        Write-Error "Failed to set ${Variable} = '${Value}'"
+        return
+    }
+
+}
+
+function Write-MachineEnvironmentVariable {
+    param(
+        [string]
+        [parameter(Mandatory=$true)]
+        $Variable
+    )
+
+    $ProgressPreference = "SilentlyContinue"
+    $ErrorActionPreference = "Stop"
+
+    $val = [Environment]::GetEnvironmentVariable("${Variable}", [System.EnvironmentVariableTarget]::Machine)
+    Set-Item -Path "env:${Variable}" -Value "${val}"
+
+    Write-Warning "Set env:${Variable} to $val"
+}
diff --git a/windows/image/installers/install-cmake.ps1 b/windows/image/installers/install-cmake.ps1
@@ -3,4 +3,6 @@ Invoke-WebRequest -Uri "https://github.com/Kitware/CMake/releases/download/v3.27
 Start-Process -NoNewWindow -Wait -FilePath msiexec -ArgumentList "/i C:\cmake_installer.msi ADD_CMAKE_TO_PATH=All /qn"
 Remove-Item "C:\cmake_installer.msi"
 
-$ENV:PATH = "C:\Program Files\CMake\bin;$ENV:PATH"
+. "$PSScriptRoot/envvars.ps1"
+
+Set-MachineEnvironmentVariable -Append -Variable "PATH" -Value "C:\Program Files\CMake\bin"
diff --git a/windows/image/installers/install-cuda.ps1 b/windows/image/installers/install-cuda.ps1
@@ -27,7 +27,7 @@ $cudaMajorUri = @{
 }["$major"]
 
 $cudaVersionUrl = "https://developer.download.nvidia.com/compute/cuda/$cudaMajorUri"
-$cudaComponents = 
+$cudaComponents =
     "nvcc_$mmVersionTag",
     "curand_$mmVersionTag",
     "curand_dev_$mmVersionTag",
@@ -37,11 +37,13 @@ $cudaComponents =
     "nvrtc_dev_$mmVersionTag",
     "nvml_dev_$mmVersionTag",
     "nvtx_$mmVersionTag"
-    
+
 Invoke-WebRequest -Uri "$cudaVersionUrl" -OutFile "./cuda_network.exe" -UseBasicParsing
 Start-Process -Wait -PassThru -FilePath .\cuda_network.exe -ArgumentList "-s $cudaComponents"
 
-$ENV:PATH="$ENV:PATH;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$mmVersionTag\bin"
-$ENV:CUDA_PATH="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$mmVersionTag"
+. "$PSScriptRoot/envvars.ps1"
+
+Set-MachineEnvironmentVariable -Append -Variable "PATH" -Value "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$mmVersionTag\bin"
+Set-MachineEnvironmentVariable -Variable "CUDA_PATH" -Value "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$mmVersionTag"
 
 Remove-Item .\cuda_network.exe
diff --git a/windows/image/installers/install-git.ps1 b/windows/image/installers/install-git.ps1
@@ -3,5 +3,6 @@ Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download
 Start-Process -NoNewWindow -Wait -FilePath "C:\git_installer.exe" -ArgumentList "/S /VERYSILENT /NORESTART /NOCANCEL /SP- /CLOSEAPPLICATIONS /RESTARTAPPLICATIONS"
 Remove-Item "C:\git_installer.exe"
 
-$ENV:PATH = "C:\Program Files\Git\bin;$ENV:PATH"
+. "$PSScriptRoot/envvars.ps1"
 
+Set-MachineEnvironmentVariable -Append -Variable "PATH" -Value "C:\Program Files\Git\bin"
diff --git a/windows/image/installers/install-lit.ps1 b/windows/image/installers/install-lit.ps1
@@ -1,8 +1,13 @@
-Invoke-WebRequest -Uri "https://www.python.org/ftp/python/3.10.7/python-3.10.7-amd64.exe" -OutFile ./python_installer.exe -UseBasicParsing
+Invoke-WebRequest -Uri "https://www.python.org/ftp/python/3.13.1/python-3.13.1-amd64.exe" -OutFile ./python_installer.exe -UseBasicParsing
 Start-Process -Wait -FilePath .\python_installer.exe -ArgumentList "/quiet Include_test=0 InstallAllUsers=1 TargetDir=C:\Python"
 
-$ENV:PATH = "C:\Python;$ENV:PATH"
-$ENV:PATH = "C:\Python\Scripts;$ENV:PATH"
+. "$PSScriptRoot/envvars.ps1"
+
+Set-MachineEnvironmentVariable -Append -Variable "PATH" -Value "C:\Python"
+Set-MachineEnvironmentVariable -Append -Variable "PATH" -Value "C:\Python\Scripts"
+
+# Immediately update envvars
+Write-MachineEnvironmentVariable -Variable "PATH"
 
 python -m pip install --upgrade pip
 python -m pip install wheel

diff --git a/windows/image/installers/install-ninja.ps1 b/windows/image/installers/install-ninja.ps1
@@ -1,6 +1,8 @@
-Invoke-WebRequest -Uri "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-win.zip" -OutFile "./ninja.zip" -UseBasicParsing
+Invoke-WebRequest -Uri "https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-win.zip" -OutFile "./ninja.zip" -UseBasicParsing
 Expand-Archive .\ninja.zip
 
-$ENV:PATH="$ENV:PATH;$(Get-Location)\ninja"
+. "$PSScriptRoot/envvars.ps1"
+
+Set-MachineEnvironmentVariable -Append -Variable "PATH" -Value "$(Get-Location)\ninja"
 
 Remove-Item .\ninja.zip
diff --git a/windows/image/installers/install-sccache.ps1 b/windows/image/installers/install-sccache.ps1
@@ -5,6 +5,8 @@ iex "& {$(irm get.scoop.sh)} -RunAsAdmin"
 
 scoop install sccache
 
-[System.Environment]::SetEnvironmentVariable('CMAKE_CUDA_COMPILER_LAUNCHER','sccache')
-[System.Environment]::SetEnvironmentVariable('CMAKE_CXX_COMPILER_LAUNCHER','sccache')
-[System.Environment]::SetEnvironmentVariable('CMAKE_C_COMPILER_LAUNCHER','sccache')
+. "$PSScriptRoot/envvars.ps1"
+
+Set-MachineEnvironmentVariable -Variable "CMAKE_CUDA_COMPILER_LAUNCHER" -Value "sccache"
+Set-MachineEnvironmentVariable -Variable "CMAKE_CXX_COMPILER_LAUNCHER" -Value "sccache"
+Set-MachineEnvironmentVariable -Variable "CMAKE_C_COMPILER_LAUNCHER" -Value "sccache"
diff --git a/windows/image/installers/install-tbb.ps1 b/windows/image/installers/install-tbb.ps1
@@ -5,8 +5,7 @@ Remove-Item .\tbb.zip
 
 $cwd = Get-Location
 
-# CMake 3.27 or greater can locate packages from this env var:
-$ENV:TBB_ROOT = "$cwd\tbb\oneapi-tbb-$TBB_VERSION"
+. "$PSScriptRoot/envvars.ps1"
 
-# Add dlls to path:
-$ENV:PATH="$ENV:PATH;$ENV:TBB_ROOT\redist\intel64\vc14\"
+Set-MachineEnvironmentVariable -Append -Variable "PATH" -Value "$ENV:TBB_ROOT\redist\intel64\vc14\"
+Set-MachineEnvironmentVariable -Variable "TBB_ROOT" -Value "$cwd\tbb\oneapi-tbb-$TBB_VERSION"
diff --git a/windows/image/installers/install-vs.ps1 b/windows/image/installers/install-vs.ps1
@@ -8,6 +8,7 @@ Param(
     $clVersion
 )
 
+
 $msvcPath = "C:\msbuild\$msvcVersion"
 
 $vsComponentsMap = @{
@@ -26,6 +27,9 @@ $vsComponentsMap = @{
     "14.37"  = "Microsoft.VisualStudio.Component.VC.14.37.17.7.x86.x64"
     "14.38"  = "Microsoft.VisualStudio.Component.VC.14.38.17.8.x86.x64"
     "14.39"  = "Microsoft.VisualStudio.Component.VC.14.39.17.9.x86.x64"
+    "14.40"  = "Microsoft.VisualStudio.Component.VC.14.40.17.10.x86.x64"
+    "14.41"  = "Microsoft.VisualStudio.Component.VC.14.41.17.11.x86.x64"
+    "14.42"  = "Microsoft.VisualStudio.Component.VC.14.42.17.12.x86.x64"
     "latest" = "Microsoft.VisualStudio.Component.VC.Tools.x86.x64"
 }
 
@@ -48,4 +52,7 @@ Invoke-WebRequest -Uri "https://aka.ms/vs/$msvcVersion/release/vs_buildtools.exe
 Write-Output "Installing components: $vsComponentString"
 Start-Process -NoNewWindow -PassThru -Wait -FilePath .\vs_buildtools.exe -ArgumentList "install --installWhileDownloading --installPath $msvcPath --wait --norestart --nocache --quiet $vsComponentString"
 
+# Add VS to the global environment
+. "$PSScriptRoot/build-env.ps1" -vcver "$clVersion"
+
 Remove-Item .\vs_buildtools.exe
diff --git a/windows/image/installers/profile.ps1 b/windows/image/installers/profile.ps1
diff --git a/windows/vs-version-matrix.ps1 b/windows/vs-version-matrix.ps1
@@ -1,7 +1,7 @@
 $global:vsVerToCompilers = @{
     "2017"=("14.14", "14.15", "14.16");
     "2019"=("14.27", "14.28", "14.29");
-    "2022"=("14.35", "14.36", "14.37", "14.38", "14.39");
+    "2022"=("14.35", "14.36", "14.37", "14.38", "14.39", "14.40", "14.41", "14.42");
 }
 
 $global:vsCompilersToYear = @{
@@ -17,6 +17,9 @@ $global:vsCompilersToYear = @{
     "14.37"="2022"
     "14.38"="2022"
     "14.39"="2022"
+    "14.40"="2022"
+    "14.41"="2022"
+    "14.42"="2022"
     "latest"="2022"
 }
 

diff --git a/windows/windows.Dockerfile b/windows/windows.Dockerfile
@@ -1,4 +1,4 @@
-ARG ROOT_IMAGE="mcr.microsoft.com/windows:ltsc2019"
+ARG ROOT_IMAGE
 
 FROM $ROOT_IMAGE as PreCompilerEnv
 
@@ -18,6 +18,3 @@ ARG MSVC_COMPILER_VER
 
 ENV MSVC_COMPILER_VER=${MSVC_COMPILER_VER}
 RUN /tools/install-compiler.ps1 -msvcVersion $ENV:MSVC_VER -clversion $ENV:MSVC_COMPILER_VER
-
-ADD installers/build-env.psm1  /Users/ContainerAdministrator/Documents/WindowsPowerShell/Modules/build-env/build-env.psm1
-ADD installers/profile.ps1  /Users/ContainerAdministrator/Documents/WindowsPowerShell/Microsoft.PowerShell_profile.ps1