Use Windows_SDK.json from UE to determine what VS components to install

Author: slonopotamus

docs/ue4-docker-build.adoc

@@ -147,14 +147,6 @@ Use Linux containers under Windows hosts (useful when testing Docker Desktop or
 *--random-memory*::
 Use a random memory limit for Windows containers
 
-*--visual-studio {2017,2019,2022}*::
-Specify Visual Studio Build Tools version.
-+
-By default, ue4-docker uses Visual Studio Build Tools 2017 to build Unreal Engine.
-Starting with Unreal Engine 4.25, you may choose to use Visual Studio Build Tools 2019 instead.
-+
-Unreal Engine 5.0 adds support for VS2022 but removes support for VS2017.
-
 == Environment
 
 This section describes several environment variables that affect how `ue4-docker build` operates.

src/ue4docker/build.py

@@ -200,9 +200,6 @@ def build():
                 "Detected max image size:      {:.0f}GB".format(DockerUtils.maxsize()),
                 False,
             )
-            logger.info(
-                "Visual Studio:                {}".format(config.visualStudio), False
-            )
 
             # Verify that the host OS is not a release that is blacklisted due to critical bugs
             if (
@@ -405,9 +402,6 @@ def build():
                     prereqsArgs = prereqsArgs + [
                         "--build-arg",
                         "DLLSRCIMAGE=" + config.dllSrcImage,
-                        "--build-arg",
-                        "VISUAL_STUDIO_BUILD_NUMBER="
-                        + config.visualStudio.build_number,
                     ]
 
                 custom_prerequisites_dockerfile = config.args.prerequisites_dockerfile

src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile

@@ -56,9 +56,8 @@ LABEL com.adamrehn.ue4-docker.sentinel="1"
 RUN reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f
 
 # Install Chocolatey
-RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "$env:chocolateyVersion = '1.4.0'; Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))"
+RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))"
 
 # Install the rest of our build prerequisites and clean up afterwards to minimise image size
 COPY install-prerequisites.ps1 C:\
-ARG VISUAL_STUDIO_BUILD_NUMBER
-RUN powershell -ExecutionPolicy Bypass -File C:\install-prerequisites.ps1 %VISUAL_STUDIO_BUILD_NUMBER%
+RUN powershell -ExecutionPolicy Bypass -File C:\install-prerequisites.ps1

src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1

@@ -54,69 +54,12 @@ Invoke-WebRequest -Uri "https://sdk.lunarg.com/sdk/download/1.4.304.0/windows/Vu
 Expand-Archive -Path "$env:TEMP\vulkan-runtime-components.zip" -DestinationPath "$env:TEMP"
 Copy-Item -Path "*\x64\vulkan-1.dll" -Destination C:\Windows\System32\
 
-$visual_studio_build = $args[0]
-
-if ($visual_studio_build -eq "15")
-{
-    $windows_sdk_version = 18362
-}
-else
-{
-    $windows_sdk_version = 20348
-}
-
-# NOTE: We use the Visual Studio 2022 installer even for Visual Studio 2019 and 2017 here because the old (2017) installer now breaks
-Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "$env:TEMP\vs_buildtools.exe"
-
-# NOTE: Microsoft.NetCore.Component.SDK only exists for VS2019+. And it is actually *needed* only for UE5
-# NOTE: .NET 4.5 is required for some programs even in UE5, for example https://github.com/EpicGames/UnrealEngine/blob/5.0.1-release/Engine/Source/Programs/UnrealSwarm/SwarmCoordinator/SwarmCoordinator.csproj#L26
-# NOTE: Microsoft.NetCore.Component.Runtime.3.1 is required by the AutomationTool tool and does not come installed with VS2022 so it needs targetting here.
-$vs_args = @(
-    "--quiet",
-    "--wait",
-    "--norestart",
-    "--nocache",
-    "--installPath", "C:\BuildTools",
-    "--channelUri", "https://aka.ms/vs/$visual_studio_build/release/channel",
-    "--installChannelUri", "https://aka.ms/vs/$visual_studio_build/release/channel",
-    "--channelId", "VisualStudio.$visual_studio_build.Release",
-    "--productId", "Microsoft.VisualStudio.Product.BuildTools",
-    "--locale", "en-US",
-    "--add", "Microsoft.VisualStudio.Workload.VCTools",
-    "--add", "Microsoft.VisualStudio.Workload.MSBuildTools",
-    "--add", "Microsoft.VisualStudio.Component.NuGet",
-    "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
-    "--add", "Microsoft.VisualStudio.Component.Windows10SDK.$windows_sdk_version",
-    "--add", "Microsoft.Net.Component.4.5.TargetingPack",
-    "--add", "Microsoft.Net.Component.4.6.2.TargetingPack",
-    "--add", "Microsoft.Net.ComponentGroup.DevelopmentPrerequisites",
-    "--add", "Microsoft.NetCore.Component.SDK",
-    "--add", "Microsoft.NetCore.Component.Runtime.3.1"
-)
-
-# Install the Visual Studio Build Tools workloads and components we need
-RunProcessChecked "$env:TEMP\vs_buildtools.exe" $vs_args
-
-# NOTE: Install the .Net 4.5 Framework Pack via NuGet as it is no longer available via Visual Studio, but still needed
-# NOTE: some programs even in UE5, for example https://github.com/EpicGames/UnrealEngine/blob/5.0.1-release/Engine/Source/Programs/UnrealSwarm/SwarmCoordinator/SwarmCoordinator.csproj#L26
-Invoke-WebRequest -Uri "https://www.nuget.org/api/v2/package/Microsoft.NETFramework.ReferenceAssemblies.net45/1.0.3" -OutFile "$env:TEMP\DotNet45.zip"
-Expand-Archive -Path "$env:TEMP\DotNet45.zip" -DestinationPath "$env:TEMP"
-Copy-Item -Path "$env:TEMP\build\.NETFramework\v4.5\*" -Destination "C:\Program Files (x86)\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5\" -Recurse -Force
-
 # Clean up any temp files generated during prerequisite installation
 Remove-Item -LiteralPath "$env:TEMP" -Recurse -Force
 New-Item -Type directory -Path "$env:TEMP"
 
 # This shaves off ~300MB as of 2021-08-31
 RunProcessChecked "choco-cleaner" @("--dummy")
 
-# Something that gets installed in ue4-build-prerequisites creates a bogus NuGet config file
-# Just remove it, so a proper one will be generated on next NuGet run
-# See https://github.com/adamrehn/ue4-docker/issues/171#issuecomment-852136034
-if (Test-Path "$env:APPDATA\NuGet")
-{
-    Remove-Item -LiteralPath "$env:APPDATA\NuGet" -Recurse -Force
-}
-
 # Display a human-readable completion message
 Write-Output "Finished installing build prerequisites and cleaning up temporary files."

src/ue4docker/dockerfiles/ue4-source/windows/Dockerfile

@@ -1,10 +1,10 @@
 # escape=`
 {% if combine %}
-FROM prerequisites as source
+FROM prerequisites as source-prep
 {% else %}
 ARG NAMESPACE
 ARG PREREQS_TAG
-FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG}
+FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG} as source-prep
 {% endif %}
 
 {% if source_mode == "copy" %}
@@ -79,3 +79,19 @@ RUN Setup.bat -no-cache {{ gitdependencies_args }}
 ARG CHANGELIST
 COPY set-changelist.py C:\set-changelist.py
 RUN python C:\set-changelist.py C:\UnrealEngine\Engine\Build\Build.version %CHANGELIST%
+
+{% if combine %}
+FROM prerequisites as source
+{% else %}
+ARG NAMESPACE
+ARG PREREQS_TAG
+FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG} as source
+{% endif %}
+
+# Only copy files required to determine Visual Studio version to avoid reinstalling it when minor engine changes happen
+COPY --from=source-prep C:\UnrealEngine\Engine\Config\Windows\Windows_SDK.json C:\UnrealEngine\Engine\Config\Windows\Windows_SDK.json
+
+COPY install-vs.py C:\install-vs.py
+RUN python C:\install-vs.py C:\UnrealEngine\Engine\Config\Windows\Windows_SDK.json
+
+COPY --from=source-prep C:\UnrealEngine C:\UnrealEngine

src/ue4docker/dockerfiles/ue4-source/windows/install-vs.py

@@ -0,0 +1,63 @@
+import json
+import subprocess
+import sys
+from typing import Any
+from urllib import request
+
+
+class VisualStudio:
+    def __init__(self, json_config: Any):
+        self.version = json_config["MinimumVisualStudio2022Version"]
+        self.components = sorted(set(
+            [
+                "Microsoft.VisualStudio.Workload.VCTools",
+                "Microsoft.VisualStudio.Workload.MSBuildTools",
+                "Microsoft.VisualStudio.Component.NuGet",
+                "Microsoft.Net.ComponentGroup.DevelopmentPrerequisites",
+                "Microsoft.NetCore.Component.SDK",
+            ]
+            + json_config["VisualStudioSuggestedComponents"]
+            + json_config["VisualStudio2022SuggestedComponents"]
+        ))
+
+    def download_installer(self, installer_path: str):
+        print(f"Downloading Visual Studio {self.version}...")
+        vs_installer_url = (
+            f"https://aka.ms/vs/17/release.ltsc.{self.version}/vs_buildtools.exe"
+        )
+        request.urlretrieve(vs_installer_url, installer_path)
+
+    def install(self, installer_path: str):
+        argv = [
+            installer_path,
+            "--quiet",
+            "--wait",
+            "--norestart",
+            "--nocache",
+            "--installPath",
+            "C:/BuildTools",
+            "--locale",
+            "en-US",
+        ]
+
+        print(f"Installing Visual Studio {self.version}...")
+        print("Components:")
+        for component in self.components:
+            argv.append("--add")
+            argv.append(component)
+            print(f" * {component}")
+
+        sys.stdout.flush()
+
+        subprocess.run(argv, check=True)
+
+
+if __name__ == "__main__":
+    with open(sys.argv[1]) as windows_sdk_file:
+        windows_sdk_json = json.load(windows_sdk_file)
+
+    vs = VisualStudio(windows_sdk_json)
+
+    vs_installer_path = "C:/vs_buildtools.exe"
+    vs.download_installer(vs_installer_path)
+    vs.install(vs_installer_path)

src/ue4docker/infrastructure/BuildConfiguration.py

@@ -1,7 +1,6 @@
 import json
 import platform
 import random
-from typing import Optional
 
 import humanfriendly
 from packaging.version import Version, InvalidVersion
@@ -40,53 +39,6 @@
 }
 
 
-class VisualStudio(object):
-    def __init__(
-        self,
-        name: str,
-        build_number: str,
-        supported_since: Version,
-        unsupported_since: Optional[Version],
-        pass_version_to_buildgraph: bool,
-    ):
-        self.name = name
-        self.build_number = build_number
-        self.supported_since = supported_since
-        self.unsupported_since = unsupported_since
-        self.pass_version_to_buildgraph = pass_version_to_buildgraph
-
-    def __str__(self) -> str:
-        return self.name
-
-
-DefaultVisualStudio = "2017"
-
-VisualStudios = {
-    "2017": VisualStudio(
-        name="2017",
-        build_number="15",
-        # We do not support versions older than 4.27
-        supported_since=Version("4.27"),
-        unsupported_since=Version("5.0"),
-        pass_version_to_buildgraph=False,
-    ),
-    "2019": VisualStudio(
-        name="2019",
-        build_number="16",
-        supported_since=Version("4.27"),
-        unsupported_since=Version("5.4"),
-        pass_version_to_buildgraph=True,
-    ),
-    "2022": VisualStudio(
-        name="2022",
-        build_number="17",
-        supported_since=Version("5.0"),
-        unsupported_since=None,
-        pass_version_to_buildgraph=True,
-    ),
-}
-
-
 class ExcludedComponent(object):
     """
     The different components that we support excluding from the built images
@@ -196,12 +148,6 @@ def addArguments(parser):
             metavar="VERSION",
             help="Add CUDA support as well as OpenGL support when building Linux containers",
         )
-        parser.add_argument(
-            "--visual-studio",
-            default=DefaultVisualStudio,
-            choices=VisualStudios.keys(),
-            help="Specify Visual Studio Build Tools version to use for Windows containers",
-        )
         parser.add_argument(
             "-username",
             default=None,
@@ -476,7 +422,7 @@ def __init__(self, parser, argv, logger):
         self.platformArgs = ["--no-cache"] if self.args.no_cache == True else []
         self.excludedComponents = set(self.args.exclude)
         self.baseImage = None
-        self.prereqsTag = None
+        self.prereqsTag: str | None = None
         self.ignoreBlacklist = self.args.ignore_blacklist
         self.verbose = self.args.verbose
         self.layoutDir = self.args.layout
@@ -605,41 +551,6 @@ def describeExcludedComponents(self):
         )
 
     def _generateWindowsConfig(self):
-        self.visualStudio = VisualStudios.get(self.args.visual_studio)
-        if self.visualStudio is None:
-            raise RuntimeError(
-                f"unknown Visual Studio version: {self.args.visual_studio}"
-            )
-
-        if self.release is not None and not self.custom:
-            # Check whether specified Unreal Engine release is compatible with specified Visual Studio
-            if (
-                self.visualStudio.supported_since is not None
-                and Version(self.release) < self.visualStudio.supported_since
-            ):
-                raise RuntimeError(
-                    f"specified version of Unreal Engine is too old for Visual Studio {self.visualStudio.name}"
-                )
-
-            if (
-                self.visualStudio.unsupported_since is not None
-                and Version(self.release) >= self.visualStudio.unsupported_since
-            ):
-                raise RuntimeError(
-                    "Visual Studio {} is too old for specified version of Unreal Engine".format(
-                        self.visualStudio
-                    )
-                )
-
-        # See https://github.com/EpicGames/UnrealEngine/commit/72585138472785e2ee58aab9950a7260275ee2ac
-        # Note: We must not pass VS2019 arg for older UE4 versions that didn't have VS2019 variable in their build graph xml.
-        # Otherwise, UAT errors out with "Unknown argument: VS2019".
-        if self.visualStudio.pass_version_to_buildgraph:
-            self.opts["buildgraph_args"] = (
-                self.opts.get("buildgraph_args", "")
-                + f" -set:VS{self.visualStudio.name}=true"
-            )
-
         # Determine base tag for the Windows release of the host system
         self.hostBasetag = WindowsUtils.getHostBaseTag()
 
@@ -655,7 +566,7 @@ def _generateWindowsConfig(self):
 
         self.baseImage = "mcr.microsoft.com/windows/servercore:" + self.basetag
         self.dllSrcImage = WindowsUtils.getDllSrcImage(self.basetag)
-        self.prereqsTag = self.basetag + "-vs" + self.visualStudio.name
+        self.prereqsTag = self.basetag
 
         # If the user has explicitly specified an isolation mode then use it, otherwise auto-detect
         if self.args.isolation is not None:

test-suite/test-ue-releases.py

@@ -14,12 +14,11 @@
 
 class UERelease:
     def __init__(
-        self, name: str, tag: str, repo: str, vsVersion: int, ubuntuVersion: str | None
+        self, name: str, tag: str, repo: str, ubuntuVersion: str | None
     ) -> None:
         self.name = name
         self.tag = tag
         self.repo = repo
-        self.vsVersion = vsVersion
         self.ubuntuVersion = ubuntuVersion
 
 
@@ -29,13 +28,13 @@ def __init__(
 
 # The list of Unreal Engine releases that are currently supported by ue4-docker
 SUPPORTED_RELEASES = [
-    UERelease("4.27", "4.27.2-fixed", COMMITDEPS_REPO, 2017, None),
-    UERelease("5.0", "5.0.3-fixed", COMMITDEPS_REPO, 2019, "20.04"),
-    UERelease("5.1", "5.1.1-fixed", COMMITDEPS_REPO, 2019, None),
-    UERelease("5.2", "5.2.1-release", UPSTREAM_REPO, 2022, None),
-    UERelease("5.3", "5.3.2-release", UPSTREAM_REPO, 2022, None),
-    UERelease("5.4", "5.4.4-release", UPSTREAM_REPO, 2022, None),
-    UERelease("5.5", "5.5.1-release", UPSTREAM_REPO, 2022, None),
+    UERelease("4.27", "4.27.2-fixed", COMMITDEPS_REPO, None),
+    UERelease("5.0", "5.0.3-fixed", COMMITDEPS_REPO, "20.04"),
+    UERelease("5.1", "5.1.1-fixed", COMMITDEPS_REPO, None),
+    UERelease("5.2", "5.2.1-release", UPSTREAM_REPO, None),
+    UERelease("5.3", "5.3.2-release", UPSTREAM_REPO, None),
+    UERelease("5.4", "5.4.4-release", UPSTREAM_REPO, None),
+    UERelease("5.5", "5.5.1-release", UPSTREAM_REPO, None),
 ]
 
 
@@ -82,9 +81,7 @@ def testRelease(
     ]
 
     # Apply any platform-specific flags
-    if platform.system() == "Windows":
-        command += ["--visual-studio", release.vsVersion]
-    elif release.ubuntuVersion is not None:
+    if release.ubuntuVersion is not None:
         command += ["-basetag", f"ubuntu{release.ubuntuVersion}"]
 
     # Attempt to run the build