talezero/integration
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More work on build.py

Browse Source
This commit is contained in:
jyelon
2025-06-11 20:38:54 -04:00
parent 584cde6f91
commit af69dbe15d
1 changed files with 73 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
build.py
View File

@@ -1,32 +1,35 @@
#!/usr/bin/python3
#
# This python script builds integration from scratch. That includes
# everything we need:
# This build script can be run in three modes:
#
# - Generates BuildConfiguration.xml in integration repository
# - Generates BuildConfiguration.xml in UnrealEngine repository
# - Hardwires paths into Source/Integration/lpx-paths.hpp
# - Generates Integration.uproject
# - Generates Integration.code-workspace
# - Applies patch to Unreal Engine source.
# - Runs Setup.sh in the UnrealEngine repository
# - Builds luprex
# - Builds Unreal Build Tool and Shader Compile Worker
# - Builds Unreal Engine and Unreal Editor
# - Builds integration
# build.py all - rebuilds everything
# build.py c++ - works when you've only edited c++ code
# build.py clean - remove some build products
#
# Once this is all done, everything is ready to go. It is now possible
# to start up the IDE and run luprex in the debugger.
# If you clone the UnrealEngine and integration repositories,
# and then run "build.py all", this will build both unreal engine and
# integration. It will also build project files for vscode, an
# intellisense database, and several other things. Once it is done,
# you can start up vscode using 'code Integration.code-workspace'.
# Then, you can run our game in the vscode debugger.
#
# If you edit the code, you can run this python script again to
# rebuild. It is always safe to use this script to rebuild after
# editing anything.
# You can also use "build.py all" to rebuild. This is the preferred
# way to rebuild when you aren't sure what's been edited. However,
# this takes almost 30 seconds even when there's nothing new to build,
# so it can be a little slow.
#
# If you're sure that the only thing you've edited recently are
# C++ files in the integration repository, then you can get away with
# doing a lightweight build: "build.py c++". This only works if you've
# already completed a successful full build, and the only thing you've
# done since then is edit C++ files in integration. If you've edited
# anything else, you need to use "build.py all" to rebuild.
#
# Editing Lua or Blueprint code doesn't require any kind of rebuild.
#
# However, if you only edited C++ code and Unreal Build.cs files, then
# it may be quicker and more convenient to rebuild from inside the
# VSCODE IDE. Bear in mind that doing so only works if you only edited
# the C++ code and the blueprint code. If you edited anything else, you
# should rerun this python script.
# We have a "build.py clean", but it hasn't gotten a lot of love.
# I honestly have no idea what it removes and what it doesn't.
#
#
import sys, os, json, shutil, subprocess, re, time, tarfile, itertools, hashlib
@@ -73,7 +76,13 @@ def read_if_exists(fn):
JSON_VAR_REGEX = re.compile(r'\[([A-Z0-9_]+)\]')
def expand_json(data, vars):
"A simple JSON preprocessor that can expand for-each loops and substitute variables"
"""
A JSON preprocessor that can expand for-each loops and substitute variables.
The for-loop thing is mainly used by the code-workspace template, to avoid
extreme code duplication in the launch configurations and build configurations.
Variable substitution is used to plug hardwired paths into the file, and also
to implement the for-loop thing.
"""
if isinstance(data, dict):
if "for-each" in data and "body" in data:
body = data["body"]
@@ -90,8 +99,11 @@ def expand_json(data, vars):
else:
return data
def expand_json_file(sourcefile, outputfile, config):
"Apply the json preprocessor to a file on disk"
"""
Apply the json preprocessor to a file on disk
"""
Path(outputfile).unlink(missing_ok=True)
data = json.loads(Path(sourcefile).read_text())
expanded = expand_json(data, vars(config))
@@ -120,7 +132,7 @@ def autodetect_system_config():
This autodetects where integration and unrealengine are installed,
and it also autodetects your operating system. Based on these,
it returns a config object containing a variety of useful
configuration settings.
configuration settings. We haven't tested the Windows version.
"""
config = SimpleNamespace()
if sys.platform == "windows":
@@ -199,7 +211,15 @@ def generate_lpx_paths():
def patch_unrealengine_source_code():
"""
Apply a patch to the UnrealEngine source code.
I tried to avoid patching Unreal engine, but it turns out there are a
few things that I just couldn't avoid having to fix.
Patch application consists of two steps: first, checkout clean, unpatched
versions of the files from git. Then, apply the patch to the clean code.
Note: don't git-commit the patch in UnrealEngine. Instead,
just let this script reapply this patch as necessary.
"""
patch = Path(f"{INTEGRATION}/EnginePatches/EnginePatch")
patch_lines = patch.read_text().splitlines()
patched_files = [line[6:] for line in patch_lines if line.startswith("--- a/")]
@@ -226,20 +246,6 @@ def run_unrealengine_setup_bat():
shell(UNREALENGINE, f"{UNREALENGINE}/Setup.{BAT}")
def generate_integration_code_workspace_ubt():
"""
Use UnrealBuildTool to generate Integration.code-workspace.ubt
We're not going to use it, but we keep it as a reference that you can
use when editing Integration.code-workspace.tpl.json.
"""
workspace = Path(f"{INTEGRATION}/Integration.code-workspace")
workspace_ubt = Path(f"{INTEGRATION}/Integration.code-workspace.ubt")
workspace.unlink(missing_ok=True)
workspace_ubt.unlink(missing_ok=True)
shell(INTEGRATION, f'{UNREALENGINE}/GenerateProjectFiles.{BAT} -projectfiles -project="{INTEGRATION}/Integration.uproject" -game')
workspace.rename(workspace_ubt)
def build_unrealbuildtool():
"""
Build the unreal build tool itself, also including shader compile worker.
@@ -249,7 +255,7 @@ def build_unrealbuildtool():
def build_luprex_and_integration():
"""
This builds our code.
This builds our C++ code, and also UnrealEngine's C++ code.
"""
shell(f"{INTEGRATION}/luprex", "make all")
shell(INTEGRATION, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex IntegrationEditor {OS} DebugGame {INTEGRATION}/Integration.uproject")
@@ -258,9 +264,11 @@ def build_luprex_and_integration():
def build_intellisense_database_for_clangd(force):
"""
This builds compile_commands.json, which tells the intellisense system
based on clangd how to compile each source file. We only rebuild the
file if a C++ file has been added or removed, or if the force argument
is true.
based on clangd how to compile each source file. We automatically rebuild
if a C++ file has been added or removed. That probably isn't sufficient:
we should also rebuild if a Build.cs file has been edited. Coming Soon.
Until then, if your intellisense starts acting badly, do a 'build.py all',
which sets the 'force' argument above to true.
Rebuilding the intellisense database touches rsp files, which
unfortunately, causes the entire C++ build to be restarted the next
@@ -268,6 +276,8 @@ def build_intellisense_database_for_clangd(force):
system. We have a hacky workaround: we save the RSP files before
running the intellisense build, then we restore them afterward.
"""
Path(f"{INTEGRATION}/.vscode").mkdir(exist_ok = True)
Path(f"{UNREALENGINE}/.vscode").mkdir(exist_ok = True)
hash_file = Path(f"{INTEGRATION}/.vscode/cpp_hash")
new_hash = hash_json(find_cpp(f"{INTEGRATION}/Source"))
old_hash = read_if_exists(hash_file).strip()
@@ -293,12 +303,27 @@ def build_intellisense_database_for_clangd(force):
def generate_integration_code_workspace():
"""
We build Integration.code-workspace from a template that we
wrote ourselves, Integration.code-workspace.tpl.json.
We use UnrealBuildTool to generate Integration.code-workspace.ubt,
but we don't use it: we just keep it as a reference that you can
refer to when editing the template.
"""
workspace = f"{INTEGRATION}/Integration.code-workspace"
workspace_ubt = f"{INTEGRATION}/Integration.code-workspace.ubt"
template = f"{INTEGRATION}/Integration.code-workspace.tpl.json"
target = f"{INTEGRATION}/Integration.code-workspace"
expand_json_file(template, target, CONFIG)
Path(workspace).unlink(missing_ok=True)
Path(workspace_ubt).unlink(missing_ok=True)
shell(INTEGRATION, f'{UNREALENGINE}/GenerateProjectFiles.{BAT} -projectfiles -project="{INTEGRATION}/Integration.uproject" -game')
Path(workspace).rename(workspace_ubt)
expand_json_file(template, workspace, CONFIG)
def build_clean():
"""
This code is underdeveloped.
"""
shell(f"{INTEGRATION}/luprex", "make clean")
shell(INTEGRATION, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex IntegrationEditor {OS} DebugGame {INTEGRATION}/Integration.uproject -clean")
Path(f"{INTEGRATION}/.vscode/compile_commands.json").unlink(missing_ok = True)
@@ -312,9 +337,6 @@ CONFIG = autodetect_system_config()
store_system_config_in_globals(CONFIG)
os.chdir(f"{INTEGRATION}/EnginePatches")
if MODE == "experiment":
build_unrealbuildtool()
if MODE == "all":
generate_buildconfiguration_xml()
generate_lpx_paths()
@@ -322,15 +344,12 @@ if MODE == "all":
generate_integration_uproject()
run_unrealengine_setup_bat()
build_unrealbuildtool()
generate_integration_code_workspace()
if MODE in ["all", "c++"]:
build_luprex_and_integration()
build_intellisense_database_for_clangd(MODE == "all")
if MODE == "all":
generate_integration_code_workspace_ubt()
generate_integration_code_workspace()
if MODE == "clean":
build_clean()