diff --git a/build.py b/build.py index 0ee63b5e..2ea245b3 100755 --- a/build.py +++ b/build.py @@ -1,32 +1,35 @@ #!/usr/bin/python3 # -# This python script builds integration from scratch. That includes -# everything we need: +# This build script can be run in three modes: # -# - Generates BuildConfiguration.xml in integration repository -# - Generates BuildConfiguration.xml in UnrealEngine repository -# - Hardwires paths into Source/Integration/lpx-paths.hpp -# - Generates Integration.uproject -# - Generates Integration.code-workspace -# - Applies patch to Unreal Engine source. -# - Runs Setup.sh in the UnrealEngine repository -# - Builds luprex -# - Builds Unreal Build Tool and Shader Compile Worker -# - Builds Unreal Engine and Unreal Editor -# - Builds integration +# build.py all - rebuilds everything +# build.py c++ - works when you've only edited c++ code +# build.py clean - remove some build products # -# Once this is all done, everything is ready to go. It is now possible -# to start up the IDE and run luprex in the debugger. +# If you clone the UnrealEngine and integration repositories, +# and then run "build.py all", this will build both unreal engine and +# integration. It will also build project files for vscode, an +# intellisense database, and several other things. Once it is done, +# you can start up vscode using 'code Integration.code-workspace'. +# Then, you can run our game in the vscode debugger. # -# If you edit the code, you can run this python script again to -# rebuild. It is always safe to use this script to rebuild after -# editing anything. +# You can also use "build.py all" to rebuild. This is the preferred +# way to rebuild when you aren't sure what's been edited. However, +# this takes almost 30 seconds even when there's nothing new to build, +# so it can be a little slow. +# +# If you're sure that the only thing you've edited recently are +# C++ files in the integration repository, then you can get away with +# doing a lightweight build: "build.py c++". This only works if you've +# already completed a successful full build, and the only thing you've +# done since then is edit C++ files in integration. If you've edited +# anything else, you need to use "build.py all" to rebuild. +# +# Editing Lua or Blueprint code doesn't require any kind of rebuild. # -# However, if you only edited C++ code and Unreal Build.cs files, then -# it may be quicker and more convenient to rebuild from inside the -# VSCODE IDE. Bear in mind that doing so only works if you only edited -# the C++ code and the blueprint code. If you edited anything else, you -# should rerun this python script. +# We have a "build.py clean", but it hasn't gotten a lot of love. +# I honestly have no idea what it removes and what it doesn't. +# # import sys, os, json, shutil, subprocess, re, time, tarfile, itertools, hashlib @@ -73,7 +76,13 @@ def read_if_exists(fn): JSON_VAR_REGEX = re.compile(r'\[([A-Z0-9_]+)\]') def expand_json(data, vars): - "A simple JSON preprocessor that can expand for-each loops and substitute variables" + """ + A JSON preprocessor that can expand for-each loops and substitute variables. + The for-loop thing is mainly used by the code-workspace template, to avoid + extreme code duplication in the launch configurations and build configurations. + Variable substitution is used to plug hardwired paths into the file, and also + to implement the for-loop thing. + """ if isinstance(data, dict): if "for-each" in data and "body" in data: body = data["body"] @@ -90,8 +99,11 @@ def expand_json(data, vars): else: return data + def expand_json_file(sourcefile, outputfile, config): - "Apply the json preprocessor to a file on disk" + """ + Apply the json preprocessor to a file on disk + """ Path(outputfile).unlink(missing_ok=True) data = json.loads(Path(sourcefile).read_text()) expanded = expand_json(data, vars(config)) @@ -120,7 +132,7 @@ def autodetect_system_config(): This autodetects where integration and unrealengine are installed, and it also autodetects your operating system. Based on these, it returns a config object containing a variety of useful - configuration settings. + configuration settings. We haven't tested the Windows version. """ config = SimpleNamespace() if sys.platform == "windows": @@ -199,7 +211,15 @@ def generate_lpx_paths(): def patch_unrealengine_source_code(): - + """ + Apply a patch to the UnrealEngine source code. + I tried to avoid patching Unreal engine, but it turns out there are a + few things that I just couldn't avoid having to fix. + Patch application consists of two steps: first, checkout clean, unpatched + versions of the files from git. Then, apply the patch to the clean code. + Note: don't git-commit the patch in UnrealEngine. Instead, + just let this script reapply this patch as necessary. + """ patch = Path(f"{INTEGRATION}/EnginePatches/EnginePatch") patch_lines = patch.read_text().splitlines() patched_files = [line[6:] for line in patch_lines if line.startswith("--- a/")] @@ -226,20 +246,6 @@ def run_unrealengine_setup_bat(): shell(UNREALENGINE, f"{UNREALENGINE}/Setup.{BAT}") -def generate_integration_code_workspace_ubt(): - """ - Use UnrealBuildTool to generate Integration.code-workspace.ubt - We're not going to use it, but we keep it as a reference that you can - use when editing Integration.code-workspace.tpl.json. - """ - workspace = Path(f"{INTEGRATION}/Integration.code-workspace") - workspace_ubt = Path(f"{INTEGRATION}/Integration.code-workspace.ubt") - workspace.unlink(missing_ok=True) - workspace_ubt.unlink(missing_ok=True) - shell(INTEGRATION, f'{UNREALENGINE}/GenerateProjectFiles.{BAT} -projectfiles -project="{INTEGRATION}/Integration.uproject" -game') - workspace.rename(workspace_ubt) - - def build_unrealbuildtool(): """ Build the unreal build tool itself, also including shader compile worker. @@ -249,7 +255,7 @@ def build_unrealbuildtool(): def build_luprex_and_integration(): """ - This builds our code. + This builds our C++ code, and also UnrealEngine's C++ code. """ shell(f"{INTEGRATION}/luprex", "make all") shell(INTEGRATION, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex IntegrationEditor {OS} DebugGame {INTEGRATION}/Integration.uproject") @@ -258,9 +264,11 @@ def build_luprex_and_integration(): def build_intellisense_database_for_clangd(force): """ This builds compile_commands.json, which tells the intellisense system - based on clangd how to compile each source file. We only rebuild the - file if a C++ file has been added or removed, or if the force argument - is true. + based on clangd how to compile each source file. We automatically rebuild + if a C++ file has been added or removed. That probably isn't sufficient: + we should also rebuild if a Build.cs file has been edited. Coming Soon. + Until then, if your intellisense starts acting badly, do a 'build.py all', + which sets the 'force' argument above to true. Rebuilding the intellisense database touches rsp files, which unfortunately, causes the entire C++ build to be restarted the next @@ -268,6 +276,8 @@ def build_intellisense_database_for_clangd(force): system. We have a hacky workaround: we save the RSP files before running the intellisense build, then we restore them afterward. """ + Path(f"{INTEGRATION}/.vscode").mkdir(exist_ok = True) + Path(f"{UNREALENGINE}/.vscode").mkdir(exist_ok = True) hash_file = Path(f"{INTEGRATION}/.vscode/cpp_hash") new_hash = hash_json(find_cpp(f"{INTEGRATION}/Source")) old_hash = read_if_exists(hash_file).strip() @@ -293,12 +303,27 @@ def build_intellisense_database_for_clangd(force): def generate_integration_code_workspace(): + """ + We build Integration.code-workspace from a template that we + wrote ourselves, Integration.code-workspace.tpl.json. + We use UnrealBuildTool to generate Integration.code-workspace.ubt, + but we don't use it: we just keep it as a reference that you can + refer to when editing the template. + """ + workspace = f"{INTEGRATION}/Integration.code-workspace" + workspace_ubt = f"{INTEGRATION}/Integration.code-workspace.ubt" template = f"{INTEGRATION}/Integration.code-workspace.tpl.json" - target = f"{INTEGRATION}/Integration.code-workspace" - expand_json_file(template, target, CONFIG) + Path(workspace).unlink(missing_ok=True) + Path(workspace_ubt).unlink(missing_ok=True) + shell(INTEGRATION, f'{UNREALENGINE}/GenerateProjectFiles.{BAT} -projectfiles -project="{INTEGRATION}/Integration.uproject" -game') + Path(workspace).rename(workspace_ubt) + expand_json_file(template, workspace, CONFIG) def build_clean(): + """ + This code is underdeveloped. + """ shell(f"{INTEGRATION}/luprex", "make clean") shell(INTEGRATION, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex IntegrationEditor {OS} DebugGame {INTEGRATION}/Integration.uproject -clean") Path(f"{INTEGRATION}/.vscode/compile_commands.json").unlink(missing_ok = True) @@ -312,9 +337,6 @@ CONFIG = autodetect_system_config() store_system_config_in_globals(CONFIG) os.chdir(f"{INTEGRATION}/EnginePatches") -if MODE == "experiment": - build_unrealbuildtool() - if MODE == "all": generate_buildconfiguration_xml() generate_lpx_paths() @@ -322,15 +344,12 @@ if MODE == "all": generate_integration_uproject() run_unrealengine_setup_bat() build_unrealbuildtool() + generate_integration_code_workspace() if MODE in ["all", "c++"]: build_luprex_and_integration() build_intellisense_database_for_clangd(MODE == "all") -if MODE == "all": - generate_integration_code_workspace_ubt() - generate_integration_code_workspace() - if MODE == "clean": build_clean()