New build.py script, replaces build-everything.py and Makefile
This commit is contained in:
334
build.py
Executable file
334
build.py
Executable file
@@ -0,0 +1,334 @@
|
||||
#!/usr/bin/python3
|
||||
#
|
||||
# This python script builds integration from scratch. That includes
|
||||
# everything we need:
|
||||
#
|
||||
# - Generates BuildConfiguration.xml in integration repository
|
||||
# - Generates BuildConfiguration.xml in UnrealEngine repository
|
||||
# - Hardwires paths into Source/Integration/lpx-paths.hpp
|
||||
# - Generates Integration.uproject
|
||||
# - Generates Integration.code-workspace
|
||||
# - Applies patch to Unreal Engine source.
|
||||
# - Runs Setup.sh in the UnrealEngine repository
|
||||
# - Builds luprex
|
||||
# - Builds ShaderCompileWorker
|
||||
# - Builds Unreal Engine and Unreal Editor
|
||||
# - Builds integration
|
||||
#
|
||||
# Once this is all done, everything is ready to go. It is now possible
|
||||
# to start up the IDE and run luprex in the debugger.
|
||||
#
|
||||
# If you edit the code, you can run this python script again to
|
||||
# rebuild. It is always safe to use this script to rebuild after
|
||||
# editing anything.
|
||||
#
|
||||
# However, if you only edited C++ code and Unreal Build.cs files, then
|
||||
# it may be quicker and more convenient to rebuild from inside the
|
||||
# VSCODE IDE. Bear in mind that doing so only works if you only edited
|
||||
# the C++ code and the blueprint code. If you edited anything else, you
|
||||
# should rerun this python script.
|
||||
#
|
||||
|
||||
import sys, os, json, shutil, subprocess, re, time, tarfile, itertools, hashlib
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
||||
#
|
||||
# Utility subroutines
|
||||
#
|
||||
|
||||
def shell(dir, cmd):
|
||||
"Run a shell command in a directory"
|
||||
subprocess.run(cmd, shell=True, check=True, cwd=dir)
|
||||
|
||||
def create_tarfile(directory, glob_pattern, outputfile):
|
||||
"Create a tarfile from a source directory and a glob pattern"
|
||||
directory = Path(directory)
|
||||
with tarfile.open(outputfile, mode='w:gz') as tar:
|
||||
for path in directory.rglob(glob_pattern):
|
||||
if path.is_file():
|
||||
tar.add(path, arcname=path.relative_to(directory))
|
||||
|
||||
def find_cpp(dir):
|
||||
"Find all the C++ and C files in a given directory"
|
||||
list1 = list(Path(dir).rglob("*.[ch]pp"))
|
||||
list2 = list(Path(dir).rglob("*.[ch]"))
|
||||
return sorted([str(x) for x in list1 + list2])
|
||||
|
||||
def hash_json(data):
|
||||
"Calculate a sha256 hexdigest of any valid json data structure"
|
||||
serialized = json.dumps(data, sort_keys=True).encode("utf-8")
|
||||
return hashlib.sha256(serialized).hexdigest()
|
||||
|
||||
def read_if_exists(fn):
|
||||
"Read a file if it exists, otherwise, return null string"
|
||||
try: return Path(fn).read_text()
|
||||
except: return ""
|
||||
|
||||
#
|
||||
# A JSON preprocessor.
|
||||
#
|
||||
|
||||
JSON_VAR_REGEX = re.compile(r'\[([A-Z0-9_]+)\]')
|
||||
|
||||
def expand_json(data, vars):
|
||||
"A simple JSON preprocessor that can expand for-each loops and substitute variables"
|
||||
if isinstance(data, dict):
|
||||
if "for-each" in data and "body" in data:
|
||||
body = data["body"]
|
||||
foreach = data["for-each"]
|
||||
return [ expand_json(body, vars | lvars) for lvars in foreach ]
|
||||
else:
|
||||
return { key: expand_json(value, vars) for key, value in data.items() }
|
||||
elif isinstance(data, list):
|
||||
return [ expand_json(item, vars) for item in data ]
|
||||
elif isinstance(data, str):
|
||||
return JSON_VAR_REGEX.sub(lambda m: str(vars.get(m.group(1), m.group(0))), data)
|
||||
else:
|
||||
return data
|
||||
|
||||
def expand_json_file(sourcefile, outputfile, config):
|
||||
"Apply the json preprocessor to a file on disk"
|
||||
Path(outputfile).unlink(missing_ok=True)
|
||||
data = json.loads(Path(sourcefile).read_text())
|
||||
expanded = expand_json(data, vars(config))
|
||||
Path(outputfile).write_text(json.dumps(expanded, indent=4))
|
||||
|
||||
#
|
||||
# Determining the build configuration.
|
||||
#
|
||||
|
||||
def get_build_mode_from_command_line():
|
||||
"""
|
||||
Build.py accepts a single argument, which should
|
||||
be one of the following modes: all, c++, clean.
|
||||
If nothing is specified, the mode is all.
|
||||
We understand cpp and cxx as synonyms for c++.
|
||||
"""
|
||||
mode = sys.argv[1].lower() if len(sys.argv) > 1 else 'all'
|
||||
if mode in ["cpp", "cxx"]: mode = "c++"
|
||||
if not mode in ["all", "c++", "clean", "experiment"]:
|
||||
sys.exit(f"Invalid build mode: {mode}")
|
||||
return mode
|
||||
|
||||
|
||||
def autodetect_system_config():
|
||||
"""
|
||||
This autodetects where integration and unrealengine are installed,
|
||||
and it also autodetects your operating system. Based on these,
|
||||
it returns a config object containing a variety of useful
|
||||
configuration settings.
|
||||
"""
|
||||
config = SimpleNamespace()
|
||||
if sys.platform == "windows":
|
||||
config.OS = "Windows"
|
||||
config.DLL = "dll"
|
||||
config.BAT = "bat"
|
||||
config.DOT_EXE = ".exe"
|
||||
config.USER = "Unknown"
|
||||
config.BUILD_BAT = "Build.bat"
|
||||
else:
|
||||
config.OS = "Linux"
|
||||
config.DLL = "so"
|
||||
config.BAT = "sh"
|
||||
config.DOT_EXE = ""
|
||||
config.USER = os.environ["USER"]
|
||||
config.BUILD_BAT = "Linux/Build.sh"
|
||||
config.INTEGRATION = os.path.dirname(os.path.abspath(sys.argv[0]))
|
||||
config.UNREALENGINE = os.path.join(os.path.dirname(config.INTEGRATION), "UnrealEngine")
|
||||
test1 = Path(f"{config.INTEGRATION}/Source/Integration")
|
||||
test2 = Path(f"{config.UNREALENGINE}/Engine/Source/Editor")
|
||||
if not test1.is_dir(): sys.exit(f"Integration repository is not valid: {config.INTEGRATION}")
|
||||
if not test2.is_dir(): sys.exit(f"UnrealEngine repository is not valid: {config.UNREALENGINE}")
|
||||
return config
|
||||
|
||||
|
||||
def store_system_config_in_globals(config):
|
||||
"""
|
||||
Copy all the config data from the config object into global variables.
|
||||
"""
|
||||
global OS,DLL,BAT,DOT_EXE,USER,BUILD_BAT,INTEGRATION,UNREALENGINE
|
||||
OS = config.OS
|
||||
DLL = config.DLL
|
||||
BAT = config.BAT
|
||||
DOT_EXE = config.DOT_EXE
|
||||
USER = config.USER
|
||||
BUILD_BAT = config.BUILD_BAT
|
||||
INTEGRATION = config.INTEGRATION
|
||||
UNREALENGINE = config.UNREALENGINE
|
||||
|
||||
#
|
||||
# The actual build steps.
|
||||
#
|
||||
|
||||
def generate_buildconfiguration_xml():
|
||||
"""
|
||||
Generates BuildConfiguration.xml. We actually have two versions of this
|
||||
file in git, one for windows, one for linux. This just copies the appropriate
|
||||
version to the relevant target directories.
|
||||
"""
|
||||
dir1 = Path(f"{INTEGRATION}/Saved/UnrealBuildTool")
|
||||
dir2 = Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool")
|
||||
target1 = Path(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml")
|
||||
target2 = Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml")
|
||||
source = Path(f"{INTEGRATION}/EnginePatches/BuildConfiguration{OS}.xml")
|
||||
template = source.read_text();
|
||||
dir1.mkdir(parents=True, exist_ok=True)
|
||||
dir2.mkdir(parents=True, exist_ok=True)
|
||||
target1.unlink(missing_ok=True)
|
||||
target2.unlink(missing_ok=True)
|
||||
target1.write_text(template)
|
||||
target2.write_text(template)
|
||||
|
||||
|
||||
def generate_lpx_paths():
|
||||
"""
|
||||
Unreal needs to be able to find the Luprex DLL, and it also needs to find the
|
||||
Lua source code. For now, we just compile some hardwired paths into the
|
||||
binary. Someday we'll do something more sophisticated.
|
||||
"""
|
||||
target = Path(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp")
|
||||
line1 = f'#define LUPREX_DLL_PATH "{INTEGRATION}/luprex/build/{OS}/luprexlib.{DLL}"'
|
||||
line2 = f'#define LUPREX_ROOT_PATH "{INTEGRATION}/luprex"'
|
||||
code = line1 + "\n" + line2 + "\n"
|
||||
target.unlink(missing_ok=True)
|
||||
target.write_text(code)
|
||||
|
||||
|
||||
def patch_unrealengine_source_code():
|
||||
|
||||
patch = Path(f"{INTEGRATION}/EnginePatches/EnginePatch")
|
||||
patch_lines = patch.read_text().splitlines()
|
||||
patched_files = [line[6:] for line in patch_lines if line.startswith("--- a/")]
|
||||
for file in patched_files:
|
||||
shell(UNREALENGINE, f"git show HEAD:{file} > {file}")
|
||||
shell(UNREALENGINE, f"git apply {INTEGRATION}/EnginePatches/EnginePatch")
|
||||
|
||||
|
||||
def generate_integration_uproject():
|
||||
"""
|
||||
Generate integration.uproject
|
||||
The uproject file is used by UnrealBuildTool to guide the build process.
|
||||
"""
|
||||
template = f"{INTEGRATION}/Integration.uproject.tpl.json"
|
||||
target = f"{INTEGRATION}/Integration.uproject"
|
||||
expand_json_file(template, target, CONFIG)
|
||||
|
||||
|
||||
def run_unrealengine_setup_bat():
|
||||
"""
|
||||
Run Setup.bat in UnrealEngine.
|
||||
This script downloads assets that aren't stored in git.
|
||||
"""
|
||||
shell(UNREALENGINE, f"{UNREALENGINE}/Setup.{BAT}")
|
||||
|
||||
|
||||
def generate_integration_code_workspace_ubt():
|
||||
"""
|
||||
Use UnrealBuildTool to generate Integration.code-workspace.ubt
|
||||
We're not going to use it, but we keep it as a reference that you can
|
||||
use when editing Integration.code-workspace.tpl.json.
|
||||
"""
|
||||
workspace = Path(f"{INTEGRATION}/Integration.code-workspace")
|
||||
workspace_ubt = Path(f"{INTEGRATION}/Integration.code-workspace.ubt")
|
||||
workspace.unlink(missing_ok=True)
|
||||
workspace_ubt.unlink(missing_ok=True)
|
||||
shell(INTEGRATION, f'{UNREALENGINE}/GenerateProjectFiles.{BAT} -projectfiles -project="{INTEGRATION}/Integration.uproject" -game')
|
||||
workspace.rename(workspace_ubt)
|
||||
|
||||
|
||||
def build_shadercompileworker():
|
||||
"""
|
||||
I have no idea why shadercompileworker isn't built automatically by
|
||||
unreal's build system, but we have to do this separately.
|
||||
"""
|
||||
shell(UNREALENGINE, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex ShaderCompileWorker {OS} Shipping")
|
||||
Path(f"Engine/Binaries/{OS}/ShaderCompileWorker{DOT_EXE}").unlink(missing_ok=True)
|
||||
shutil.copyfile(f"{UNREALENGINE}/Engine/Binaries/{OS}/ShaderCompileWorker-{OS}-Shipping{DOT_EXE}", f"{UNREALENGINE}/Engine/Binaries/{OS}/ShaderCompileWorker{DOT_EXE}")
|
||||
|
||||
|
||||
def build_luprex_and_integration():
|
||||
"""
|
||||
This builds our code.
|
||||
"""
|
||||
shell(f"{INTEGRATION}/luprex", "make all")
|
||||
shell(INTEGRATION, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex IntegrationEditor {OS} DebugGame {INTEGRATION}/Integration.uproject")
|
||||
|
||||
|
||||
def build_intellisense_database_for_clangd(force):
|
||||
"""
|
||||
This builds compile_commands.json, which tells the intellisense system
|
||||
based on clangd how to compile each source file. We only rebuild the
|
||||
file if a C++ file has been added or removed, or if the force argument
|
||||
is true.
|
||||
|
||||
Rebuilding the intellisense database touches rsp files, which
|
||||
unfortunately, causes the entire C++ build to be restarted the next
|
||||
time you run 'build'. This is terrible behavior from Unreal's build
|
||||
system. We have a hacky workaround: we save the RSP files before
|
||||
running the intellisense build, then we restore them afterward.
|
||||
"""
|
||||
hash_file = Path(f"{INTEGRATION}/.vscode/cpp_hash")
|
||||
new_hash = hash_json(find_cpp(f"{INTEGRATION}/Source"))
|
||||
old_hash = read_if_exists(hash_file).strip()
|
||||
if (new_hash != old_hash) or force:
|
||||
hash_file.unlink(missing_ok=True)
|
||||
create_tarfile(f"{INTEGRATION}/Intermediate", "*.rsp", f"{INTEGRATION}/rsp_files.tgz")
|
||||
create_tarfile(f"{UNREALENGINE}/Engine", "*.rsp", f"{UNREALENGINE}/rsp_files.tgz")
|
||||
try:
|
||||
shell(INTEGRATION, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex IntegrationEditor {OS} DebugGame {INTEGRATION}/Integration.uproject -mode=GenerateClangDatabase -OutputDir={UNREALENGINE}/.vscode")
|
||||
shell(UNREALENGINE, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex UnrealEditor {OS} DebugGame -mode=GenerateClangDatabase -OutputDir={UNREALENGINE}/.vscode")
|
||||
shell(INTEGRATION, f"cat {UNREALENGINE}/.vscode/compile_commands.json >> {INTEGRATION}/.vscode/compile_commands.json")
|
||||
except Exception as e:
|
||||
error = e
|
||||
else:
|
||||
error = None
|
||||
finally:
|
||||
tarfile.open(f"{INTEGRATION}/rsp_files.tgz").extractall(path=f"{INTEGRATION}/Intermediate")
|
||||
tarfile.open(f"{UNREALENGINE}/rsp_files.tgz").extractall(path=f"{UNREALENGINE}/Engine")
|
||||
Path(f"{INTEGRATION}/rsp_files.tgz").unlink()
|
||||
Path(f"{UNREALENGINE}/rsp_files.tgz").unlink()
|
||||
if error: raise error
|
||||
hash_file.write_text(new_hash)
|
||||
|
||||
|
||||
def generate_integration_code_workspace():
|
||||
template = f"{INTEGRATION}/Integration.code-workspace.tpl.json"
|
||||
target = f"{INTEGRATION}/Integration.code-workspace"
|
||||
expand_json_file(template, target, CONFIG)
|
||||
|
||||
|
||||
def build_clean():
|
||||
shell(f"{INTEGRATION}/luprex", "make clean")
|
||||
shell(INTEGRATION, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex IntegrationEditor {OS} DebugGame {INTEGRATION}/Integration.uproject -clean")
|
||||
Path(f"{INTEGRATION}/.vscode/compile_commands.json").unlink(missing_ok = True)
|
||||
|
||||
#
|
||||
# MAIN PROGRAM
|
||||
#
|
||||
|
||||
MODE = get_build_mode_from_command_line()
|
||||
CONFIG = autodetect_system_config()
|
||||
store_system_config_in_globals(CONFIG)
|
||||
os.chdir(f"{INTEGRATION}/EnginePatches")
|
||||
|
||||
if MODE == "all":
|
||||
generate_buildconfiguration_xml()
|
||||
generate_lpx_paths()
|
||||
patch_unrealengine_source_code()
|
||||
generate_integration_uproject()
|
||||
run_unrealengine_setup_bat()
|
||||
build_shadercompileworker()
|
||||
|
||||
if MODE in ["all", "c++"]:
|
||||
build_luprex_and_integration()
|
||||
build_intellisense_database_for_clangd(MODE == "all")
|
||||
|
||||
if MODE == "all":
|
||||
generate_integration_code_workspace_ubt()
|
||||
generate_integration_code_workspace()
|
||||
|
||||
if MODE == "clean":
|
||||
build_clean()
|
||||
|
||||
Reference in New Issue
Block a user