Lots of work on build-everything

This commit is contained in:
2025-06-06 19:12:32 -04:00
parent e26739a0ac
commit 61209d4c2f
4 changed files with 192 additions and 215 deletions

View File

@@ -1,38 +1,23 @@
{
"macros": {
"macros-explanation" : [
"We have written a simple macro preprocessor for json.",
"The following macros can be invoked from later in the json",
"file. The macro preprocessing is done as part of the",
"python script build-everything.py"
],
"luprex-launch-config" : {
"name": "Server=SERVER",
"request": "launch",
"program": "UNREALENGINE/Engine/Binaries/Linux/UnrealEditor-Linux-DebugGame",
"preLaunchTask": "Make All",
"args": [
"INTEGRATION/Integration.uproject",
"-userdir=User/USERNAME",
"-LuprexServer=SERVER"
],
"cwd": "INTEGRATION",
"type": "lldb",
"initCommands": [
"command script import UNREALENGINE/Engine/Extras/LLDBDataFormatters/UEDataFormatters_2ByteChars.py",
"settings set target.inline-breakpoint-strategy always",
"target stop-hook add --one-liner \"p ::UngrabAllInputImpl()\""
]
}
},
"about-the-build-system": [
"",
"The unreal build system generates Integration.code-workspace.",
"That generated file is no good. Instead, we generate our own,",
"as part of build-everything.py",
"",
"Don't edit Integration.code-worspace, instead, edit",
"Integration.code-workspace.tpl.json, and then run",
"build-everything.py to rebuild.",
""
],
"folders": [
{
"name": "Integration",
"path": "INTEGRATION"
"path": "[INTEGRATION]"
},
{
"name": "UE5",
"path": "UNREALENGINE"
"path": "[UNREALENGINE]"
}
],
"settings": {
@@ -93,15 +78,29 @@
},
"launch": {
"version": "0.2.0",
"configurations": [
{
"macro": "luprex-launch-config",
"vars": { "SERVER":"Standalone"}
},
{
"macro": "luprex-launch-config",
"vars": { "SERVER":"192.168.0.100"}
"configurations": {
"for-each": [
{ "SERVER" : "Standalone" },
{ "SERVER" : "192.168.0.100" }
],
"body": {
"name": "Server=[SERVER]",
"request": "launch",
"program": "[UNREALENGINE]/Engine/Binaries/Linux/UnrealEditor-Linux-DebugGame",
"preLaunchTask": "Make All",
"args": [
"[INTEGRATION]/Integration.uproject",
"-userdir=User/[USER]",
"-LuprexServer=[SERVER]"
],
"cwd": "[INTEGRATION]",
"type": "lldb",
"initCommands": [
"command script import [UNREALENGINE]/Engine/Extras/LLDBDataFormatters/UEDataFormatters_2ByteChars.py",
"settings set target.inline-breakpoint-strategy always",
"target stop-hook add --one-liner \"p ::UngrabAllInputImpl()\""
]
}
]
}
}
}

View File

@@ -1,5 +1,16 @@
{
"about-the-build-system": [
"",
"The unreal build system generates Integration.uproject.",
"That generated file is no good. Instead, we generate our own,",
"as part of build-everything.py",
"",
"Don't edit Integration.uproject, instead, edit",
"Integration.uproject.tpl.json, and then run",
"build-everything.py to rebuild.",
""
],
"FileVersion": 3,
"EngineAssociation": "10.0",
"Category": "",

12
Makefile.tpl.txt Normal file
View File

@@ -0,0 +1,12 @@
# This makefile just invokes the unreal build system, then the luprex build system.
#
# Do not edit Makefile, instead, edit Makefile.tpl.txt
#
all:
(cd luprex ; make all)
[UNREALENGINE]/Engine/Build/BatchFiles/[BUILD_BAT] IntegrationEditor [OS] DebugGame [INTEGRATION]/Integration.uproject -waitmutex
clean:
(cd luprex ; make clean)
[UNREALENGINE]/Engine/Build/BatchFiles/[BUILD_BAT] IntegrationEditor [OS] DebugGame [INTEGRATION]/Integration.uproject -waitmutex -clean

View File

@@ -1,207 +1,179 @@
#!/usr/bin/python3
#
# This python script builds integration from scratch. That includes:
# This python script builds integration from scratch. That includes
# everything we need:
#
# - Generating BuildConfiguration.xml in integration repository
# - Generating BuildConfiguration.xml in UnrealEngine repository
# - Hardwiring paths into Source/Integration/lpx-paths.hpp
# - Generating Integration.uproject
# - Generating Integration.code-workspace
# - Generates BuildConfiguration.xml in integration repository
# - Generates BuildConfiguration.xml in UnrealEngine repository
# - Hardwires paths into Source/Integration/lpx-paths.hpp
# - Generates Integration.uproject
# - Generates Integration.code-workspace
# - Applies patch to Unreal Engine source.
# - Running Setup.sh in the UnrealEngine repository
# - Building luprex
# - Building ShaderCompileWorker
# - Building integration
# - Runs Setup.sh in the UnrealEngine repository
# - Builds luprex
# - Builds ShaderCompileWorker
# - Builds Unreal Engine and Unreal Editor
# - Builds integration
#
# Once this is all done, everything is ready to go. It is now possible to
# start up the IDE and run luprex in the debugger.
# Once this is all done, everything is ready to go. It is now possible
# to start up the IDE and run luprex in the debugger.
#
# This script is mainly intended for the *initial* build.
# If you want to edit the code and recompile, it is okay to use
# this script a second time, but it's unnecessarily slow.
# It's much faster to edit and recompile using the IDE.
# If you edit the code, you can run this python script again to
# rebuild. It is always safe to use this script to rebuild after
# editing anything.
#
# However, if you only edited C++ code and Unreal Build.cs files, then
# it may be quicker and more convenient to rebuild from inside the
# VSCODE IDE. Bear in mind that doing so only works if you only edited
# the C++ code and the blueprint code. If you edited anything else, you
# should rerun this python script.
#
import sys, os, json, shutil, subprocess
import sys, os, json, shutil, subprocess, re, time
from pathlib import Path
#
# These things are operating system specific.
# Build the config table: a set of global constants that affect
# just about everything. These values become global variables
# in this script, they are also used as variables when expanding
# template files.
#
CONFIG = {}
if sys.platform == "windows":
OS = "Windows"
DLL = "dll"
BAT = "bat"
DOT_EXE = ".exe"
USER = "Unknown"
BUILD_BAT = "Build.bat"
CONFIG["OS"] = "Windows"
CONFIG["DLL"] = "dll"
CONFIG["BAT"] = "bat"
CONFIG["DOT_EXE"] = ".exe"
CONFIG["USER"] = "Unknown"
CONFIG["BUILD_BAT"] = "Build.bat"
else:
OS = "Linux"
DLL = "so"
BAT = "sh"
DOT_EXE = ""
USER = os.environ["USER"]
BUILD_BAT = "Linux/Build.sh"
CONFIG["OS"] = "Linux"
CONFIG["DLL"] = "so"
CONFIG["BAT"] = "sh"
CONFIG["DOT_EXE"] = ""
CONFIG["USER"] = os.environ["USER"]
CONFIG["BUILD_BAT"] = "Linux/Build.sh"
CONFIG["INTEGRATION"] = os.path.dirname(os.path.abspath(sys.argv[0]))
CONFIG["UNREALENGINE"] = os.path.join(os.path.dirname(CONFIG["INTEGRATION"]), "UnrealEngine")
globals().update(CONFIG)
#
# Some handy utility functions
# Sanity check the INTEGRATION and UNREALENGINE paths.
#
def readfile(fn):
with open(fn) as f:
return f.read()
def writefile(fn, str):
with open(fn, "w") as f:
f.write(str)
def shell(dir, cmd):
print("Running:", cmd)
subprocess.run(cmd, shell=True, check=True, cwd=dir)
#
# This is the code for a simple json macro preprocessor.
# It is used to write json files containing macros, which can
# then be macroexpanded later in the json file. The json
# file should have a 'macros' section at the top level.
#
def replace_strings_recursively(template, variables):
if isinstance(template, str):
# Then apply macro-local substitutions
for key, value in variables.items():
template = template.replace(key, str(value))
return template
elif isinstance(template, list):
return [replace_strings_recursively(item, variables) for item in template]
elif isinstance(template, dict):
return {k: replace_strings_recursively(v, variables) for k, v in template.items()}
else:
return template
def macroexpand_json_recursively(data, macros):
if isinstance(data, dict):
if "macro" in data and "vars" in data:
macro_name = data["macro"]
variables = data["vars"]
base_macro = macros[macro_name]
expanded = replace_strings_recursively(base_macro, variables)
return macroexpand_json_recursively(expanded, macros)
else:
return {
key: macroexpand_json_recursively(value, macros)
for key, value in data.items()
}
elif isinstance(data, list):
return [macroexpand_json_recursively(item, macros) for item in data]
else:
return data
def macroexpand_json(source_filename, output_filename, globals):
"""
Load JSON from `source_filename`, perform macro expansion using any
macros defined in the "macros" block, and write the expanded JSON to
`output_filename`. Global variables are passed as a Python dict.
"""
# Load the input template
with open(source_filename, "r") as f:
data = json.load(f)
# Extract and remove macros
macros = data.pop("macros", {})
# Expand macros
expanded = macroexpand_json_recursively(data, macros)
# Expand global variables
expanded2 = replace_strings_recursively(expanded, globals)
# Write output
with open(output_filename, "w") as f:
json.dump(expanded2, f, indent=4)
#
# Find the two repositories and verify them.
#
INTEGRATION=os.path.dirname(os.path.abspath(sys.argv[0]))
UNREALENGINE=os.path.join(os.path.dirname(INTEGRATION), "UnrealEngine")
if not os.path.isdir(f"{INTEGRATION}/Source/Integration"):
sys.exit(f"Integration repository is not valid: {INTEGRATION}")
if not os.path.isdir(f"{UNREALENGINE}/Engine/Source/Editor"):
sys.exit(f"Integration repository is not valid: {UNREALENGINE}")
JSONGLOBALS= {
"INTEGRATION": INTEGRATION,
"UNREALENGINE": UNREALENGINE,
"USERNAME": USER,
}
#
# Create the Saved/UnrealBuildTool directories. These will hold
# the file BuildConfiguration.xml
#
# Change directory to one of these in order to force ourselves
# to specify all paths explicitly.
# This is the code for a simple json preprocessor that can
# expand "for-each" loops and substitute variables. Because
# a single string is valid json, you can also use this to
# substitute variables in a string or a text file.
#
Path(f"{INTEGRATION}/Saved/UnrealBuildTool").mkdir(parents=True, exist_ok=True)
Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool").mkdir(parents=True, exist_ok=True)
os.chdir(f"{INTEGRATION}/Saved/UnrealBuildTool")
JSON_VAR_REGEX = re.compile(r'\[([A-Z0-9_]+)\]')
def expand_json(data, vars):
if isinstance(data, dict):
if "for-each" in data and "body" in data:
body = data["body"]
foreach = data["for-each"]
return [ expand_json(body, vars | lvars) for lvars in foreach ]
else:
return { key: expand_json(value, vars) for key, value in data.items() }
elif isinstance(data, list):
return [ expand_json(item, vars) for item in data ]
elif isinstance(data, str):
return JSON_VAR_REGEX.sub(lambda m: str(vars.get(m.group(1), m.group(0))), data)
else:
return data
#
# Remove previously-generated files.
# Apply the json expander to a file on disk: read the source file,
# apply the expander, write the result back out to disk.
#
Path(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml").unlink(missing_ok=True)
Path(f"{INTEGRATION}/Integration.uproject").unlink(missing_ok=True)
Path(f"{INTEGRATION}/Integration.code-workspace").unlink(missing_ok=True)
Path(f"{INTEGRATION}/Makefile").unlink(missing_ok=True)
Path(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp").unlink(missing_ok=True)
Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml").unlink(missing_ok=True)
def expand_json_file(sourcefile, outputfile):
Path(outputfile).unlink(missing_ok=True)
data = json.loads(Path(sourcefile).read_text())
expanded = expand_json(data, CONFIG)
Path(outputfile).write_text(json.dumps(expanded, indent=4))
def expand_text_file(sourcefile, outputfile):
Path(outputfile).unlink(missing_ok=True)
data = Path(sourcefile).read_text()
expanded = expand_json(data, CONFIG)
Path(outputfile).write_text(expanded)
#
# A simplified interface to subprocess.run
#
def shell(dir, cmd):
start = time.time()
subprocess.run(cmd, shell=True, check=True, cwd=dir)
elapsed = time.time() - start
if elapsed > 0.2:
print("")
print("TIMING: ", elapsed, " CMD:", cmd)
print("")
#
# Change directory to an arbitrary subdirectory. Doing this enforces
# the rule that we specify absolute paths for everything.
#
os.chdir(f"{INTEGRATION}/EnginePatches")
#
# Write BuildConfiguration.xml
#
BUILDCONFIG=readfile(f"{INTEGRATION}/EnginePatches/BuildConfiguration{OS}.xml")
writefile(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml", BUILDCONFIG)
writefile(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml", BUILDCONFIG)
BUILDCONFIG=Path(f"{INTEGRATION}/EnginePatches/BuildConfiguration{OS}.xml").read_text()
Path(f"{INTEGRATION}/Saved/UnrealBuildTool").mkdir(parents=True, exist_ok=True)
Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool").mkdir(parents=True, exist_ok=True)
Path(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml").unlink(missing_ok=True)
Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml").unlink(missing_ok=True)
Path(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml").write_text(BUILDCONFIG)
Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml").write_text(BUILDCONFIG)
#
# Write lpx-paths.hpp.
#
writefile(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp", f"""
Path(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp").unlink(missing_ok=True)
Path(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp").write_text(f"""
#define LUPREX_DLL_PATH "{INTEGRATION}/luprex/build/{OS}/luprexlib.{DLL}"
#define LUPREX_ROOT_PATH "{INTEGRATION}/luprex"
""")
#
# Apply patch to the unreal engine source.
# Restore any affected sourcefiles before applying patch.
# Apply patch to the unreal engine source. Check out HEAD version of
# affected sourcefiles before applying patch.
#
print("Applying patch to Unreal Engine...")
PATCHED_FILES = []
for line in readfile(f"{INTEGRATION}/EnginePatches/EnginePatch").splitlines():
if line.startswith("--- a/"):
PATCHED_FILES.append(line[6:])
if PATCHED_FILES:
shell(UNREALENGINE, "git checkout HEAD -- " + " ".join(PATCHED_FILES))
PATCH_LINES = Path(f"{INTEGRATION}/EnginePatches/EnginePatch").read_text().splitlines()
PATCHED_FILES = [line[6:] for line in PATCH_LINES if line.startswith("--- a/")]
for file in PATCHED_FILES:
shell(UNREALENGINE, f"git show HEAD:{file} > {file}")
shell(UNREALENGINE, f"git apply {INTEGRATION}/EnginePatches/EnginePatch")
#
# Write Integration.uproject.
#
macroexpand_json(f"{INTEGRATION}/Integration.uproject.tpl.json",
f"{INTEGRATION}/Integration.uproject",
JSONGLOBALS)
expand_json_file(f"{INTEGRATION}/Integration.uproject.tpl.json",
f"{INTEGRATION}/Integration.uproject")
#
# Run Setup.sh in UNREALENGINE
@@ -209,27 +181,11 @@ macroexpand_json(f"{INTEGRATION}/Integration.uproject.tpl.json",
shell(UNREALENGINE, f"{UNREALENGINE}/Setup.{BAT}")
#
# Create a trivial makefile that calls into the unreal build system.
#
writefile(f"{INTEGRATION}/Makefile", f"""
# This makefile just invokes the unreal build system, then the luprex build system.
all:
\t{UNREALENGINE}/Engine/Build/BatchFiles/Linux/Build.sh IntegrationEditor Linux DebugGame {INTEGRATION}/Integration.uproject -waitmutex
\t(cd luprex ; make all)
clean:
\t{UNREALENGINE}/Engine/Build/BatchFiles/Linux/Build.sh IntegrationEditor Linux DebugGame {INTEGRATION}/Integration.uproject -waitmutex -clean
\t(cd luprex ; make clean)
""")
#
# Use UnrealBuildTool to generate Integration.code-workspace.ubt
#
# We're not going to use it, but we keep it as a reference that you can
# use when editing Integration.workspace-template.
# use when editing Integration.code-workspace.tpl.json.
#
Path(f"{INTEGRATION}/Integration.code-workspace").unlink(missing_ok=True)
@@ -238,19 +194,19 @@ shell(INTEGRATION, f'{UNREALENGINE}/GenerateProjectFiles.{BAT} -projectfiles -pr
Path(f"{INTEGRATION}/Integration.code-workspace").rename(f"{INTEGRATION}/Integration.code-workspace.ubt")
#
# Build Integration.code-workspace from Integration.workspace-template.
# Build Integration.code-workspace from Integration.code-workspace.tpl.json.
#
macroexpand_json(f"{INTEGRATION}/Integration.code-workspace.tpl.json",
f"{INTEGRATION}/Integration.code-workspace",
JSONGLOBALS)
expand_json_file(f"{INTEGRATION}/Integration.code-workspace.tpl.json",
f"{INTEGRATION}/Integration.code-workspace")
#
# Do an initial build of Luprex
# Create Makefile from Makefile.tpl.txt
#
print("Building luprex...")
shell(f"{INTEGRATION}/luprex", "make")
expand_text_file(f"{INTEGRATION}/Makefile.tpl.txt",
f"{INTEGRATION}/Makefile")
#
# Build ShaderCompileWorker
@@ -262,9 +218,8 @@ Path(f"Engine/Binaries/{OS}/ShaderCompileWorker{DOT_EXE}").unlink(missing_ok=Tru
shutil.copyfile(f"{UNREALENGINE}/Engine/Binaries/{OS}/ShaderCompileWorker-{OS}-Shipping{DOT_EXE}", f"{UNREALENGINE}/Engine/Binaries/{OS}/ShaderCompileWorker{DOT_EXE}")
#
# Build Integration
# Run Make
#
print("Building integration...")
shell(INTEGRATION, f'{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} IntegrationEditor {OS} DebugGame -project="{INTEGRATION}/Integration.uproject" -waitmutex')
shell(INTEGRATION, 'make')