talezero/integration
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lots of work on build-everything

Browse Source
This commit is contained in:
jyelon
2025-06-06 19:12:32 -04:00
parent e26739a0ac
commit 61209d4c2f
4 changed files with 192 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
Integration.code-workspace.tpl.json
View File

@@ -1,38 +1,23 @@
{ {
"macros": { "about-the-build-system": [
"macros-explanation" : [ "",
"We have written a simple macro preprocessor for json.", "The unreal build system generates Integration.code-workspace.",
"The following macros can be invoked from later in the json", "That generated file is no good. Instead, we generate our own,",
"file. The macro preprocessing is done as part of the", "as part of build-everything.py",
"python script build-everything.py" "",
"Don't edit Integration.code-worspace, instead, edit",
"Integration.code-workspace.tpl.json, and then run",
"build-everything.py to rebuild.",
""
], ],
"luprex-launch-config" : {
"name": "Server=SERVER",
"request": "launch",
"program": "UNREALENGINE/Engine/Binaries/Linux/UnrealEditor-Linux-DebugGame",
"preLaunchTask": "Make All",
"args": [
"INTEGRATION/Integration.uproject",
"-userdir=User/USERNAME",
"-LuprexServer=SERVER"
],
"cwd": "INTEGRATION",
"type": "lldb",
"initCommands": [
"command script import UNREALENGINE/Engine/Extras/LLDBDataFormatters/UEDataFormatters_2ByteChars.py",
"settings set target.inline-breakpoint-strategy always",
"target stop-hook add --one-liner \"p ::UngrabAllInputImpl()\""
]
}
},
"folders": [ "folders": [
{ {
"name": "Integration", "name": "Integration",
"path": "INTEGRATION" "path": "[INTEGRATION]"
}, },
{ {
"name": "UE5", "name": "UE5",
"path": "UNREALENGINE" "path": "[UNREALENGINE]"
} }
], ],
"settings": { "settings": {
@@ -93,15 +78,29 @@
}, },
"launch": { "launch": {
"version": "0.2.0", "version": "0.2.0",
"configurations": [ "configurations": {
{ "for-each": [
"macro": "luprex-launch-config", { "SERVER" : "Standalone" },
"vars": { "SERVER":"Standalone"} { "SERVER" : "192.168.0.100" }
}, ],
{ "body": {
"macro": "luprex-launch-config", "name": "Server=[SERVER]",
"vars": { "SERVER":"192.168.0.100"} "request": "launch",
} "program": "[UNREALENGINE]/Engine/Binaries/Linux/UnrealEditor-Linux-DebugGame",
"preLaunchTask": "Make All",
"args": [
"[INTEGRATION]/Integration.uproject",
"-userdir=User/[USER]",
"-LuprexServer=[SERVER]"
],
"cwd": "[INTEGRATION]",
"type": "lldb",
"initCommands": [
"command script import [UNREALENGINE]/Engine/Extras/LLDBDataFormatters/UEDataFormatters_2ByteChars.py",
"settings set target.inline-breakpoint-strategy always",
"target stop-hook add --one-liner \"p ::UngrabAllInputImpl()\""
] ]
} }
} }
}
}

13
Integration.uproject.tpl.json
View File

@@ -1,5 +1,16 @@
{ {
"about-the-build-system": [
"",
"The unreal build system generates Integration.uproject.",
"That generated file is no good. Instead, we generate our own,",
"as part of build-everything.py",
"",
"Don't edit Integration.uproject, instead, edit",
"Integration.uproject.tpl.json, and then run",
"build-everything.py to rebuild.",
""
],
"FileVersion": 3, "FileVersion": 3,
"EngineAssociation": "10.0", "EngineAssociation": "10.0",
"Category": "", "Category": "",

12
Makefile.tpl.txt Normal file
View File

@@ -0,0 +1,12 @@
# This makefile just invokes the unreal build system, then the luprex build system.
#
# Do not edit Makefile, instead, edit Makefile.tpl.txt
#
all:
(cd luprex ; make all)
[UNREALENGINE]/Engine/Build/BatchFiles/[BUILD_BAT] IntegrationEditor [OS] DebugGame [INTEGRATION]/Integration.uproject -waitmutex
clean:
(cd luprex ; make clean)
[UNREALENGINE]/Engine/Build/BatchFiles/[BUILD_BAT] IntegrationEditor [OS] DebugGame [INTEGRATION]/Integration.uproject -waitmutex -clean

307
build-everything.py
View File

@@ -1,207 +1,179 @@
#!/usr/bin/python3 #!/usr/bin/python3
# #
# This python script builds integration from scratch. That includes: # This python script builds integration from scratch. That includes
# everything we need:
# #
# - Generating BuildConfiguration.xml in integration repository # - Generates BuildConfiguration.xml in integration repository
# - Generating BuildConfiguration.xml in UnrealEngine repository # - Generates BuildConfiguration.xml in UnrealEngine repository
# - Hardwiring paths into Source/Integration/lpx-paths.hpp # - Hardwires paths into Source/Integration/lpx-paths.hpp
# - Generating Integration.uproject # - Generates Integration.uproject
# - Generating Integration.code-workspace # - Generates Integration.code-workspace
# - Applies patch to Unreal Engine source. # - Applies patch to Unreal Engine source.
# - Running Setup.sh in the UnrealEngine repository # - Runs Setup.sh in the UnrealEngine repository
# - Building luprex # - Builds luprex
# - Building ShaderCompileWorker # - Builds ShaderCompileWorker
# - Building integration # - Builds Unreal Engine and Unreal Editor
# - Builds integration
# #
# Once this is all done, everything is ready to go. It is now possible to # Once this is all done, everything is ready to go. It is now possible
# start up the IDE and run luprex in the debugger. # to start up the IDE and run luprex in the debugger.
# #
# This script is mainly intended for the *initial* build. # If you edit the code, you can run this python script again to
# If you want to edit the code and recompile, it is okay to use # rebuild. It is always safe to use this script to rebuild after
# this script a second time, but it's unnecessarily slow. # editing anything.
# It's much faster to edit and recompile using the IDE. #
# However, if you only edited C++ code and Unreal Build.cs files, then
# it may be quicker and more convenient to rebuild from inside the
# VSCODE IDE. Bear in mind that doing so only works if you only edited
# the C++ code and the blueprint code. If you edited anything else, you
# should rerun this python script.
# #
import sys, os, json, shutil, subprocess import sys, os, json, shutil, subprocess, re, time
from pathlib import Path from pathlib import Path
# #
# These things are operating system specific. # Build the config table: a set of global constants that affect
# just about everything. These values become global variables
# in this script, they are also used as variables when expanding
# template files.
# #
CONFIG = {}
if sys.platform == "windows": if sys.platform == "windows":
OS = "Windows" CONFIG["OS"] = "Windows"
DLL = "dll" CONFIG["DLL"] = "dll"
BAT = "bat" CONFIG["BAT"] = "bat"
DOT_EXE = ".exe" CONFIG["DOT_EXE"] = ".exe"
USER = "Unknown" CONFIG["USER"] = "Unknown"
BUILD_BAT = "Build.bat" CONFIG["BUILD_BAT"] = "Build.bat"
else: else:
OS = "Linux" CONFIG["OS"] = "Linux"
DLL = "so" CONFIG["DLL"] = "so"
BAT = "sh" CONFIG["BAT"] = "sh"
DOT_EXE = "" CONFIG["DOT_EXE"] = ""
USER = os.environ["USER"] CONFIG["USER"] = os.environ["USER"]
BUILD_BAT = "Linux/Build.sh" CONFIG["BUILD_BAT"] = "Linux/Build.sh"
CONFIG["INTEGRATION"] = os.path.dirname(os.path.abspath(sys.argv[0]))
CONFIG["UNREALENGINE"] = os.path.join(os.path.dirname(CONFIG["INTEGRATION"]), "UnrealEngine")
globals().update(CONFIG)
# #
# Some handy utility functions # Sanity check the INTEGRATION and UNREALENGINE paths.
# #
def readfile(fn):
with open(fn) as f:
return f.read()
def writefile(fn, str):
with open(fn, "w") as f:
f.write(str)
def shell(dir, cmd):
print("Running:", cmd)
subprocess.run(cmd, shell=True, check=True, cwd=dir)
#
# This is the code for a simple json macro preprocessor.
# It is used to write json files containing macros, which can
# then be macroexpanded later in the json file. The json
# file should have a 'macros' section at the top level.
#
def replace_strings_recursively(template, variables):
if isinstance(template, str):
# Then apply macro-local substitutions
for key, value in variables.items():
template = template.replace(key, str(value))
return template
elif isinstance(template, list):
return [replace_strings_recursively(item, variables) for item in template]
elif isinstance(template, dict):
return {k: replace_strings_recursively(v, variables) for k, v in template.items()}
else:
return template
def macroexpand_json_recursively(data, macros):
if isinstance(data, dict):
if "macro" in data and "vars" in data:
macro_name = data["macro"]
variables = data["vars"]
base_macro = macros[macro_name]
expanded = replace_strings_recursively(base_macro, variables)
return macroexpand_json_recursively(expanded, macros)
else:
return {
key: macroexpand_json_recursively(value, macros)
for key, value in data.items()
}
elif isinstance(data, list):
return [macroexpand_json_recursively(item, macros) for item in data]
else:
return data
def macroexpand_json(source_filename, output_filename, globals):
"""
Load JSON from `source_filename`, perform macro expansion using any
macros defined in the "macros" block, and write the expanded JSON to
`output_filename`. Global variables are passed as a Python dict.
"""
# Load the input template
with open(source_filename, "r") as f:
data = json.load(f)
# Extract and remove macros
macros = data.pop("macros", {})
# Expand macros
expanded = macroexpand_json_recursively(data, macros)
# Expand global variables
expanded2 = replace_strings_recursively(expanded, globals)
# Write output
with open(output_filename, "w") as f:
json.dump(expanded2, f, indent=4)
#
# Find the two repositories and verify them.
#
INTEGRATION=os.path.dirname(os.path.abspath(sys.argv[0]))
UNREALENGINE=os.path.join(os.path.dirname(INTEGRATION), "UnrealEngine")
if not os.path.isdir(f"{INTEGRATION}/Source/Integration"): if not os.path.isdir(f"{INTEGRATION}/Source/Integration"):
sys.exit(f"Integration repository is not valid: {INTEGRATION}") sys.exit(f"Integration repository is not valid: {INTEGRATION}")
if not os.path.isdir(f"{UNREALENGINE}/Engine/Source/Editor"): if not os.path.isdir(f"{UNREALENGINE}/Engine/Source/Editor"):
sys.exit(f"Integration repository is not valid: {UNREALENGINE}") sys.exit(f"Integration repository is not valid: {UNREALENGINE}")
JSONGLOBALS= {
"INTEGRATION": INTEGRATION,
"UNREALENGINE": UNREALENGINE,
"USERNAME": USER,
}
# #
# Create the Saved/UnrealBuildTool directories. These will hold # This is the code for a simple json preprocessor that can
# the file BuildConfiguration.xml # expand "for-each" loops and substitute variables. Because
# # a single string is valid json, you can also use this to
# Change directory to one of these in order to force ourselves # substitute variables in a string or a text file.
# to specify all paths explicitly.
# #
Path(f"{INTEGRATION}/Saved/UnrealBuildTool").mkdir(parents=True, exist_ok=True) JSON_VAR_REGEX = re.compile(r'\[([A-Z0-9_]+)\]')
Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool").mkdir(parents=True, exist_ok=True)
os.chdir(f"{INTEGRATION}/Saved/UnrealBuildTool") def expand_json(data, vars):
if isinstance(data, dict):
if "for-each" in data and "body" in data:
body = data["body"]
foreach = data["for-each"]
return [ expand_json(body, vars | lvars) for lvars in foreach ]
else:
return { key: expand_json(value, vars) for key, value in data.items() }
elif isinstance(data, list):
return [ expand_json(item, vars) for item in data ]
elif isinstance(data, str):
return JSON_VAR_REGEX.sub(lambda m: str(vars.get(m.group(1), m.group(0))), data)
else:
return data
# #
# Remove previously-generated files. # Apply the json expander to a file on disk: read the source file,
# apply the expander, write the result back out to disk.
# #
Path(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml").unlink(missing_ok=True) def expand_json_file(sourcefile, outputfile):
Path(f"{INTEGRATION}/Integration.uproject").unlink(missing_ok=True) Path(outputfile).unlink(missing_ok=True)
Path(f"{INTEGRATION}/Integration.code-workspace").unlink(missing_ok=True) data = json.loads(Path(sourcefile).read_text())
Path(f"{INTEGRATION}/Makefile").unlink(missing_ok=True) expanded = expand_json(data, CONFIG)
Path(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp").unlink(missing_ok=True) Path(outputfile).write_text(json.dumps(expanded, indent=4))
Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml").unlink(missing_ok=True)
def expand_text_file(sourcefile, outputfile):
Path(outputfile).unlink(missing_ok=True)
data = Path(sourcefile).read_text()
expanded = expand_json(data, CONFIG)
Path(outputfile).write_text(expanded)
#
# A simplified interface to subprocess.run
#
def shell(dir, cmd):
start = time.time()
subprocess.run(cmd, shell=True, check=True, cwd=dir)
elapsed = time.time() - start
if elapsed > 0.2:
print("")
print("TIMING: ", elapsed, " CMD:", cmd)
print("")
#
# Change directory to an arbitrary subdirectory. Doing this enforces
# the rule that we specify absolute paths for everything.
#
os.chdir(f"{INTEGRATION}/EnginePatches")
# #
# Write BuildConfiguration.xml # Write BuildConfiguration.xml
# #
BUILDCONFIG=readfile(f"{INTEGRATION}/EnginePatches/BuildConfiguration{OS}.xml") BUILDCONFIG=Path(f"{INTEGRATION}/EnginePatches/BuildConfiguration{OS}.xml").read_text()
writefile(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml", BUILDCONFIG)
writefile(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml", BUILDCONFIG) Path(f"{INTEGRATION}/Saved/UnrealBuildTool").mkdir(parents=True, exist_ok=True)
Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool").mkdir(parents=True, exist_ok=True)
Path(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml").unlink(missing_ok=True)
Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml").unlink(missing_ok=True)
Path(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml").write_text(BUILDCONFIG)
Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml").write_text(BUILDCONFIG)
# #
# Write lpx-paths.hpp. # Write lpx-paths.hpp.
# #
writefile(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp", f""" Path(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp").unlink(missing_ok=True)
Path(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp").write_text(f"""
#define LUPREX_DLL_PATH "{INTEGRATION}/luprex/build/{OS}/luprexlib.{DLL}" #define LUPREX_DLL_PATH "{INTEGRATION}/luprex/build/{OS}/luprexlib.{DLL}"
#define LUPREX_ROOT_PATH "{INTEGRATION}/luprex" #define LUPREX_ROOT_PATH "{INTEGRATION}/luprex"
""") """)
# #
# Apply patch to the unreal engine source. # Apply patch to the unreal engine source. Check out HEAD version of
# Restore any affected sourcefiles before applying patch. # affected sourcefiles before applying patch.
# #
print("Applying patch to Unreal Engine...") print("Applying patch to Unreal Engine...")
PATCHED_FILES = [] PATCH_LINES = Path(f"{INTEGRATION}/EnginePatches/EnginePatch").read_text().splitlines()
for line in readfile(f"{INTEGRATION}/EnginePatches/EnginePatch").splitlines(): PATCHED_FILES = [line[6:] for line in PATCH_LINES if line.startswith("--- a/")]
if line.startswith("--- a/"): for file in PATCHED_FILES:
PATCHED_FILES.append(line[6:]) shell(UNREALENGINE, f"git show HEAD:{file} > {file}")
if PATCHED_FILES:
shell(UNREALENGINE, "git checkout HEAD -- " + " ".join(PATCHED_FILES))
shell(UNREALENGINE, f"git apply {INTEGRATION}/EnginePatches/EnginePatch") shell(UNREALENGINE, f"git apply {INTEGRATION}/EnginePatches/EnginePatch")
# #
# Write Integration.uproject. # Write Integration.uproject.
# #
macroexpand_json(f"{INTEGRATION}/Integration.uproject.tpl.json", expand_json_file(f"{INTEGRATION}/Integration.uproject.tpl.json",
f"{INTEGRATION}/Integration.uproject", f"{INTEGRATION}/Integration.uproject")
JSONGLOBALS)
# #
# Run Setup.sh in UNREALENGINE # Run Setup.sh in UNREALENGINE
@@ -209,27 +181,11 @@ macroexpand_json(f"{INTEGRATION}/Integration.uproject.tpl.json",
shell(UNREALENGINE, f"{UNREALENGINE}/Setup.{BAT}") shell(UNREALENGINE, f"{UNREALENGINE}/Setup.{BAT}")
#
# Create a trivial makefile that calls into the unreal build system.
#
writefile(f"{INTEGRATION}/Makefile", f"""
# This makefile just invokes the unreal build system, then the luprex build system.
all:
\t{UNREALENGINE}/Engine/Build/BatchFiles/Linux/Build.sh IntegrationEditor Linux DebugGame {INTEGRATION}/Integration.uproject -waitmutex
\t(cd luprex ; make all)
clean:
\t{UNREALENGINE}/Engine/Build/BatchFiles/Linux/Build.sh IntegrationEditor Linux DebugGame {INTEGRATION}/Integration.uproject -waitmutex -clean
\t(cd luprex ; make clean)
""")
# #
# Use UnrealBuildTool to generate Integration.code-workspace.ubt # Use UnrealBuildTool to generate Integration.code-workspace.ubt
# #
# We're not going to use it, but we keep it as a reference that you can # We're not going to use it, but we keep it as a reference that you can
# use when editing Integration.workspace-template. # use when editing Integration.code-workspace.tpl.json.
# #
Path(f"{INTEGRATION}/Integration.code-workspace").unlink(missing_ok=True) Path(f"{INTEGRATION}/Integration.code-workspace").unlink(missing_ok=True)
@@ -238,19 +194,19 @@ shell(INTEGRATION, f'{UNREALENGINE}/GenerateProjectFiles.{BAT} -projectfiles -pr
Path(f"{INTEGRATION}/Integration.code-workspace").rename(f"{INTEGRATION}/Integration.code-workspace.ubt") Path(f"{INTEGRATION}/Integration.code-workspace").rename(f"{INTEGRATION}/Integration.code-workspace.ubt")
# #
# Build Integration.code-workspace from Integration.workspace-template. # Build Integration.code-workspace from Integration.code-workspace.tpl.json.
# #
macroexpand_json(f"{INTEGRATION}/Integration.code-workspace.tpl.json", expand_json_file(f"{INTEGRATION}/Integration.code-workspace.tpl.json",
f"{INTEGRATION}/Integration.code-workspace", f"{INTEGRATION}/Integration.code-workspace")
JSONGLOBALS)
# #
# Do an initial build of Luprex # Create Makefile from Makefile.tpl.txt
# #
print("Building luprex...") expand_text_file(f"{INTEGRATION}/Makefile.tpl.txt",
shell(f"{INTEGRATION}/luprex", "make") f"{INTEGRATION}/Makefile")
# #
# Build ShaderCompileWorker # Build ShaderCompileWorker
@@ -262,9 +218,8 @@ Path(f"Engine/Binaries/{OS}/ShaderCompileWorker{DOT_EXE}").unlink(missing_ok=Tru
shutil.copyfile(f"{UNREALENGINE}/Engine/Binaries/{OS}/ShaderCompileWorker-{OS}-Shipping{DOT_EXE}", f"{UNREALENGINE}/Engine/Binaries/{OS}/ShaderCompileWorker{DOT_EXE}") shutil.copyfile(f"{UNREALENGINE}/Engine/Binaries/{OS}/ShaderCompileWorker-{OS}-Shipping{DOT_EXE}", f"{UNREALENGINE}/Engine/Binaries/{OS}/ShaderCompileWorker{DOT_EXE}")
# #
# Build Integration # Run Make
# #
print("Building integration...") shell(INTEGRATION, 'make')
shell(INTEGRATION, f'{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} IntegrationEditor {OS} DebugGame -project="{INTEGRATION}/Integration.uproject" -waitmutex')