diff --git a/Integration.code-workspace.tpl.json b/Integration.code-workspace.tpl.json index 821203c2..1402875e 100644 --- a/Integration.code-workspace.tpl.json +++ b/Integration.code-workspace.tpl.json @@ -1,38 +1,23 @@ { - "macros": { - "macros-explanation" : [ - "We have written a simple macro preprocessor for json.", - "The following macros can be invoked from later in the json", - "file. The macro preprocessing is done as part of the", - "python script build-everything.py" - ], - "luprex-launch-config" : { - "name": "Server=SERVER", - "request": "launch", - "program": "UNREALENGINE/Engine/Binaries/Linux/UnrealEditor-Linux-DebugGame", - "preLaunchTask": "Make All", - "args": [ - "INTEGRATION/Integration.uproject", - "-userdir=User/USERNAME", - "-LuprexServer=SERVER" - ], - "cwd": "INTEGRATION", - "type": "lldb", - "initCommands": [ - "command script import UNREALENGINE/Engine/Extras/LLDBDataFormatters/UEDataFormatters_2ByteChars.py", - "settings set target.inline-breakpoint-strategy always", - "target stop-hook add --one-liner \"p ::UngrabAllInputImpl()\"" - ] - } - }, + "about-the-build-system": [ + "", + "The unreal build system generates Integration.code-workspace.", + "That generated file is no good. Instead, we generate our own,", + "as part of build-everything.py", + "", + "Don't edit Integration.code-worspace, instead, edit", + "Integration.code-workspace.tpl.json, and then run", + "build-everything.py to rebuild.", + "" + ], "folders": [ { "name": "Integration", - "path": "INTEGRATION" + "path": "[INTEGRATION]" }, { "name": "UE5", - "path": "UNREALENGINE" + "path": "[UNREALENGINE]" } ], "settings": { @@ -93,15 +78,29 @@ }, "launch": { "version": "0.2.0", - "configurations": [ - { - "macro": "luprex-launch-config", - "vars": { "SERVER":"Standalone"} - }, - { - "macro": "luprex-launch-config", - "vars": { "SERVER":"192.168.0.100"} + "configurations": { + "for-each": [ + { "SERVER" : "Standalone" }, + { "SERVER" : "192.168.0.100" } + ], + "body": { + "name": "Server=[SERVER]", + "request": "launch", + "program": "[UNREALENGINE]/Engine/Binaries/Linux/UnrealEditor-Linux-DebugGame", + "preLaunchTask": "Make All", + "args": [ + "[INTEGRATION]/Integration.uproject", + "-userdir=User/[USER]", + "-LuprexServer=[SERVER]" + ], + "cwd": "[INTEGRATION]", + "type": "lldb", + "initCommands": [ + "command script import [UNREALENGINE]/Engine/Extras/LLDBDataFormatters/UEDataFormatters_2ByteChars.py", + "settings set target.inline-breakpoint-strategy always", + "target stop-hook add --one-liner \"p ::UngrabAllInputImpl()\"" + ] } - ] + } } } \ No newline at end of file diff --git a/Integration.uproject.tpl.json b/Integration.uproject.tpl.json index 56b727f8..274836d7 100644 --- a/Integration.uproject.tpl.json +++ b/Integration.uproject.tpl.json @@ -1,5 +1,16 @@ - { + "about-the-build-system": [ + "", + "The unreal build system generates Integration.uproject.", + "That generated file is no good. Instead, we generate our own,", + "as part of build-everything.py", + "", + "Don't edit Integration.uproject, instead, edit", + "Integration.uproject.tpl.json, and then run", + "build-everything.py to rebuild.", + "" + ], + "FileVersion": 3, "EngineAssociation": "10.0", "Category": "", diff --git a/Makefile.tpl.txt b/Makefile.tpl.txt new file mode 100644 index 00000000..b2b068a3 --- /dev/null +++ b/Makefile.tpl.txt @@ -0,0 +1,12 @@ +# This makefile just invokes the unreal build system, then the luprex build system. +# +# Do not edit Makefile, instead, edit Makefile.tpl.txt +# + +all: + (cd luprex ; make all) + [UNREALENGINE]/Engine/Build/BatchFiles/[BUILD_BAT] IntegrationEditor [OS] DebugGame [INTEGRATION]/Integration.uproject -waitmutex + +clean: + (cd luprex ; make clean) + [UNREALENGINE]/Engine/Build/BatchFiles/[BUILD_BAT] IntegrationEditor [OS] DebugGame [INTEGRATION]/Integration.uproject -waitmutex -clean diff --git a/build-everything.py b/build-everything.py index 2ba6a358..7b44e5a2 100755 --- a/build-everything.py +++ b/build-everything.py @@ -1,207 +1,179 @@ #!/usr/bin/python3 # -# This python script builds integration from scratch. That includes: +# This python script builds integration from scratch. That includes +# everything we need: # -# - Generating BuildConfiguration.xml in integration repository -# - Generating BuildConfiguration.xml in UnrealEngine repository -# - Hardwiring paths into Source/Integration/lpx-paths.hpp -# - Generating Integration.uproject -# - Generating Integration.code-workspace +# - Generates BuildConfiguration.xml in integration repository +# - Generates BuildConfiguration.xml in UnrealEngine repository +# - Hardwires paths into Source/Integration/lpx-paths.hpp +# - Generates Integration.uproject +# - Generates Integration.code-workspace # - Applies patch to Unreal Engine source. -# - Running Setup.sh in the UnrealEngine repository -# - Building luprex -# - Building ShaderCompileWorker -# - Building integration +# - Runs Setup.sh in the UnrealEngine repository +# - Builds luprex +# - Builds ShaderCompileWorker +# - Builds Unreal Engine and Unreal Editor +# - Builds integration # -# Once this is all done, everything is ready to go. It is now possible to -# start up the IDE and run luprex in the debugger. +# Once this is all done, everything is ready to go. It is now possible +# to start up the IDE and run luprex in the debugger. # -# This script is mainly intended for the *initial* build. -# If you want to edit the code and recompile, it is okay to use -# this script a second time, but it's unnecessarily slow. -# It's much faster to edit and recompile using the IDE. +# If you edit the code, you can run this python script again to +# rebuild. It is always safe to use this script to rebuild after +# editing anything. +# +# However, if you only edited C++ code and Unreal Build.cs files, then +# it may be quicker and more convenient to rebuild from inside the +# VSCODE IDE. Bear in mind that doing so only works if you only edited +# the C++ code and the blueprint code. If you edited anything else, you +# should rerun this python script. # -import sys, os, json, shutil, subprocess +import sys, os, json, shutil, subprocess, re, time from pathlib import Path # -# These things are operating system specific. -# +# Build the config table: a set of global constants that affect +# just about everything. These values become global variables +# in this script, they are also used as variables when expanding +# template files. +# + +CONFIG = {} if sys.platform == "windows": - OS = "Windows" - DLL = "dll" - BAT = "bat" - DOT_EXE = ".exe" - USER = "Unknown" - BUILD_BAT = "Build.bat" + CONFIG["OS"] = "Windows" + CONFIG["DLL"] = "dll" + CONFIG["BAT"] = "bat" + CONFIG["DOT_EXE"] = ".exe" + CONFIG["USER"] = "Unknown" + CONFIG["BUILD_BAT"] = "Build.bat" else: - OS = "Linux" - DLL = "so" - BAT = "sh" - DOT_EXE = "" - USER = os.environ["USER"] - BUILD_BAT = "Linux/Build.sh" + CONFIG["OS"] = "Linux" + CONFIG["DLL"] = "so" + CONFIG["BAT"] = "sh" + CONFIG["DOT_EXE"] = "" + CONFIG["USER"] = os.environ["USER"] + CONFIG["BUILD_BAT"] = "Linux/Build.sh" + +CONFIG["INTEGRATION"] = os.path.dirname(os.path.abspath(sys.argv[0])) +CONFIG["UNREALENGINE"] = os.path.join(os.path.dirname(CONFIG["INTEGRATION"]), "UnrealEngine") + +globals().update(CONFIG) # -# Some handy utility functions +# Sanity check the INTEGRATION and UNREALENGINE paths. # -def readfile(fn): - with open(fn) as f: - return f.read() - -def writefile(fn, str): - with open(fn, "w") as f: - f.write(str) - -def shell(dir, cmd): - print("Running:", cmd) - subprocess.run(cmd, shell=True, check=True, cwd=dir) - -# -# This is the code for a simple json macro preprocessor. -# It is used to write json files containing macros, which can -# then be macroexpanded later in the json file. The json -# file should have a 'macros' section at the top level. -# - -def replace_strings_recursively(template, variables): - if isinstance(template, str): - # Then apply macro-local substitutions - for key, value in variables.items(): - template = template.replace(key, str(value)) - return template - elif isinstance(template, list): - return [replace_strings_recursively(item, variables) for item in template] - elif isinstance(template, dict): - return {k: replace_strings_recursively(v, variables) for k, v in template.items()} - else: - return template - - -def macroexpand_json_recursively(data, macros): - if isinstance(data, dict): - if "macro" in data and "vars" in data: - macro_name = data["macro"] - variables = data["vars"] - base_macro = macros[macro_name] - expanded = replace_strings_recursively(base_macro, variables) - return macroexpand_json_recursively(expanded, macros) - else: - return { - key: macroexpand_json_recursively(value, macros) - for key, value in data.items() - } - elif isinstance(data, list): - return [macroexpand_json_recursively(item, macros) for item in data] - else: - return data - - -def macroexpand_json(source_filename, output_filename, globals): - """ - Load JSON from `source_filename`, perform macro expansion using any - macros defined in the "macros" block, and write the expanded JSON to - `output_filename`. Global variables are passed as a Python dict. - """ - # Load the input template - with open(source_filename, "r") as f: - data = json.load(f) - - # Extract and remove macros - macros = data.pop("macros", {}) - - # Expand macros - expanded = macroexpand_json_recursively(data, macros) - - # Expand global variables - expanded2 = replace_strings_recursively(expanded, globals) - - # Write output - with open(output_filename, "w") as f: - json.dump(expanded2, f, indent=4) - -# -# Find the two repositories and verify them. -# - -INTEGRATION=os.path.dirname(os.path.abspath(sys.argv[0])) -UNREALENGINE=os.path.join(os.path.dirname(INTEGRATION), "UnrealEngine") - if not os.path.isdir(f"{INTEGRATION}/Source/Integration"): sys.exit(f"Integration repository is not valid: {INTEGRATION}") if not os.path.isdir(f"{UNREALENGINE}/Engine/Source/Editor"): sys.exit(f"Integration repository is not valid: {UNREALENGINE}") -JSONGLOBALS= { - "INTEGRATION": INTEGRATION, - "UNREALENGINE": UNREALENGINE, - "USERNAME": USER, -} - # -# Create the Saved/UnrealBuildTool directories. These will hold -# the file BuildConfiguration.xml -# -# Change directory to one of these in order to force ourselves -# to specify all paths explicitly. +# This is the code for a simple json preprocessor that can +# expand "for-each" loops and substitute variables. Because +# a single string is valid json, you can also use this to +# substitute variables in a string or a text file. # -Path(f"{INTEGRATION}/Saved/UnrealBuildTool").mkdir(parents=True, exist_ok=True) -Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool").mkdir(parents=True, exist_ok=True) -os.chdir(f"{INTEGRATION}/Saved/UnrealBuildTool") +JSON_VAR_REGEX = re.compile(r'\[([A-Z0-9_]+)\]') + +def expand_json(data, vars): + if isinstance(data, dict): + if "for-each" in data and "body" in data: + body = data["body"] + foreach = data["for-each"] + return [ expand_json(body, vars | lvars) for lvars in foreach ] + else: + return { key: expand_json(value, vars) for key, value in data.items() } + elif isinstance(data, list): + return [ expand_json(item, vars) for item in data ] + elif isinstance(data, str): + return JSON_VAR_REGEX.sub(lambda m: str(vars.get(m.group(1), m.group(0))), data) + else: + return data # -# Remove previously-generated files. +# Apply the json expander to a file on disk: read the source file, +# apply the expander, write the result back out to disk. # -Path(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml").unlink(missing_ok=True) -Path(f"{INTEGRATION}/Integration.uproject").unlink(missing_ok=True) -Path(f"{INTEGRATION}/Integration.code-workspace").unlink(missing_ok=True) -Path(f"{INTEGRATION}/Makefile").unlink(missing_ok=True) -Path(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp").unlink(missing_ok=True) -Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml").unlink(missing_ok=True) +def expand_json_file(sourcefile, outputfile): + Path(outputfile).unlink(missing_ok=True) + data = json.loads(Path(sourcefile).read_text()) + expanded = expand_json(data, CONFIG) + Path(outputfile).write_text(json.dumps(expanded, indent=4)) + +def expand_text_file(sourcefile, outputfile): + Path(outputfile).unlink(missing_ok=True) + data = Path(sourcefile).read_text() + expanded = expand_json(data, CONFIG) + Path(outputfile).write_text(expanded) + +# +# A simplified interface to subprocess.run +# + +def shell(dir, cmd): + start = time.time() + subprocess.run(cmd, shell=True, check=True, cwd=dir) + elapsed = time.time() - start + if elapsed > 0.2: + print("") + print("TIMING: ", elapsed, " CMD:", cmd) + print("") + +# +# Change directory to an arbitrary subdirectory. Doing this enforces +# the rule that we specify absolute paths for everything. +# + +os.chdir(f"{INTEGRATION}/EnginePatches") # # Write BuildConfiguration.xml # -BUILDCONFIG=readfile(f"{INTEGRATION}/EnginePatches/BuildConfiguration{OS}.xml") -writefile(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml", BUILDCONFIG) -writefile(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml", BUILDCONFIG) +BUILDCONFIG=Path(f"{INTEGRATION}/EnginePatches/BuildConfiguration{OS}.xml").read_text() + +Path(f"{INTEGRATION}/Saved/UnrealBuildTool").mkdir(parents=True, exist_ok=True) +Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool").mkdir(parents=True, exist_ok=True) + +Path(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml").unlink(missing_ok=True) +Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml").unlink(missing_ok=True) + +Path(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml").write_text(BUILDCONFIG) +Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml").write_text(BUILDCONFIG) # # Write lpx-paths.hpp. # -writefile(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp", f""" +Path(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp").unlink(missing_ok=True) +Path(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp").write_text(f""" #define LUPREX_DLL_PATH "{INTEGRATION}/luprex/build/{OS}/luprexlib.{DLL}" #define LUPREX_ROOT_PATH "{INTEGRATION}/luprex" """) # -# Apply patch to the unreal engine source. -# Restore any affected sourcefiles before applying patch. +# Apply patch to the unreal engine source. Check out HEAD version of +# affected sourcefiles before applying patch. # print("Applying patch to Unreal Engine...") -PATCHED_FILES = [] -for line in readfile(f"{INTEGRATION}/EnginePatches/EnginePatch").splitlines(): - if line.startswith("--- a/"): - PATCHED_FILES.append(line[6:]) -if PATCHED_FILES: - shell(UNREALENGINE, "git checkout HEAD -- " + " ".join(PATCHED_FILES)) +PATCH_LINES = Path(f"{INTEGRATION}/EnginePatches/EnginePatch").read_text().splitlines() +PATCHED_FILES = [line[6:] for line in PATCH_LINES if line.startswith("--- a/")] +for file in PATCHED_FILES: + shell(UNREALENGINE, f"git show HEAD:{file} > {file}") shell(UNREALENGINE, f"git apply {INTEGRATION}/EnginePatches/EnginePatch") # # Write Integration.uproject. # -macroexpand_json(f"{INTEGRATION}/Integration.uproject.tpl.json", - f"{INTEGRATION}/Integration.uproject", - JSONGLOBALS) +expand_json_file(f"{INTEGRATION}/Integration.uproject.tpl.json", + f"{INTEGRATION}/Integration.uproject") # # Run Setup.sh in UNREALENGINE @@ -209,27 +181,11 @@ macroexpand_json(f"{INTEGRATION}/Integration.uproject.tpl.json", shell(UNREALENGINE, f"{UNREALENGINE}/Setup.{BAT}") -# -# Create a trivial makefile that calls into the unreal build system. -# - -writefile(f"{INTEGRATION}/Makefile", f""" -# This makefile just invokes the unreal build system, then the luprex build system. - -all: -\t{UNREALENGINE}/Engine/Build/BatchFiles/Linux/Build.sh IntegrationEditor Linux DebugGame {INTEGRATION}/Integration.uproject -waitmutex -\t(cd luprex ; make all) - -clean: -\t{UNREALENGINE}/Engine/Build/BatchFiles/Linux/Build.sh IntegrationEditor Linux DebugGame {INTEGRATION}/Integration.uproject -waitmutex -clean -\t(cd luprex ; make clean) -""") - # # Use UnrealBuildTool to generate Integration.code-workspace.ubt # # We're not going to use it, but we keep it as a reference that you can -# use when editing Integration.workspace-template. +# use when editing Integration.code-workspace.tpl.json. # Path(f"{INTEGRATION}/Integration.code-workspace").unlink(missing_ok=True) @@ -238,19 +194,19 @@ shell(INTEGRATION, f'{UNREALENGINE}/GenerateProjectFiles.{BAT} -projectfiles -pr Path(f"{INTEGRATION}/Integration.code-workspace").rename(f"{INTEGRATION}/Integration.code-workspace.ubt") # -# Build Integration.code-workspace from Integration.workspace-template. +# Build Integration.code-workspace from Integration.code-workspace.tpl.json. # -macroexpand_json(f"{INTEGRATION}/Integration.code-workspace.tpl.json", - f"{INTEGRATION}/Integration.code-workspace", - JSONGLOBALS) +expand_json_file(f"{INTEGRATION}/Integration.code-workspace.tpl.json", + f"{INTEGRATION}/Integration.code-workspace") # -# Do an initial build of Luprex +# Create Makefile from Makefile.tpl.txt # -print("Building luprex...") -shell(f"{INTEGRATION}/luprex", "make") +expand_text_file(f"{INTEGRATION}/Makefile.tpl.txt", + f"{INTEGRATION}/Makefile") + # # Build ShaderCompileWorker @@ -262,9 +218,8 @@ Path(f"Engine/Binaries/{OS}/ShaderCompileWorker{DOT_EXE}").unlink(missing_ok=Tru shutil.copyfile(f"{UNREALENGINE}/Engine/Binaries/{OS}/ShaderCompileWorker-{OS}-Shipping{DOT_EXE}", f"{UNREALENGINE}/Engine/Binaries/{OS}/ShaderCompileWorker{DOT_EXE}") # -# Build Integration +# Run Make # -print("Building integration...") -shell(INTEGRATION, f'{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} IntegrationEditor {OS} DebugGame -project="{INTEGRATION}/Integration.uproject" -waitmutex') +shell(INTEGRATION, 'make')