integration/build-everything.py

#!/usr/bin/python3
#
# This python script builds integration from scratch.  That includes:
#
#   - Generating BuildConfiguration.xml in integration repository
#   - Generating BuildConfiguration.xml in UnrealEngine repository
#   - Hardwiring paths into Source/Integration/lpx-paths.hpp
#   - Generating Integration.uproject
#   - Generating Integration.code-workspace
#   - Applies patch to Unreal Engine source.
#   - Running Setup.sh in the UnrealEngine repository
#   - Building luprex
#   - Building ShaderCompileWorker
#   - Building integration
#
# Once this is all done, everything is ready to go.  It is now possible to
# start up the IDE and run luprex in the debugger.
#
# This script is mainly intended for the *initial* build.
# If you want to edit the code and recompile, it is okay to use
# this script a second time, but it's unnecessarily slow.
# It's much faster to edit and recompile using the IDE.
#

import sys, os, json, shutil, subprocess
from pathlib import Path

#
# Some handy utility functions
#

def readfile(fn):
	with open(fn) as f:
		return f.read()

def writefile(fn, str):
	with open(fn, "w") as f:
		f.write(str)

def shell(cmd):
    print("Running:", cmd)
    subprocess.run(cmd, shell=True, check=True)

#
# These are some directory paths that we will need.
#

USER = os.environ["USER"]
INTEGRATION = os.path.dirname(os.path.abspath(sys.argv[0]))
UNREALENGINE = os.environ["HOME"] + "/UnrealEngine"
if not os.path.isdir(f"{INTEGRATION}/Source/Integration"):
	error("Could not figure out the correct path for the INTEGRATION repository.")
if not os.path.isdir(f"{UNREALENGINE}/Engine/Source"):
	error("Could not figure out the correct path for the UNREALENGINE repository.")
os.chdir(INTEGRATION)

#
# Remove previously-generated files.
# 
Path(f"Saved/UnrealBuildTool").mkdir(parents=True, exist_ok=True)
Path(f"Saved/UnrealBuildTool/BuildConfiguration.xml").unlink(missing_ok=True)
Path(f"Integration.uproject").unlink(missing_ok=True)
Path(f"Integration.code-workspace").unlink(missing_ok=True)
Path(f"Makefile").unlink(missing_ok=True)
Path(f"Source/Integration/lpx-paths.hpp").unlink(missing_ok=True)
Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool").mkdir(parents=True, exist_ok=True)
Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml").unlink(missing_ok=True)

#
# Write BuildConfiguration.xml
#

BUILDCONFIG=readfile("EnginePatches/BuildConfigurationLinux.xml")
writefile("Saved/UnrealBuildTool/BuildConfiguration.xml", BUILDCONFIG)
writefile(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml", BUILDCONFIG)

#
# Write lpx-paths.hpp.
#

writefile("Source/Integration/lpx-paths.hpp", f"""
#define LUPREX_DLL_PATH "{INTEGRATION}/luprex/build/linux/luprexlib.so"
#define LUPREX_ROOT_PATH "{INTEGRATION}/luprex"
""")

#
# Apply patch to the unreal engine source.
# Restore any affected sourcefiles before applying patch.
#

os.chdir(UNREALENGINE)
print("Applying patch to Unreal Engine...")
for line in readfile(f"{INTEGRATION}/EnginePatches/EnginePatch").splitlines():
    if line.startswith("--- a/"):
        shell(f"git checkout HEAD {line[6:]}")
shell(f"git apply {INTEGRATION}/EnginePatches/EnginePatch")
os.chdir(INTEGRATION)

#
# Write Integration.uproject.
#

UPROJECTTEMPLATE=readfile("EnginePatches/uproject")
UPROJECT=json.loads(UPROJECTTEMPLATE)
with open("Integration.uproject", "w") as rewritten:
	json.dump(UPROJECT, rewritten, indent=4)

#
# Run Setup.sh in UNREALENGINE
#

os.chdir(UNREALENGINE)
shell("./Setup.sh")
os.chdir(INTEGRATION)

#
# Use UnrealBuildTool to generate a rough draft of Integration.code-workspace.
#

shell(f'{UNREALENGINE}/GenerateProjectFiles.sh -projectfiles -project="{INTEGRATION}/Integration.uproject" -game')

#
# Load the rough Integration.code-workspace into RAM, then delete the rough draft.
#

with open("Integration.code-workspace") as original:
    WORKSPACE=json.load(original)
Path("Integration.code-workspace").unlink()

#
# Configure the correct build task as the default task.
#

for task in WORKSPACE["tasks"]["tasks"]:
    if task["label"] == "IntegrationEditor Linux DebugGame Build":
        task["group"] = { "kind": "build", "isDefault": "true" }

#
# Delete all build tasks that aren't relevant.
#

def goodtask(task):
    return task["label"].startswith("IntegrationEditor Linux DebugGame")

WORKSPACE["tasks"]["tasks"] = [x for x in WORKSPACE["tasks"]["tasks"] if goodtask(x)]

#
# Add a build task for Luprex
#

LUPREXBUILDTASK={}
WORKSPACE["tasks"]["tasks"].append(LUPREXBUILDTASK)
LUPREXBUILDTASK["label"] = "Build Luprex"
LUPREXBUILDTASK["group"] = "build"
LUPREXBUILDTASK["command"] = "make"
LUPREXBUILDTASK["problemMatcher"] = "$msCompile"
LUPREXBUILDTASK["type"] = "shell"
LUPREXBUILDTASK["options"] = {}
LUPREXBUILDTASK["options"]["cwd"] = f"{INTEGRATION}/luprex"

#
# Convert all launch configurations to lldb.
#

LLDBINIT=[
    f'command script import {UNREALENGINE}/Engine/Extras/LLDBDataFormatters/UEDataFormatters_2ByteChars.py',
    f'settings set target.inline-breakpoint-strategy always',
    f'target stop-hook add --one-liner "p ::UngrabAllInputImpl()"',
]

for config in WORKSPACE["launch"]["configurations"]:
    config["type"] = "lldb"
    config["initCommands"] = LLDBINIT
    config["args"] = [ f"{INTEGRATION}/Integration.uproject", f"-userdir=User/{USER}" ]
    config.pop("visualizerFile", None)
    config.pop("showDisplayString", None)

#
# Delete all but the relevant launch configuration.
#

def goodconf(config):
    return config["name"] == "Launch IntegrationEditor (DebugGame)"

WORKSPACE["launch"]["configurations"] = [x for x in WORKSPACE["launch"]["configurations"] if goodconf(x)]

#
# Add some recommended extensions.
#

EXTENSIONS=set(WORKSPACE["extensions"]["recommendations"])
EXTENSIONS.add("ms-python.python")
EXTENSIONS.add("vadimcn.vscode-lldb")
WORKSPACE["extensions"]["recommendations"] = list(EXTENSIONS)

#
# Tell vscode not to try watching all the UnrealEngine source code for modifications.
# Attempting this overruns a Linux hardwired limit on file watches.
#

WORKSPACE["settings"]["files.watcherExclude"] = {
    f'{UNREALENGINE}/Engine/**' : True,
    f'{UNREALENGINE}/Samples/**' : True,
    f'{UNREALENGINE}/Templates/**' : True
}

#
# Write Integration.code-workspace.
#

with open("Integration.code-workspace", "w") as rewritten:
    json.dump(WORKSPACE, rewritten, indent=4)

#
# Do an initial build of Luprex
#

os.chdir(f"{INTEGRATION}/luprex")
print("Building luprex...")
shell("make")
os.chdir(INTEGRATION)

#
# Build ShaderCompileWorker
#

os.chdir(UNREALENGINE)
print("Building ShaderCompileWorker...")
shell("Engine/Build/BatchFiles/Linux/Build.sh ShaderCompileWorker Linux Shipping -waitmutex")
Path("Engine/Binaries/Linux/ShaderCompileWorker").unlink(missing_ok=True)
shutil.copyfile("Engine/Binaries/Linux/ShaderCompileWorker-Linux-Shipping", "Engine/Binaries/Linux/ShaderCompileWorker")
os.chdir(INTEGRATION)

#
# Build Integration
#

print("Building integration...")
os.system(f'dotnet {UNREALENGINE}/Engine/Binaries/DotNET/UnrealBuildTool/UnrealBuildTool.dll IntegrationEditor Linux DebugGame -project="{INTEGRATION}/Integration.uproject" -waitmutex')