integration/build-everything.py

#!/usr/bin/python3
#
# This python script builds integration from scratch.  That includes:
#
#   - Generating BuildConfiguration.xml in integration repository
#   - Generating BuildConfiguration.xml in UnrealEngine repository
#   - Hardwiring paths into Source/Integration/lpx-paths.hpp
#   - Generating Integration.uproject
#   - Generating Integration.code-workspace
#   - Applies patch to Unreal Engine source.
#   - Running Setup.sh in the UnrealEngine repository
#   - Building luprex
#   - Building ShaderCompileWorker
#   - Building integration
#
# Once this is all done, everything is ready to go.  It is now possible to
# start up the IDE and run luprex in the debugger.
#
# This script is mainly intended for the *initial* build.
# If you want to edit the code and recompile, it is okay to use
# this script a second time, but it's unnecessarily slow.
# It's much faster to edit and recompile using the IDE.
#

import sys, os, json, shutil, subprocess
from pathlib import Path

#
# These things are operating system specific.
#

if sys.platform == "windows":
    OS = "Windows"
    DLL = "dll"
    BAT = "bat"
    DOT_EXE = ".exe"
    USER = "Unknown"
    BUILD_BAT = "Build.bat"
else:
    OS = "Linux"
    DLL = "so"
    BAT = "sh"
    DOT_EXE = ""
    USER = os.environ["USER"]
    BUILD_BAT = "Linux/Build.sh"

#
# Some handy utility functions
#

def readfile(fn):
	with open(fn) as f:
		return f.read()

def writefile(fn, str):
	with open(fn, "w") as f:
		f.write(str)

def shell(dir, cmd):
    print("Running:", cmd)
    subprocess.run(cmd, shell=True, check=True, cwd=dir)

#
# Find the two repositories and verify them.
#

INTEGRATION=os.path.dirname(os.path.abspath(sys.argv[0]))
UNREALENGINE=os.path.join(os.path.dirname(INTEGRATION), "UnrealEngine")

if not os.path.isdir(f"{INTEGRATION}/Source/Integration"):
    sys.exit(f"Integration repository is not valid: {INTEGRATION}")
if not os.path.isdir(f"{UNREALENGINE}/Engine/Source/Editor"):
    sys.exit(f"Integration repository is not valid: {UNREALENGINE}")

#
# Create the Saved/UnrealBuildTool directories.  These will hold
# the file BuildConfiguration.xml
#
# Change directory to one of these in order to force ourselves
# to specify all paths explicitly.
#

Path(f"{INTEGRATION}/Saved/UnrealBuildTool").mkdir(parents=True, exist_ok=True)
Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool").mkdir(parents=True, exist_ok=True)
os.chdir(f"{INTEGRATION}/Saved/UnrealBuildTool")

#
# Remove previously-generated files.
#

Path(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml").unlink(missing_ok=True)
Path(f"{INTEGRATION}/Integration.uproject").unlink(missing_ok=True)
Path(f"{INTEGRATION}/Integration.code-workspace").unlink(missing_ok=True)
Path(f"{INTEGRATION}/Makefile").unlink(missing_ok=True)
Path(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp").unlink(missing_ok=True)
Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml").unlink(missing_ok=True)

#
# Write BuildConfiguration.xml
#

BUILDCONFIG=readfile(f"{INTEGRATION}/EnginePatches/BuildConfiguration{OS}.xml")
writefile(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml", BUILDCONFIG)
writefile(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml", BUILDCONFIG)

#
# Write lpx-paths.hpp.
#

writefile(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp", f"""
#define LUPREX_DLL_PATH "{INTEGRATION}/luprex/build/{OS}/luprexlib.{DLL}"
#define LUPREX_ROOT_PATH "{INTEGRATION}/luprex"
""")

#
# Apply patch to the unreal engine source.
# Restore any affected sourcefiles before applying patch.
#

print("Applying patch to Unreal Engine...")
for line in readfile(f"{INTEGRATION}/EnginePatches/EnginePatch").splitlines():
    if line.startswith("--- a/"):
        shell(UNREALENGINE, f"git checkout HEAD {line[6:]}")
shell(UNREALENGINE, f"git apply {INTEGRATION}/EnginePatches/EnginePatch")

#
# Write Integration.uproject.
#

UPROJECTTEMPLATE=readfile(f"{INTEGRATION}/EnginePatches/uproject")
UPROJECT=json.loads(UPROJECTTEMPLATE)
with open(f"{INTEGRATION}/Integration.uproject", "w") as rewritten:
	json.dump(UPROJECT, rewritten, indent=4)

#
# Run Setup.sh in UNREALENGINE
#

shell(UNREALENGINE, f"{UNREALENGINE}/Setup.{BAT}")

#
# Use UnrealBuildTool to generate a rough draft of Integration.code-workspace.
#

shell(INTEGRATION, f'{UNREALENGINE}/GenerateProjectFiles.{BAT} -projectfiles -project="{INTEGRATION}/Integration.uproject" -game')

#
# Load the rough Integration.code-workspace into RAM, then delete the rough draft.
#

with open(f"{INTEGRATION}/Integration.code-workspace") as original:
    WORKSPACE=json.load(original)
Path(f"{INTEGRATION}/Integration.code-workspace").unlink()

#
# Configure the correct build task as the default task.
#

for task in WORKSPACE["tasks"]["tasks"]:
    if task["label"] == f"IntegrationEditor {OS} DebugGame Build":
        task["group"] = { "kind": "build", "isDefault": True }

#
# Delete all build tasks that aren't relevant.
#

def goodtask(task):
    return task["label"].startswith(f"IntegrationEditor {OS} DebugGame")

WORKSPACE["tasks"]["tasks"] = [x for x in WORKSPACE["tasks"]["tasks"] if goodtask(x)]

#
# Add a build task for Luprex
#

LUPREXBUILDTASK={}
WORKSPACE["tasks"]["tasks"].append(LUPREXBUILDTASK)
LUPREXBUILDTASK["label"] = "Build Luprex"
LUPREXBUILDTASK["group"] = "build"
LUPREXBUILDTASK["command"] = "make"
LUPREXBUILDTASK["problemMatcher"] = "$msCompile"
LUPREXBUILDTASK["type"] = "shell"
LUPREXBUILDTASK["options"] = {}
LUPREXBUILDTASK["options"]["cwd"] = f"{INTEGRATION}/luprex"

#
# Convert all launch configurations to lldb.
#

LLDBINIT=[
    f'command script import {UNREALENGINE}/Engine/Extras/LLDBDataFormatters/UEDataFormatters_2ByteChars.py',
    f'settings set target.inline-breakpoint-strategy always',
    f'target stop-hook add --one-liner "p ::UngrabAllInputImpl()"',
]

for config in WORKSPACE["launch"]["configurations"]:
    config["type"] = "lldb"
    config["initCommands"] = LLDBINIT
    config["args"] = [ f"{INTEGRATION}/Integration.uproject", f"-userdir=User/{USER}" ]
    config.pop("visualizerFile", None)
    config.pop("showDisplayString", None)

#
# Delete all but the relevant launch configuration.
#

def goodconf(config):
    return config["name"] == "Launch IntegrationEditor (DebugGame)"

WORKSPACE["launch"]["configurations"] = [x for x in WORKSPACE["launch"]["configurations"] if goodconf(x)]

#
# Add some recommended extensions.
#

EXTENSIONS=set(WORKSPACE["extensions"]["recommendations"])
EXTENSIONS.add("ms-python.python")
EXTENSIONS.add("vadimcn.vscode-lldb")
WORKSPACE["extensions"]["recommendations"] = list(EXTENSIONS)

#
# Tell vscode not to try watching all the UnrealEngine source code for modifications.
# Attempting this overruns a Linux hardwired limit on file watches.
#

WORKSPACE["settings"]["files.watcherExclude"] = {
    f'{UNREALENGINE}/Engine/**' : True,
    f'{UNREALENGINE}/Samples/**' : True,
    f'{UNREALENGINE}/Templates/**' : True
}

#
# Tell the LLDB plugin not to dereference pointers.  This is dumb behavior,
# it dereferences "char *" and shows only the first character.  Not dereferencing
# the pointer shows the whole string.
#
WORKSPACE["settings"]["lldb.dereferencePointers"] = False

#
# Write Integration.code-workspace.
#

with open(f"{INTEGRATION}/Integration.code-workspace", "w") as rewritten:
    json.dump(WORKSPACE, rewritten, indent=4)

#
# Do an initial build of Luprex
#

print("Building luprex...")
shell(f"{INTEGRATION}/luprex", "make")

#
# Build ShaderCompileWorker
#

print("Building ShaderCompileWorker...")
shell(UNREALENGINE, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} ShaderCompileWorker {OS} Shipping -waitmutex")
Path(f"Engine/Binaries/{OS}/ShaderCompileWorker{DOT_EXE}").unlink(missing_ok=True)
shutil.copyfile(f"{UNREALENGINE}/Engine/Binaries/{OS}/ShaderCompileWorker-{OS}-Shipping{DOT_EXE}", f"{UNREALENGINE}/Engine/Binaries/{OS}/ShaderCompileWorker{DOT_EXE}")

#
# Build Integration
#

print("Building integration...")
shell(INTEGRATION, f'{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} IntegrationEditor {OS} DebugGame -project="{INTEGRATION}/Integration.uproject" -waitmutex')