integration/build-everything.py

#!/usr/bin/python3
#
# This python script builds integration from scratch.  That includes:
#
#   - Generating BuildConfiguration.xml in integration repository
#   - Generating BuildConfiguration.xml in UnrealEngine repository
#   - Hardwiring paths into Source/Integration/lpx-paths.hpp
#   - Generating Integration.uproject
#   - Generating Integration.code-workspace
#   - Applies patch to Unreal Engine source.
#   - Running Setup.sh in the UnrealEngine repository
#   - Building luprex
#   - Building ShaderCompileWorker
#   - Building integration
#
# Once this is all done, everything is ready to go.  It is now possible to
# start up the IDE and run luprex in the debugger.
#
# This script is mainly intended for the *initial* build.
# If you want to edit the code and recompile, it is okay to use
# this script a second time, but it's unnecessarily slow.
# It's much faster to edit and recompile using the IDE.
#

import sys, os, json, shutil, subprocess
from pathlib import Path

#
# These things are operating system specific.
#

if sys.platform == "windows":
    OS = "Windows"
    DLL = "dll"
    BAT = "bat"
    DOT_EXE = ".exe"
    USER = "Unknown"
    BUILD_BAT = "Build.bat"
else:
    OS = "Linux"
    DLL = "so"
    BAT = "sh"
    DOT_EXE = ""
    USER = os.environ["USER"]
    BUILD_BAT = "Linux/Build.sh"

#
# Some handy utility functions
#

def readfile(fn):
	with open(fn) as f:
		return f.read()

def writefile(fn, str):
	with open(fn, "w") as f:
		f.write(str)

def shell(dir, cmd):
    print("Running:", cmd)
    subprocess.run(cmd, shell=True, check=True, cwd=dir)

#
# Find the two repositories and verify them.
#

INTEGRATION=os.path.dirname(os.path.abspath(sys.argv[0]))
UNREALENGINE=os.path.join(os.path.dirname(INTEGRATION), "UnrealEngine")

if not os.path.isdir(f"{INTEGRATION}/Source/Integration"):
    sys.exit(f"Integration repository is not valid: {INTEGRATION}")
if not os.path.isdir(f"{UNREALENGINE}/Engine/Source/Editor"):
    sys.exit(f"Integration repository is not valid: {UNREALENGINE}")

#
# Create the Saved/UnrealBuildTool directories.  These will hold
# the file BuildConfiguration.xml
#
# Change directory to one of these in order to force ourselves
# to specify all paths explicitly.
#

Path(f"{INTEGRATION}/Saved/UnrealBuildTool").mkdir(parents=True, exist_ok=True)
Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool").mkdir(parents=True, exist_ok=True)
os.chdir(f"{INTEGRATION}/Saved/UnrealBuildTool")

#
# Remove previously-generated files.
#

Path(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml").unlink(missing_ok=True)
Path(f"{INTEGRATION}/Integration.uproject").unlink(missing_ok=True)
Path(f"{INTEGRATION}/Integration.code-workspace").unlink(missing_ok=True)
Path(f"{INTEGRATION}/Makefile").unlink(missing_ok=True)
Path(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp").unlink(missing_ok=True)
Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml").unlink(missing_ok=True)

#
# Write BuildConfiguration.xml
#

BUILDCONFIG=readfile(f"{INTEGRATION}/EnginePatches/BuildConfiguration{OS}.xml")
writefile(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml", BUILDCONFIG)
writefile(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml", BUILDCONFIG)

#
# Write lpx-paths.hpp.
#

writefile(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp", f"""
#define LUPREX_DLL_PATH "{INTEGRATION}/luprex/build/{OS}/luprexlib.{DLL}"
#define LUPREX_ROOT_PATH "{INTEGRATION}/luprex"
""")

#
# Apply patch to the unreal engine source.
# Restore any affected sourcefiles before applying patch.
#

print("Applying patch to Unreal Engine...")
for line in readfile(f"{INTEGRATION}/EnginePatches/EnginePatch").splitlines():
    if line.startswith("--- a/"):
        shell(UNREALENGINE, f"git checkout HEAD {line[6:]}")
shell(UNREALENGINE, f"git apply {INTEGRATION}/EnginePatches/EnginePatch")

#
# Write Integration.uproject.
#

UPROJECTTEMPLATE=readfile(f"{INTEGRATION}/EnginePatches/uproject")
UPROJECT=json.loads(UPROJECTTEMPLATE)
with open(f"{INTEGRATION}/Integration.uproject", "w") as rewritten:
	json.dump(UPROJECT, rewritten, indent=4)

#
# Run Setup.sh in UNREALENGINE
#

shell(UNREALENGINE, f"{UNREALENGINE}/Setup.{BAT}")

#
# Use UnrealBuildTool to generate a rough draft of Integration.code-workspace.
# We're not going to use it, but we set it aside as a reference that you can
# study to make changes to this script.
#

Path(f"{INTEGRATION}/Integration.code-workspace").unlink(missing_ok=True)
Path(f"{INTEGRATION}/Integration.code-workspace.old").unlink(missing_ok=True)
shell(INTEGRATION, f'{UNREALENGINE}/GenerateProjectFiles.{BAT} -projectfiles -project="{INTEGRATION}/Integration.uproject" -game')
Path(f"{INTEGRATION}/Integration.code-workspace").rename(f"{INTEGRATION}/Integration.code-workspace.old")

#
# Create a trivial makefile that calls into the unreal build system.
#

writefile(f"{INTEGRATION}/Makefile", f"""
# This makefile just invokes the unreal build system, then the luprex build system.

all:
\t{UNREALENGINE}/Engine/Build/BatchFiles/Linux/Build.sh IntegrationEditor Linux DebugGame {INTEGRATION}/Integration.uproject -waitmutex
\t(cd luprex ; make all)

clean:
\t{UNREALENGINE}/Engine/Build/BatchFiles/Linux/Build.sh IntegrationEditor Linux DebugGame {INTEGRATION}/Integration.uproject -waitmutex -clean
\t(cd luprex ; make clean)
""")

#
# Build our own Integration.code-workspace from scratch.
#

WORKSPACE={}

WORKSPACE["folders"] = []
WORKSPACE["folders"].append({ "name": "Integration", "path": "." })
WORKSPACE["folders"].append({ "name": "UE5", "path": UNREALENGINE })

WORKSPACE["settings"] = {}
WORKSPACE["settings"]["typescript.tsc.autoDetect"] = "off"
WORKSPACE["settings"]["lldb.dereferencePointers"] = False
WORKSPACE["settings"]["npm.autoDetect"] = "off"

WORKSPACE["settings"]["files.watcherExclude"] = {}
WORKSPACE["settings"]["files.watcherExclude"]["/home/jyelon/UnrealEngine/Engine/**"] = True
WORKSPACE["settings"]["files.watcherExclude"]["/home/jyelon/UnrealEngine/Samples/**"] = True
WORKSPACE["settings"]["files.watcherExclude"]["/home/jyelon/UnrealEngine/Templates/**"] = True
WORKSPACE["settings"]["files.associations"] = {
            "*.ipp": "cpp",
            "locale": "cpp",
            "random": "cpp",
            "queue": "cpp",
            "stack": "cpp",
            "__locale": "cpp",
            "functional": "cpp",
            "sstream": "cpp",
            "regex": "cpp",
            "*.inc": "cpp",
            "strstream": "cpp",
            "string_view": "cpp",
            "typeindex": "cpp",
            "typeinfo": "cpp",
            "scoped_allocator": "cpp",
            "array": "cpp",
            "hash_map": "cpp",
            "hash_set": "cpp",
            "bitset": "cpp",
            "slist": "cpp",
            "initializer_list": "cpp",
            "valarray": "cpp",
            "__hash_table": "cpp",
            "__split_buffer": "cpp",
            "__tree": "cpp",
            "deque": "cpp",
            "list": "cpp",
            "map": "cpp",
            "set": "cpp",
            "span": "cpp",
            "string": "cpp",
            "unordered_map": "cpp",
            "unordered_set": "cpp",
            "vector": "cpp",
            "ranges": "cpp",
            "utility": "cpp",
            "ratio": "cpp",
            "system_error": "cpp",
            "tuple": "cpp",
            "type_traits": "cpp",
            "__bit_reference": "cpp",
            "__node_handle": "cpp",
            "atomic": "cpp",
            "__memory": "cpp",
            "limits": "cpp",
            "optional": "cpp",
            "variant": "cpp"
}

WORKSPACE["extensions"] = {}
WORKSPACE["extensions"]["recommendations"] = [
            "vadimcn.vscode-lldb",
            "dfarley1.file-picker",
            "ms-python.python",
            "ms-vscode.cpptools",
            "ms-dotnettools.csharp",
            "ms-vscode.mono-debug"
]


WORKSPACE["tasks"] = {}
WORKSPACE["tasks"]["version"] = "2.0.0"
WORKSPACE["tasks"]["tasks"] = []
WORKSPACE["tasks"]["tasks"].append({
                "label": "Make All",
                "group": { "kind": "build", "isDefault": True },
                "command": "make all",
                "presentation" : { "clear" : True },
                "problemMatcher": "$msCompile",
                "type": "shell",
            })

WORKSPACE["tasks"]["tasks"].append({
                "label": "Make Clean",
                "group": "build",
                "command": "make clean",
                "presentation" : { "clear" : True },
                "problemMatcher": "$msCompile",
                "type": "shell",
            })


WORKSPACE["launch"] = {}
WORKSPACE["launch"]["version"] = "0.2.0"
WORKSPACE["launch"]["configurations"] = []
WORKSPACE["launch"]["configurations"].append({
                "name": "Launch Editor with Luprex",
                "request": "launch",
                "program": f"{UNREALENGINE}/Engine/Binaries/Linux/UnrealEditor-Linux-DebugGame",
                "preLaunchTask": "Make All",
                "args": [
                    f"{INTEGRATION}/Integration.uproject",
                    "-userdir=User/jyelon"
                ],
                "cwd": UNREALENGINE,
                "type": "lldb",
                "initCommands": [
                    f"command script import {UNREALENGINE}/Engine/Extras/LLDBDataFormatters/UEDataFormatters_2ByteChars.py",
                    "settings set target.inline-breakpoint-strategy always",
                    "target stop-hook add --one-liner \"p ::UngrabAllInputImpl()\""
                ]
            })


#
# Write Integration.code-workspace.
#

with open(f"{INTEGRATION}/Integration.code-workspace", "w") as rewritten:
    json.dump(WORKSPACE, rewritten, indent=4)

#
# Do an initial build of Luprex
#

print("Building luprex...")
shell(f"{INTEGRATION}/luprex", "make")

#
# Build ShaderCompileWorker
#

print("Building ShaderCompileWorker...")
shell(UNREALENGINE, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} ShaderCompileWorker {OS} Shipping -waitmutex")
Path(f"Engine/Binaries/{OS}/ShaderCompileWorker{DOT_EXE}").unlink(missing_ok=True)
shutil.copyfile(f"{UNREALENGINE}/Engine/Binaries/{OS}/ShaderCompileWorker-{OS}-Shipping{DOT_EXE}", f"{UNREALENGINE}/Engine/Binaries/{OS}/ShaderCompileWorker{DOT_EXE}")

#
# Build Integration
#

print("Building integration...")
shell(INTEGRATION, f'{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} IntegrationEditor {OS} DebugGame -project="{INTEGRATION}/Integration.uproject" -waitmutex')