integration/build.py

#!/usr/bin/python3
#
# The one-and-only build script for luprex.
# Do not follow any of the build instructions on the Unreal Engine
# websites!  Instead, use these instructions.
#
#
# HOW TO BUILD THE FIRST TIME
#
# First, install the following software:
#
#   apt-get install git-lfs
#   apt-get install code
#   apt-get install dotnet6
#   apt-get install clangd-15 or better.
#
# Then, git clone the UnrealEngine and integration repositories:
#
#   cd $HOME
#   git clone https://github.com/EpicGames/UnrealEngine.git
#   git clone https://www.gnaut.com/team/integration.git
#
# It is important that these two repositories be located
# at $HOME/UnrealEngine and $HOME/integration.
#
# Of course, you will have to jump through a bunch of hoops to
# get access to these repositories.  See the instructions on the
# unreal engine website.
#
# After cloning the two repositories, change directory into
# the "integration" repository and run "build.py all".
# This will build both unreal engine and integration.  It will
# also build project files for vscode, an intellisense database,
# and several other things.  Once it is done, everything is built.
# You can start up the debugger.
# 
# 
# HOW TO REBUILD WHEN YOU'VE EDITED SOMETHING
#
# You can also use "build.py all" to rebuild.  This is the preferred
# way to rebuild when you aren't sure what's been edited.  However,
# this takes almost 30 seconds even when there's nothing new to build,
# so it can be a little slow.
#
# If you're sure that the only thing you've edited recently are
# C++ files in the integration repository, then you can get away with
# doing a lightweight build: "build.py c++".  This only works if you've
# already completed a successful full build, and the only thing you've
# done since then is edit C++ files in integration.  If you've edited
# anything else, you need to use "build.py all" to rebuild.
#
# Editing Lua or Blueprint code doesn't require any kind of rebuild.
#
#
# USING VISUAL STUDIO CODE
# 
# To start up vscode, change directory to the integration repository,
# and run "code Integration.code-workspace".  From inside vscode, you can
# use Terminal/Run_Build_Task to run "build.py all" or "build.py c++".
# You can also select Run/Start_Debugging to launch the game.
#
# The first time you launch vscode, you will see a popup about
# recommended extensions.  These are actually recommended by this
# script, build.py, so they're actually good recommendations.  Install
# them: you will only have to do this once.
#


import sys, os, json, shutil, subprocess, re, time, tarfile, itertools, hashlib
from pathlib import Path
from types import SimpleNamespace


#
# Utility subroutines
#

def shell(dir, cmd):
    "Run a shell command in a directory"
    subprocess.run(cmd, shell=True, check=True, cwd=dir)

def create_tarfile(directory, glob_pattern, outputfile):
    "Create a tarfile from a source directory and a glob pattern"
    directory = Path(directory)
    with tarfile.open(outputfile, mode='w:gz') as tar:
        for path in directory.rglob(glob_pattern):
            if path.is_file():
                tar.add(path, arcname=path.relative_to(directory))

def find_cpp(dir):
    "Find all the C++ and C files in a given directory"
    list1 = list(Path(dir).rglob("*.[ch]pp"))
    list2 = list(Path(dir).rglob("*.[ch]"))
    return sorted([str(x) for x in list1 + list2])

def hash_json(data):
    "Calculate a sha256 hexdigest of any valid json data structure"
    serialized = json.dumps(data, sort_keys=True).encode("utf-8")
    return hashlib.sha256(serialized).hexdigest()

def read_if_exists(fn):
    "Read a file if it exists, otherwise, return null string"
    try: return Path(fn).read_text()
    except: return ""

#
# A JSON preprocessor.
#

JSON_VAR_REGEX = re.compile(r'\[([A-Z0-9_]+)\]')

def expand_json(data, vars):
    """
    A JSON preprocessor that can expand for-each loops and substitute variables.
    The for-loop thing is mainly used by the code-workspace template, to avoid
    extreme code duplication in the launch configurations and build configurations.
    Variable substitution is used to plug hardwired paths into the file, and also
    to implement the for-loop thing.
    """
    if isinstance(data, dict):
        if "for-each" in data and "body" in data:
            body = data["body"]
            foreach = data["for-each"]
            return [ expand_json(body, vars | lvars) for lvars in foreach ]
        else:
            return { key: expand_json(value, vars) for key, value in data.items() }
    elif isinstance(data, list):
        return [ expand_json(item, vars) for item in data ]
    elif isinstance(data, str):
        full = JSON_VAR_REGEX.fullmatch(data)
        if full: return vars.get(full.group(1), data)
        else: return JSON_VAR_REGEX.sub(lambda m: str(vars.get(m.group(1), m.group(0))), data)
    else:
        return data


def expand_json_file(sourcefile, outputfile, config):
    """
    Apply the json preprocessor to a file on disk
    """
    Path(outputfile).unlink(missing_ok=True)
    data = json.loads(Path(sourcefile).read_text())
    expanded = expand_json(data, vars(config))
    Path(outputfile).write_text(json.dumps(expanded, indent=4))

#
# Determining the build configuration.
#

def get_build_mode_from_command_line():
    """
    Build.py accepts a single argument, which should
    be one of the following modes: all, c++, clean.
    If nothing is specified, the mode is all.
    We understand cpp and cxx as synonyms for c++.
    """
    mode = sys.argv[1].lower() if len(sys.argv) > 1 else 'all'
    if mode in ["cpp", "cxx"]: mode = "c++"
    if not mode in ["all", "c++", "clean", "experiment"]:
        sys.exit(f"Invalid build mode: {mode}")
    return mode


def autodetect_system_config():
    """
    This autodetects where integration and unrealengine are installed,
    and it also autodetects your operating system.  Based on these,
    it returns a config object containing a variety of useful
    configuration settings.  We haven't tested the Windows version.
    """
    config = SimpleNamespace()
    if sys.platform == "windows":
        config.OS = "Windows"
        config.DLL = "dll"
        config.BAT = "bat"
        config.DOT_EXE = ".exe"
        config.USER = "Unknown"
        config.BUILD_BAT = "Build.bat"
    else:
        config.OS = "Linux"
        config.DLL = "so"
        config.BAT = "sh"
        config.DOT_EXE = ""
        config.USER = os.environ["USER"]
        config.BUILD_BAT = "Linux/Build.sh"
    config.INTEGRATION = os.path.dirname(os.path.abspath(sys.argv[0]))
    config.UNREALENGINE = os.path.join(os.path.dirname(config.INTEGRATION), "UnrealEngine")
    test1 = Path(f"{config.INTEGRATION}/Source/Integration")
    test2 = Path(f"{config.UNREALENGINE}/Engine/Source/Editor")
    if not test1.is_dir(): sys.exit(f"Integration repository is not valid: {config.INTEGRATION}")
    if not test2.is_dir(): sys.exit(f"UnrealEngine repository is not valid: {config.UNREALENGINE}")
    return config


def store_system_config_in_globals(config):
    """
    Copy all the config data from the config object into global variables.
    """
    global OS,DLL,BAT,DOT_EXE,USER,BUILD_BAT,INTEGRATION,UNREALENGINE
    OS = config.OS
    DLL = config.DLL
    BAT = config.BAT
    DOT_EXE = config.DOT_EXE
    USER = config.USER
    BUILD_BAT = config.BUILD_BAT
    INTEGRATION = config.INTEGRATION
    UNREALENGINE = config.UNREALENGINE

#
# The actual build steps.
#

def checkout_correct_unreal_engine_branch_and_apply_patch():
    """
    Check out the correct branch, then apply a patch to the UnrealEngine 
    source code.  Patch application consists of two steps: first, checkout clean,
    unpatched versions of the files from git.  Then, apply the patch to the clean code.
    Note: don't git-commit the patch in UnrealEngine.  Instead,
    just let this script reapply this patch as necessary.
    """
    patch = Path(f"{INTEGRATION}/EnginePatches/EnginePatch")
    patch_lines = patch.read_text().splitlines()
    patched_files = [line[6:] for line in patch_lines if line.startswith("--- a/")]
    for file in patched_files:
        shell(UNREALENGINE, f"git show HEAD:{file} > {file}")
    shell(UNREALENGINE, f"git checkout 5.3.1-release")
    shell(UNREALENGINE, f"git apply {INTEGRATION}/EnginePatches/EnginePatch")


def generate_buildconfiguration_xml():
    """
    Generates BuildConfiguration.xml.  We actually have two versions of this
    file in git, one for windows, one for linux.  This just copies the appropriate
    version to the relevant target directories.
    """
    dir1 = Path(f"{INTEGRATION}/Saved/UnrealBuildTool")
    dir2 = Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool")
    target1 = Path(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml")
    target2 = Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml")
    source = Path(f"{INTEGRATION}/EnginePatches/BuildConfiguration{OS}.xml")
    template = source.read_text();
    dir1.mkdir(parents=True, exist_ok=True)
    dir2.mkdir(parents=True, exist_ok=True)
    target1.unlink(missing_ok=True)
    target2.unlink(missing_ok=True)
    target1.write_text(template)
    target2.write_text(template)


def generate_lpx_paths():
    """
    Unreal needs to be able to find the Luprex DLL, and it also needs to find the
    Lua source code.  For now, we just compile some hardwired paths into the
    binary.  Someday we'll do something more sophisticated.
    """
    target = Path(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp")
    line1 = f'#define LUPREX_DLL_PATH "{INTEGRATION}/luprex/build/{OS}/luprexlib.{DLL}"'
    line2 = f'#define LUPREX_ROOT_PATH "{INTEGRATION}/luprex"'
    code = line1 + "\n" + line2 + "\n"
    target.unlink(missing_ok=True)
    target.write_text(code)


def generate_integration_uproject():
    """
    Generate integration.uproject
    The uproject file is used by UnrealBuildTool to guide the build process.
    """
    template = f"{INTEGRATION}/Integration.uproject.tpl.json"
    target = f"{INTEGRATION}/Integration.uproject"
    expand_json_file(template, target, CONFIG)


def run_unrealengine_setup_bat():
    """
    Run Setup.bat in UnrealEngine.
    This script downloads assets that aren't stored in git.
    """
    shell(UNREALENGINE, f"{UNREALENGINE}/Setup.{BAT}")


def build_unrealbuildtool():
    """
    Build the unreal build tool itself, also including shader compile worker.
    """
    shell(UNREALENGINE, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -buildubt ShaderCompileWorker {OS} Development")


def build_luprex_and_integration():
    """
    This builds our C++ code, and also UnrealEngine's C++ code.
    """
    shell(f"{INTEGRATION}/luprex", "make all")
    shell(INTEGRATION, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex IntegrationEditor {OS} DebugGame {INTEGRATION}/Integration.uproject")


def build_intellisense_database_for_clangd(force):
    """
    This builds compile_commands.json, which tells the intellisense system
    based on clangd how to compile each source file.  We automatically rebuild
    if a C++ file has been added or removed.  That probably isn't sufficient:
    we should also rebuild if a Build.cs file has been edited.  Coming Soon.
    Until then, if your intellisense starts acting badly, do a 'build.py all',
    which sets the 'force' argument above to true.

    Rebuilding the intellisense database touches rsp files, which
    unfortunately, causes the entire C++ build to be restarted the next
    time you run 'build'.  This is terrible behavior from Unreal's build
    system.  We have a hacky workaround: we save the RSP files before
    running the intellisense build, then we restore them afterward.  
    """
    Path(f"{INTEGRATION}/.vscode").mkdir(exist_ok = True)
    Path(f"{UNREALENGINE}/.vscode").mkdir(exist_ok = True)
    hash_file = Path(f"{INTEGRATION}/.vscode/cpp_hash")
    new_hash = hash_json(find_cpp(f"{INTEGRATION}/Source"))
    old_hash = read_if_exists(hash_file).strip()
    if (new_hash != old_hash) or force:
        hash_file.unlink(missing_ok=True)
        create_tarfile(f"{INTEGRATION}/Intermediate", "*.rsp", f"{INTEGRATION}/rsp_files.tgz")
        create_tarfile(f"{UNREALENGINE}/Engine", "*.rsp", f"{UNREALENGINE}/rsp_files.tgz")
        try:
            shell(INTEGRATION, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex IntegrationEditor {OS} DebugGame {INTEGRATION}/Integration.uproject -mode=GenerateClangDatabase -OutputDir={UNREALENGINE}/.vscode")
            shell(UNREALENGINE, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex UnrealEditor {OS} DebugGame -mode=GenerateClangDatabase -OutputDir={UNREALENGINE}/.vscode")
            shell(INTEGRATION, f"cat {UNREALENGINE}/.vscode/compile_commands.json >> {INTEGRATION}/.vscode/compile_commands.json")
        except Exception as e:
            error = e
        else:
            error = None
        finally:
            tarfile.open(f"{INTEGRATION}/rsp_files.tgz").extractall(path=f"{INTEGRATION}/Intermediate")
            tarfile.open(f"{UNREALENGINE}/rsp_files.tgz").extractall(path=f"{UNREALENGINE}/Engine")
            Path(f"{INTEGRATION}/rsp_files.tgz").unlink()
            Path(f"{UNREALENGINE}/rsp_files.tgz").unlink()
        if error: raise error
        hash_file.write_text(new_hash)


def generate_integration_code_workspace():
    """
    We build Integration.code-workspace from a template that we
    wrote ourselves, Integration.code-workspace.tpl.json.
    We use UnrealBuildTool to generate Integration.code-workspace.ubt,
    but we don't use it: we just keep it as a reference that you can
    refer to when editing the template.
    """
    workspace = f"{INTEGRATION}/Integration.code-workspace"
    workspace_ubt = f"{INTEGRATION}/Integration.code-workspace.ubt"
    template = f"{INTEGRATION}/Integration.code-workspace.tpl.json"
    Path(workspace).unlink(missing_ok=True)
    Path(workspace_ubt).unlink(missing_ok=True)
    shell(INTEGRATION, f'{UNREALENGINE}/GenerateProjectFiles.{BAT} -projectfiles -project="{INTEGRATION}/Integration.uproject" -game')
    Path(workspace).rename(workspace_ubt)
    expand_json_file(template, workspace, CONFIG)


def build_clean():
    """
    This code is underdeveloped.
    """
    shell(f"{INTEGRATION}/luprex", "make clean")
    shell(INTEGRATION, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex IntegrationEditor {OS} DebugGame {INTEGRATION}/Integration.uproject -clean")
    Path(f"{INTEGRATION}/.vscode/compile_commands.json").unlink(missing_ok = True)

#
# MAIN PROGRAM
#

MODE = get_build_mode_from_command_line()
CONFIG = autodetect_system_config()
store_system_config_in_globals(CONFIG)
os.chdir(f"{INTEGRATION}/EnginePatches")

if MODE == "all":
    checkout_correct_unreal_engine_branch_and_apply_patch()
    generate_buildconfiguration_xml()
    generate_lpx_paths()
    generate_integration_uproject()
    run_unrealengine_setup_bat()
    build_unrealbuildtool()
    generate_integration_code_workspace()

if MODE in ["all", "c++"]:
    build_luprex_and_integration()
    build_intellisense_database_for_clangd(MODE == "all")

if MODE == "clean":
    build_clean()