integration/build.py

#!/usr/bin/python3
#
# This is the one-and-only build script for luprex!
#
# Do not follow any of the build instructions on the Unreal Engine
# websites!  Instead, use this script.  The instructions for this
# script are in README.md
#


import sys, os, json, shutil, subprocess, re, time, tarfile
import itertools, hashlib, zipfile, fnmatch
from pathlib import Path
from types import SimpleNamespace

#
# Suppress an error in some of Unreal's scripts:
#
os.environ["GIT_DIR"] = ".git"

#
# Utility subroutines
#

def shell(dir, cmd):
    "Run a shell command in a directory"
    print("Running:", cmd)
    subprocess.run(cmd, shell=True, check=True, cwd=dir)

def create_tarfile(directory, glob_pattern, outputfile):
    "Create a tarfile from a source directory and a glob pattern"
    directory = Path(directory)
    with tarfile.open(outputfile, mode='w:gz') as tar:
        for path in directory.rglob(glob_pattern):
            if path.is_file():
                tar.add(path, arcname=path.relative_to(directory))

def find_cpp(dir):
    "Find all the C++ and C files in a given directory"
    list1 = list(Path(dir).rglob("*.[ch]pp"))
    list2 = list(Path(dir).rglob("*.[ch]"))
    return sorted([str(x) for x in list1 + list2])

def hash_json(data):
    "Calculate a sha256 hexdigest of any valid json data structure"
    serialized = json.dumps(data, sort_keys=True).encode("utf-8")
    return hashlib.sha256(serialized).hexdigest()

def read_if_exists(fn):
    "Read a file if it exists, otherwise, return null string"
    try: return Path(fn).read_text()
    except: return ""

#
# Fix a chmod bug in the Zipfile module.
#

class JZipFile(zipfile.ZipFile):
    def _extract_member(self, member, targetpath, pwd):
        if not isinstance(member, zipfile.ZipInfo):
            member = self.getinfo(member)
        targetpath = super()._extract_member(member, targetpath, pwd)
        attr = member.external_attr >> 16
        if attr != 0:
            os.chmod(targetpath, attr)
        return targetpath

#
# A JSON preprocessor.
#

JSON_VAR_REGEX = re.compile(r'\[([A-Z0-9_]+)\]')

def expand_json(data, vars):
    """
    A JSON preprocessor that can expand for-each loops and substitute variables.
    The for-loop thing is mainly used by the code-workspace template, to avoid
    extreme code duplication in the launch configurations and build configurations.
    Variable substitution is used to plug hardwired paths into the file, and also
    to implement the for-loop thing.
    """
    if isinstance(data, dict):
        if "for-each" in data and "body" in data:
            body = data["body"]
            foreach = data["for-each"]
            return [ expand_json(body, vars | lvars) for lvars in foreach ]
        if "append1" in data and "append2" in data:
            result = []
            for i in range(1, 1000):
                key = f"append{i}"
                if key not in data: break
                result = result + expand_json(data[key], vars)
            return result
        else:
            return { expand_json(key, vars): expand_json(value, vars) for key, value in data.items() }
    elif isinstance(data, list):
        return [ expand_json(item, vars) for item in data ]
    elif isinstance(data, str):
        full = JSON_VAR_REGEX.fullmatch(data)
        if full: return vars.get(full.group(1), data)
        else: return JSON_VAR_REGEX.sub(lambda m: str(vars.get(m.group(1), m.group(0))), data)
    else:
        return data


def expand_json_file(sourcefile, outputfile, config):
    """
    Apply the json preprocessor to a file on disk
    """
    Path(outputfile).unlink(missing_ok=True)
    data = json.loads(Path(sourcefile).read_text())
    expanded = expand_json(data, vars(config))
    Path(outputfile).write_text(json.dumps(expanded, indent=4))

#
# Extracting a generated module file.
#

def cpp_files_included_by(module):
    """
    Given the name of a Module.XYZ.cpp file generated by unreal
    build tool, scan the module for include directives that include
    another C++ file.  Produce a list of these C++ files.
    """
    result = []
    for line in Path(module).read_text().splitlines():
        if line.startswith("#include "):
            file = line[9:].strip().strip('"')
            if file.endswith(".cpp"):
                result.append(file)
    return result


#
# Determining the build configuration.
#

def get_build_mode_from_command_line():
    """
    Build.py accepts a single argument, which should
    be one of the following modes: all, c++, clean.
    If nothing is specified, the mode is all.
    We understand cpp and cxx as synonyms for c++.
    """
    mode = sys.argv[1].lower() if len(sys.argv) > 1 else 'all'
    if mode in ["cpp", "cxx"]: mode = "c++"
    if not mode in ["all", "c++", "clean", "ccjson", "code-workspace"]:
        sys.exit(f"Invalid build mode: {mode}")
    return mode


def autodetect_system_config():
    """
    This autodetects where integration and unrealengine are installed,
    and it also autodetects your operating system.  Based on these,
    it returns a config object containing a variety of useful
    configuration settings.  We haven't tested the Windows version.
    """
    config = SimpleNamespace()
    # Make sure we're actually inside an integration repository.
    config.INTEGRATION = os.path.dirname(os.path.abspath(sys.argv[0]))
    if not Path(f"{config.INTEGRATION}/Source/Integration").is_dir():
        sys.exit(f"Integration repository is not valid: {config.INTEGRATION}")
    config.UNREALENGINE = config.INTEGRATION + ".UE"
    # Configure other parameters.
    if sys.platform == "windows":
        config.OS = "Windows"
        config.DLL = "dll"
        config.BAT = "bat"
        config.DOT_EXE = ".exe"
        config.USER = "Unknown"
        config.BUILD_BAT = "Build.bat"
        config.DEBUG = "DebugGame"
    else:
        config.OS = "Linux"
        config.DLL = "so"
        config.BAT = "sh"
        config.DOT_EXE = ""
        config.USER = os.environ["USER"]
        config.BUILD_BAT = "Linux/Build.sh"
        config.DEBUG = "DebugGame"
    return config


def store_system_config_in_globals(config):
    """
    Copy all the config data from the config object into global variables.
    """
    global OS,DLL,BAT,DOT_EXE,USER,BUILD_BAT,DEBUG,INTEGRATION,UNREALENGINE
    OS = config.OS
    DLL = config.DLL
    BAT = config.BAT
    DOT_EXE = config.DOT_EXE
    USER = config.USER
    BUILD_BAT = config.BUILD_BAT
    DEBUG = config.DEBUG
    INTEGRATION = config.INTEGRATION
    UNREALENGINE = config.UNREALENGINE

#
# The actual build steps.
#

def unzip_unreal_engine_and_apply_patch():
    """
    Unzip the unreal engine source, then apply a patch.
    """
    zips = list(Path(INTEGRATION).glob("UnrealEngine-*-release.zip"))
    if len(zips) == 0:
        sys.exit("Cannot find UnrealEngine-*-release.zip")
    if len(zips) > 1:
        sys.exit("Found multiple files matching UnrealEngine-*-release.zip")
    print("Unreal version: ", zips[0].stem)
    zipfn = os.path.join(INTEGRATION, zips[0].name)
    extracted = os.path.join(INTEGRATION, zips[0].stem)
    touchfile = os.path.join(UNREALENGINE, zips[0].stem)
    if not Path(touchfile).is_file():
        print("Removing old version of unreal engine source...")
        shutil.rmtree(UNREALENGINE, ignore_errors=True)
        shutil.rmtree(extracted, ignore_errors=True)
        print(f"Unzipping {zipfn}...")
        with JZipFile(zipfn) as zf:
            zf.extractall(INTEGRATION)
        patchfile = f"{INTEGRATION}/EnginePatches/EnginePatch"
        shell(extracted, f"patch -p0 < {patchfile}")
        Path(extracted).rename(UNREALENGINE)
        Path(touchfile).touch()


def generate_buildconfiguration_xml():
    """
    Generates BuildConfiguration.xml.  We actually have two versions of this
    file in git, one for windows, one for linux.  This just copies the appropriate
    version to the relevant target directories.
    """
    dir1 = Path(f"{INTEGRATION}/Saved/UnrealBuildTool")
    dir2 = Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool")
    target1 = Path(f"{dir1}/BuildConfiguration.xml")
    target2 = Path(f"{dir2}/BuildConfiguration.xml")
    source = Path(f"{INTEGRATION}/EnginePatches/BuildConfiguration{OS}.xml")
    template = source.read_text();
    dir1.mkdir(parents=True, exist_ok=True)
    dir2.mkdir(parents=True, exist_ok=True)
    target1.unlink(missing_ok=True)
    target2.unlink(missing_ok=True)
    target1.write_text(template)
    target2.write_text(template)



def generate_integration_uproject():
    """
    Generate integration.uproject
    The uproject file is used by UnrealBuildTool to guide the build process.
    """
    template = f"{INTEGRATION}/Integration.uproject.tpl.json"
    target = f"{INTEGRATION}/Integration.uproject"
    expand_json_file(template, target, CONFIG)


def run_unrealengine_setup_bat_replacement():
    """
    The Setup.bat in UnrealEngine does a lot of unnecessary stuff,
    generates a lot of error messages, and pops up an interactive
    prompt in the middle of a build.  Yuk.  So we've written
    our own script that replaces Setup.bat/Setup.sh.
    """
    touch = Path(f"{UNREALENGINE}/Engine/Build/HaveGitDependencies")
    if not touch.is_file():
        if sys.platform == "windows":
            shell(UNREALENGINE, "Engine/Binaries/DotNET/GitDependencies/win-x64/GitDependencies.exe")
            shell(UNREALENGINE, "Engine/Extras/Redist/en-us/UEPrereqSetup_x64.exe /quiet /norestart")
        else:
            shell(UNREALENGINE, f"Engine/Build/BatchFiles/Linux/GitDependencies.sh --cache={INTEGRATION}/.gitdeps-cache")
            shell(f"{UNREALENGINE}/Engine/Build/BatchFiles/Linux", "./Setup.sh")
        touch.write_text("Downloaded")


def build_unrealbuildtool():
    """
    Build the unreal build tool itself, also including shader compile worker.
    """
    shell(UNREALENGINE, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -buildubt ShaderCompileWorker {OS} Development")


def build_luprex_and_integration():
    """
    This builds our C++ code, and also UnrealEngine's C++ code.
    """
    shell(INTEGRATION, "make -f luprex/Makefile all")
    shell(INTEGRATION, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex IntegrationEditor {OS} {DEBUG} {INTEGRATION}/Integration.uproject")


#
# Make a compile_commands.json file.
#

def build_compile_commands_from_luprex():
    """
    Generate compile commands for luprex by parsing make dry-run output.
    Returns a list of compile_commands.json entries.
    """
    result = subprocess.run(
        ["make", "-n", "-B", "-f", "luprex/Makefile", "all"],
        capture_output=True, text=True, check=True, cwd=INTEGRATION
    )
    entries = []
    for line in result.stdout.splitlines():
        parts = line.split()
        if (parts[0] == "g++") and ("-c" in parts):
            source = os.path.abspath(os.path.join(INTEGRATION, parts[-1]))
            entries.append({ "file": source, "command": line, "directory": INTEGRATION })
    return entries


def build_compile_commands_from_integration():
    """
    Generate compile commands for Unreal/Integration by scanning RSP files
    generated by UnrealBuildTool. Returns a list of compile_commands.json entries.
    """
    # Find clang compiler.
    clangs = list(Path(f"{UNREALENGINE}/Engine/Extras/ThirdPartyNotUE/SDKs").rglob("*-linux-gnu/bin/clang++"))
    if len(clangs) != 1: sys.exit("Couldn't identify correct clang++ compiler in UnrealEngine thirdparty directory")
    clang = str(clangs[0])
    # Build the table of source files and RSP files.
    # First, scan unity Module.*.o.rsp files and expand their #included .cpp files.
    mods1 = Path(f"{INTEGRATION}/Intermediate/Build/{OS}").rglob(f"UnrealEditor/{DEBUG}/**/Module.*.o.rsp")
    mods1p = Path(f"{INTEGRATION}/Plugins").rglob(f"Intermediate/Build/{OS}/x64/UnrealEditor/{DEBUG}/**/Module.*.o.rsp")
    mods2 = Path(f"{UNREALENGINE}/Engine/Intermediate/Build/{OS}").rglob("UnrealEditor/Development/**/Module.*.o.rsp")
    cpp_to_rsp = {}
    for mod in itertools.chain(mods1, mods1p, mods2):
        rsp = str(mod)
        cpp = os.path.abspath(rsp.removesuffix(".o.rsp"))
        cpp_to_rsp[cpp] = rsp
        for subfile in cpp_files_included_by(cpp):
            abs = os.path.abspath(os.path.join(f"{UNREALENGINE}/Engine/Source", subfile))
            cpp_to_rsp[abs] = rsp
    # Also pick up non-unity .cpp files that have their own .o.rsp (e.g. files
    # excluded from unity builds). Only include if the .cpp file actually exists.
    standalone1 = Path(f"{INTEGRATION}/Intermediate/Build/{OS}").rglob(f"UnrealEditor/{DEBUG}/**/*.cpp.o.rsp")
    standalone1p = Path(f"{INTEGRATION}/Plugins").rglob(f"Intermediate/Build/{OS}/x64/UnrealEditor/{DEBUG}/**/*.cpp.o.rsp")
    for rsp_path in itertools.chain(standalone1, standalone1p):
        rsp = str(rsp_path)
        cpp = os.path.abspath(rsp.removesuffix(".o.rsp"))
        if cpp not in cpp_to_rsp and os.path.exists(cpp):
            cpp_to_rsp[cpp] = rsp
    # Generate compile commands.
    entries = []
    ccdir = f"{UNREALENGINE}/Engine/Source"
    for cpp in sorted(cpp_to_rsp.keys()):
        rsp = cpp_to_rsp[cpp]
        args = [clang, "@"+rsp]
        entries.append({ "file": cpp, "arguments": args, "directory": ccdir })
    return entries


def build_compile_commands_json():
    """
    Build .vscode/compile_commands.json by combining compile commands
    from luprex and from the Unreal/Integration build.
    """
    ccjson = build_compile_commands_from_luprex() + build_compile_commands_from_integration()
    Path(f"{INTEGRATION}/.vscode").mkdir(exist_ok=True)
    Path(f"{INTEGRATION}/.vscode/compile_commands.json").write_text(json.dumps(ccjson, indent=2))


def run_generateprojectfiles_in_sandbox():
    """
    Unreal's GenerateProjectFiles does an absolutely terrible
    job of generating project files for vscode.  It's so bad
    that we've decided to just not use it at all.  But we
    still sometimes want to look at the output, to see what
    it would have generated.  We run GenerateProjectFiles
    in a sandbox so it can't modify the real project.  Then,
    we leave the output in the sandbox for inspection.  The
    results don't affect our build system at all.
    """
    sandbox = Path(f"{INTEGRATION}/GPF-output")
    if sandbox.exists():
        shutil.rmtree(sandbox)
    sandbox.mkdir()
    (sandbox / "Integration.uproject").write_bytes(Path(f"{INTEGRATION}/Integration.uproject").read_bytes())
    for name in ["Source", "Config", "Content", "Plugins"]:
        if (Path(INTEGRATION) / name).exists():
            (sandbox / name).symlink_to(f"../{name}")
    shell(str(sandbox), f'{UNREALENGINE}/GenerateProjectFiles.{BAT} -projectfiles -project="{sandbox}/Integration.uproject" -game')
    # Remove the symlinks and uproject copy, leaving only generated files
    for name in ["Source", "Config", "Content", "Plugins", "Integration.uproject"]:
        (sandbox / name).unlink()


def generate_integration_code_workspace():
    """
    We build Integration.code-workspace from a template that we
    wrote ourselves, Integration.code-workspace.tpl.json.
    """
    workspace = f"{INTEGRATION}/Integration.code-workspace"
    template = f"{INTEGRATION}/Integration.code-workspace.tpl.json"
    Path(workspace).unlink(missing_ok=True)
    expand_json_file(template, workspace, CONFIG)


def build_clean():
    """
    This code is underdeveloped.

    For a more aggressive form of cleaning, use 'git clean -xfd',
    which resets your git repository to its pristine state.
    DANGER: this deletes any new source code you've created!
    """
    shell(f"{INTEGRATION}/luprex", "make clean")
    shell(INTEGRATION, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex IntegrationEditor {OS} {DEBUG} {INTEGRATION}/Integration.uproject -clean")
    Path(f"{INTEGRATION}/.vscode/compile_commands.json").unlink(missing_ok = True)

#
# MAIN PROGRAM
#

MODE = get_build_mode_from_command_line()
CONFIG = autodetect_system_config()
store_system_config_in_globals(CONFIG)
os.chdir(f"{INTEGRATION}/EnginePatches")

if MODE == "ccjson":
    build_compile_commands_json()

if MODE == "code-workspace":
    generate_integration_code_workspace()

if MODE == "all":
    unzip_unreal_engine_and_apply_patch()
    generate_buildconfiguration_xml()
    generate_integration_uproject()
    run_unrealengine_setup_bat_replacement()
    run_generateprojectfiles_in_sandbox()
    build_unrealbuildtool()
    generate_integration_code_workspace()

if MODE in ["all", "c++"]:
    build_luprex_and_integration()
    build_compile_commands_json()

if MODE == "clean":
    build_clean()