integration/build.py

#!/usr/bin/python3
#
# This is the one-and-only build script for luprex!
#
# Do not follow any of the build instructions on the Unreal Engine
# websites!  Instead, use this script.  The instructions for this
# script are in README.md
#


import sys, os, json, shutil, subprocess, re, time, tarfile
import itertools, hashlib, zipfile, fnmatch
from pathlib import Path
from types import SimpleNamespace


#
# Utility subroutines
#

def shell(dir, cmd):
    "Run a shell command in a directory"
    subprocess.run(cmd, shell=True, check=True, cwd=dir)

def create_tarfile(directory, glob_pattern, outputfile):
    "Create a tarfile from a source directory and a glob pattern"
    directory = Path(directory)
    with tarfile.open(outputfile, mode='w:gz') as tar:
        for path in directory.rglob(glob_pattern):
            if path.is_file():
                tar.add(path, arcname=path.relative_to(directory))

def find_cpp(dir):
    "Find all the C++ and C files in a given directory"
    list1 = list(Path(dir).rglob("*.[ch]pp"))
    list2 = list(Path(dir).rglob("*.[ch]"))
    return sorted([str(x) for x in list1 + list2])

def hash_json(data):
    "Calculate a sha256 hexdigest of any valid json data structure"
    serialized = json.dumps(data, sort_keys=True).encode("utf-8")
    return hashlib.sha256(serialized).hexdigest()

def read_if_exists(fn):
    "Read a file if it exists, otherwise, return null string"
    try: return Path(fn).read_text()
    except: return ""

#
# A JSON preprocessor.
#

JSON_VAR_REGEX = re.compile(r'\[([A-Z0-9_]+)\]')

def expand_json(data, vars):
    """
    A JSON preprocessor that can expand for-each loops and substitute variables.
    The for-loop thing is mainly used by the code-workspace template, to avoid
    extreme code duplication in the launch configurations and build configurations.
    Variable substitution is used to plug hardwired paths into the file, and also
    to implement the for-loop thing.
    """
    if isinstance(data, dict):
        if "for-each" in data and "body" in data:
            body = data["body"]
            foreach = data["for-each"]
            return [ expand_json(body, vars | lvars) for lvars in foreach ]
        else:
            return { key: expand_json(value, vars) for key, value in data.items() }
    elif isinstance(data, list):
        return [ expand_json(item, vars) for item in data ]
    elif isinstance(data, str):
        full = JSON_VAR_REGEX.fullmatch(data)
        if full: return vars.get(full.group(1), data)
        else: return JSON_VAR_REGEX.sub(lambda m: str(vars.get(m.group(1), m.group(0))), data)
    else:
        return data


def expand_json_file(sourcefile, outputfile, config):
    """
    Apply the json preprocessor to a file on disk
    """
    Path(outputfile).unlink(missing_ok=True)
    data = json.loads(Path(sourcefile).read_text())
    expanded = expand_json(data, vars(config))
    Path(outputfile).write_text(json.dumps(expanded, indent=4))

#
# Applying patches to files with DOS line endings.
#
# We want to apply a patch to the UnrealEngine source, and some but not all
# of the files we want to patch have DOS line endings.  Unfortunately, the
# unix patch utility can't handle a codebase with mixed line endings.
# To make it work, we convert all files to UNIX line endings, apply the
# patch, and then convert the appropriate files back.
#

def patched_files(patchfn):
    patch_lines = Path(patchfn).read_text().splitlines()
    file_names = [line[4:].split()[0] for line in patch_lines if line.startswith('--- ')]
    return [ '/'.join(file.split('/')[1:]) for file in file_names ]

def dos2unix(fn):
    before = Path(fn).read_text()
    after = before.replace("\r\n", "\n")
    if before != after:
        Path(fn).write_text(after)
        return True
    return False

def unix2dos(fn):
    before = Path(fn).read_text()
    after = before.replace("\n", "\r\n")
    if before != after:
        Path(fn).write_text(after)
        return True
    return False

#
# Given a patch file, generate a list of the files being modified.
#

#
# Determining the build configuration.
#

def get_build_mode_from_command_line():
    """
    Build.py accepts a single argument, which should
    be one of the following modes: all, c++, clean.
    If nothing is specified, the mode is all.
    We understand cpp and cxx as synonyms for c++.
    """
    mode = sys.argv[1].lower() if len(sys.argv) > 1 else 'all'
    if mode in ["cpp", "cxx"]: mode = "c++"
    if not mode in ["all", "c++", "clean", "experiment"]:
        sys.exit(f"Invalid build mode: {mode}")
    return mode


def autodetect_system_config():
    """
    This autodetects where integration and unrealengine are installed,
    and it also autodetects your operating system.  Based on these,
    it returns a config object containing a variety of useful
    configuration settings.  We haven't tested the Windows version.
    """
    config = SimpleNamespace()
    # Make sure we're actually inside an integration repository.
    config.INTEGRATION = os.path.dirname(os.path.abspath(sys.argv[0]))
    if not Path(f"{config.INTEGRATION}/Source/Integration").is_dir():
        sys.exit(f"Integration repository is not valid: {config.INTEGRATION}")
    config.UNREALENGINE = config.INTEGRATION + ".UE"
    # Configure other parameters.
    if sys.platform == "windows":
        config.OS = "Windows"
        config.DLL = "dll"
        config.BAT = "bat"
        config.DOT_EXE = ".exe"
        config.USER = "Unknown"
        config.BUILD_BAT = "Build.bat"
    else:
        config.OS = "Linux"
        config.DLL = "so"
        config.BAT = "sh"
        config.DOT_EXE = ""
        config.USER = os.environ["USER"]
        config.BUILD_BAT = "Linux/Build.sh"
    return config


def store_system_config_in_globals(config):
    """
    Copy all the config data from the config object into global variables.
    """
    global OS,DLL,BAT,DOT_EXE,USER,BUILD_BAT,INTEGRATION,UNREALENGINE
    OS = config.OS
    DLL = config.DLL
    BAT = config.BAT
    DOT_EXE = config.DOT_EXE
    USER = config.USER
    BUILD_BAT = config.BUILD_BAT
    INTEGRATION = config.INTEGRATION
    UNREALENGINE = config.UNREALENGINE

#
# The actual build steps.
#

def unzip_unreal_engine_and_apply_patch():
    """
    Unzip the unreal engine source, then apply a patch.
    """
    if not Path(UNREALENGINE).is_dir():
        zipfn = f"{INTEGRATION}/UnrealEngine.zip";
        with zipfile.ZipFile(zipfn, 'r') as z:
            version = z.namelist()[0].split('/')[0]
        if not fnmatch.fnmatch(version, 'UnrealEngine-*-release'):
            sys.exit("UnrealEngine.zip does not contain UnrealEngine-*-release")
        unrealversion = os.path.join(INTEGRATION, version)
        shutil.rmtree(UNREALENGINE, ignore_errors=True)
        shutil.rmtree(unrealversion, ignore_errors=True)
        shell(INTEGRATION, "unzip UnrealEngine.zip")
        patchfile = f"{INTEGRATION}/EnginePatches/EnginePatch"
        dos2unix(patchfile)
        rel_files = patched_files(patchfile)
        abs_files = [ os.path.join(unrealversion, file) for file in rel_files ]
        dos_files = [ file for file in abs_files if dos2unix(file)]
        shell(unrealversion, f"patch -p1 < {patchfile}")
        for file in dos_files: unix2dos(file)
        Path(unrealversion).rename(UNREALENGINE)

def generate_buildconfiguration_xml():
    """
    Generates BuildConfiguration.xml.  We actually have two versions of this
    file in git, one for windows, one for linux.  This just copies the appropriate
    version to the relevant target directories.
    """
    dir1 = Path(f"{INTEGRATION}/Saved/UnrealBuildTool")
    dir2 = Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool")
    target1 = Path(f"{INTEGRATION}/Saved/UnrealBuildTool/BuildConfiguration.xml")
    target2 = Path(f"{UNREALENGINE}/Engine/Saved/UnrealBuildTool/BuildConfiguration.xml")
    source = Path(f"{INTEGRATION}/EnginePatches/BuildConfiguration{OS}.xml")
    template = source.read_text();
    dir1.mkdir(parents=True, exist_ok=True)
    dir2.mkdir(parents=True, exist_ok=True)
    target1.unlink(missing_ok=True)
    target2.unlink(missing_ok=True)
    target1.write_text(template)
    target2.write_text(template)


def generate_lpx_paths():
    """
    Unreal needs to be able to find the Luprex DLL, and it also needs to find the
    Lua source code.  For now, we just compile some hardwired paths into the
    binary.  Someday we'll do something more sophisticated.
    """
    target = Path(f"{INTEGRATION}/Source/Integration/lpx-paths.hpp")
    line1 = f'#define LUPREX_DLL_PATH "{INTEGRATION}/luprex/build/{OS}/luprexlib.{DLL}"'
    line2 = f'#define LUPREX_ROOT_PATH "{INTEGRATION}/luprex"'
    code = line1 + "\n" + line2 + "\n"
    target.unlink(missing_ok=True)
    target.write_text(code)


def generate_integration_uproject():
    """
    Generate integration.uproject
    The uproject file is used by UnrealBuildTool to guide the build process.
    """
    template = f"{INTEGRATION}/Integration.uproject.tpl.json"
    target = f"{INTEGRATION}/Integration.uproject"
    expand_json_file(template, target, CONFIG)


def run_unrealengine_setup_bat():
    """
    Run Setup.bat in UnrealEngine.
    This script downloads assets that aren't stored in git.
    """
    shell(UNREALENGINE, f"{UNREALENGINE}/Setup.{BAT}")


def build_unrealbuildtool():
    """
    Build the unreal build tool itself, also including shader compile worker.
    """
    shell(UNREALENGINE, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -buildubt ShaderCompileWorker {OS} Development")


def build_luprex_and_integration():
    """
    This builds our C++ code, and also UnrealEngine's C++ code.
    """
    shell(f"{INTEGRATION}/luprex", "make all")
    shell(INTEGRATION, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex IntegrationEditor {OS} DebugGame {INTEGRATION}/Integration.uproject")


#
# Make a compile_commands.json file.
#

def build_intellisense_database_for_clangd():
    """
    This builds compile_commands.json, which tells the intellisense system
    based on clangd how to compile each source file.
    This also installs a .clangd file in the UnrealEngine directory.
    """
    mods1 = Path(f"{INTEGRATION}/Intermediate/Build/{OS}").rglob("UnrealEditor/DebugGame/**/Module.*.o.rsp")
    mods2 = Path(f"{UNREALENGINE}/Engine/Intermediate/Build/{OS}").rglob("UnrealEditor/Development/**/Module.*.o.rsp")
    mods = list(mods1) + list(mods2)
    clangs = list(Path(f"{UNREALENGINE}/Engine/Extras/ThirdPartyNotUE/SDKs").rglob("*-linux-gnu/bin/clang++"))
    if len(clangs) != 1: sys.exit("Couldn't identify correct clang++ compiler in UnrealEngine thirdparty directory")
    clang = str(clangs[0])
    ccjson = json.loads(Path(f"{INTEGRATION}/luprex/build/{OS}/compile_commands.json").read_text())
    ccdir = f"{UNREALENGINE}/Engine/Source";
    for mod in mods:
        rsp = str(mod)
        cpp = rsp.removesuffix(".o.rsp")
        args = [clang, "@"+rsp]
        ccjson.append({ "file" : cpp, "arguments":args, "directory":ccdir })
    Path(f"{INTEGRATION}/.vscode/compile_commands.json").write_text(json.dumps(ccjson, indent=2))


def generate_integration_code_workspace():
    """
    We build Integration.code-workspace from a template that we
    wrote ourselves, Integration.code-workspace.tpl.json.
    We use UnrealBuildTool to generate Integration.code-workspace.ubt,
    but we don't use it: we just keep it as a reference that you can
    refer to when editing the template.
    """
    workspace = f"{INTEGRATION}/Integration.code-workspace"
    workspace_ubt = f"{INTEGRATION}/Integration.code-workspace.ubt"
    template = f"{INTEGRATION}/Integration.code-workspace.tpl.json"
    Path(workspace).unlink(missing_ok=True)
    Path(workspace_ubt).unlink(missing_ok=True)
    shell(INTEGRATION, f'{UNREALENGINE}/GenerateProjectFiles.{BAT} -projectfiles -project="{INTEGRATION}/Integration.uproject" -game')
    Path(workspace).rename(workspace_ubt)
    expand_json_file(template, workspace, CONFIG)


def build_clean():
    """
    This code is underdeveloped.
    """
    shell(f"{INTEGRATION}/luprex", "make clean")
    shell(INTEGRATION, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex IntegrationEditor {OS} DebugGame {INTEGRATION}/Integration.uproject -clean")
    Path(f"{INTEGRATION}/.vscode/compile_commands.json").unlink(missing_ok = True)

#
# MAIN PROGRAM
#

MODE = get_build_mode_from_command_line()
CONFIG = autodetect_system_config()
store_system_config_in_globals(CONFIG)
os.chdir(f"{INTEGRATION}/EnginePatches")

if MODE == "experiment":
    build_intellisense_database_for_clangd()

if MODE == "all":
    unzip_unreal_engine_and_apply_patch()
    generate_buildconfiguration_xml()
    generate_lpx_paths()
    generate_integration_uproject()
    run_unrealengine_setup_bat()
    build_unrealbuildtool()
    generate_integration_code_workspace()

if MODE in ["all", "c++"]:
    build_luprex_and_integration()
    build_intellisense_database_for_clangd()

if MODE == "clean":
    build_clean()