More work on build.py

build.py

@@ -1,32 +1,35 @@
 #!/usr/bin/python3
 #
-# This python script builds integration from scratch.  That includes
+# This build script can be run in three modes:
-# everything we need:
 #
-#   - Generates BuildConfiguration.xml in integration repository
+#    build.py all    - rebuilds everything
-#   - Generates BuildConfiguration.xml in UnrealEngine repository
+#    build.py c++    - works when you've only edited c++ code
-#   - Hardwires paths into Source/Integration/lpx-paths.hpp
+#    build.py clean  - remove some build products
-#   - Generates Integration.uproject
-#   - Generates Integration.code-workspace
-#   - Applies patch to Unreal Engine source.
-#   - Runs Setup.sh in the UnrealEngine repository
-#   - Builds luprex
-#   - Builds Unreal Build Tool and Shader Compile Worker
-#   - Builds Unreal Engine and Unreal Editor
-#   - Builds integration
 #
-# Once this is all done, everything is ready to go.  It is now possible
+# If you clone the UnrealEngine and integration repositories,
-# to start up the IDE and run luprex in the debugger.
+# and then run "build.py all", this will build both unreal engine and
+# integration.  It will also build project files for vscode, an
+# intellisense database, and several other things.  Once it is done,
+# you can start up vscode using 'code Integration.code-workspace'.
+# Then, you can run our game in the vscode debugger.
 #
-# If you edit the code, you can run this python script again to
+# You can also use "build.py all" to rebuild.  This is the preferred
-# rebuild.  It is always safe to use this script to rebuild after
+# way to rebuild when you aren't sure what's been edited.  However,
-# editing anything.
+# this takes almost 30 seconds even when there's nothing new to build,
+# so it can be a little slow.
+#
+# If you're sure that the only thing you've edited recently are
+# C++ files in the integration repository, then you can get away with
+# doing a lightweight build: "build.py c++".  This only works if you've
+# already completed a successful full build, and the only thing you've
+# done since then is edit C++ files in integration.  If you've edited
+# anything else, you need to use "build.py all" to rebuild.
+#
+# Editing Lua or Blueprint code doesn't require any kind of rebuild.
+# 
+# We have a "build.py clean", but it hasn't gotten a lot of love.
+# I honestly have no idea what it removes and what it doesn't.
 #
-# However, if you only edited C++ code and Unreal Build.cs files, then
-# it may be quicker and more convenient to rebuild from inside the
-# VSCODE IDE.  Bear in mind that doing so only works if you only edited
-# the C++ code and the blueprint code.  If you edited anything else, you
-# should rerun this python script.
 #
 
 import sys, os, json, shutil, subprocess, re, time, tarfile, itertools, hashlib
@@ -73,7 +76,13 @@ def read_if_exists(fn):
 JSON_VAR_REGEX = re.compile(r'\[([A-Z0-9_]+)\]')
 
 def expand_json(data, vars):
-    "A simple JSON preprocessor that can expand for-each loops and substitute variables"
+    """
+    A JSON preprocessor that can expand for-each loops and substitute variables.
+    The for-loop thing is mainly used by the code-workspace template, to avoid
+    extreme code duplication in the launch configurations and build configurations.
+    Variable substitution is used to plug hardwired paths into the file, and also
+    to implement the for-loop thing.
+    """
     if isinstance(data, dict):
         if "for-each" in data and "body" in data:
             body = data["body"]
@@ -90,8 +99,11 @@ def expand_json(data, vars):
     else:
         return data
 
+
 def expand_json_file(sourcefile, outputfile, config):
-    "Apply the json preprocessor to a file on disk"
+    """
+    Apply the json preprocessor to a file on disk
+    """
     Path(outputfile).unlink(missing_ok=True)
     data = json.loads(Path(sourcefile).read_text())
     expanded = expand_json(data, vars(config))
@@ -120,7 +132,7 @@ def autodetect_system_config():
     This autodetects where integration and unrealengine are installed,
     and it also autodetects your operating system.  Based on these,
     it returns a config object containing a variety of useful
-    configuration settings.
+    configuration settings.  We haven't tested the Windows version.
     """
     config = SimpleNamespace()
     if sys.platform == "windows":
@@ -199,7 +211,15 @@ def generate_lpx_paths():
 
 
 def patch_unrealengine_source_code():
-
+    """
+    Apply a patch to the UnrealEngine source code.
+    I tried to avoid patching Unreal engine, but it turns out there are a
+    few things that I just couldn't avoid having to fix.
+    Patch application consists of two steps: first, checkout clean, unpatched
+    versions of the files from git.  Then, apply the patch to the clean code.
+    Note: don't git-commit the patch in UnrealEngine.  Instead,
+    just let this script reapply this patch as necessary.
+    """
     patch = Path(f"{INTEGRATION}/EnginePatches/EnginePatch")
     patch_lines = patch.read_text().splitlines()
     patched_files = [line[6:] for line in patch_lines if line.startswith("--- a/")]
@@ -226,20 +246,6 @@ def run_unrealengine_setup_bat():
     shell(UNREALENGINE, f"{UNREALENGINE}/Setup.{BAT}")
 
 
-def generate_integration_code_workspace_ubt():
-    """
-    Use UnrealBuildTool to generate Integration.code-workspace.ubt
-    We're not going to use it, but we keep it as a reference that you can
-    use when editing Integration.code-workspace.tpl.json.
-    """
-    workspace = Path(f"{INTEGRATION}/Integration.code-workspace")
-    workspace_ubt = Path(f"{INTEGRATION}/Integration.code-workspace.ubt")
-    workspace.unlink(missing_ok=True)
-    workspace_ubt.unlink(missing_ok=True)
-    shell(INTEGRATION, f'{UNREALENGINE}/GenerateProjectFiles.{BAT} -projectfiles -project="{INTEGRATION}/Integration.uproject" -game')
-    workspace.rename(workspace_ubt)
-
-
 def build_unrealbuildtool():
     """
     Build the unreal build tool itself, also including shader compile worker.
@@ -249,7 +255,7 @@ def build_unrealbuildtool():
 
 def build_luprex_and_integration():
     """
-    This builds our code.
+    This builds our C++ code, and also UnrealEngine's C++ code.
     """
     shell(f"{INTEGRATION}/luprex", "make all")
     shell(INTEGRATION, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex IntegrationEditor {OS} DebugGame {INTEGRATION}/Integration.uproject")
@@ -258,9 +264,11 @@ def build_luprex_and_integration():
 def build_intellisense_database_for_clangd(force):
     """
     This builds compile_commands.json, which tells the intellisense system
-    based on clangd how to compile each source file.  We only rebuild the
+    based on clangd how to compile each source file.  We automatically rebuild
-    file if a C++ file has been added or removed, or if the force argument
+    if a C++ file has been added or removed.  That probably isn't sufficient:
-    is true.
+    we should also rebuild if a Build.cs file has been edited.  Coming Soon.
+    Until then, if your intellisense starts acting badly, do a 'build.py all',
+    which sets the 'force' argument above to true.
 
     Rebuilding the intellisense database touches rsp files, which
     unfortunately, causes the entire C++ build to be restarted the next
@@ -268,6 +276,8 @@ def build_intellisense_database_for_clangd(force):
     system.  We have a hacky workaround: we save the RSP files before
     running the intellisense build, then we restore them afterward.  
     """
+    Path(f"{INTEGRATION}/.vscode").mkdir(exist_ok = True)
+    Path(f"{UNREALENGINE}/.vscode").mkdir(exist_ok = True)
     hash_file = Path(f"{INTEGRATION}/.vscode/cpp_hash")
     new_hash = hash_json(find_cpp(f"{INTEGRATION}/Source"))
     old_hash = read_if_exists(hash_file).strip()
@@ -293,12 +303,27 @@ def build_intellisense_database_for_clangd(force):
 
 
 def generate_integration_code_workspace():
+    """
+    We build Integration.code-workspace from a template that we
+    wrote ourselves, Integration.code-workspace.tpl.json.
+    We use UnrealBuildTool to generate Integration.code-workspace.ubt,
+    but we don't use it: we just keep it as a reference that you can
+    refer to when editing the template.
+    """
+    workspace = f"{INTEGRATION}/Integration.code-workspace"
+    workspace_ubt = f"{INTEGRATION}/Integration.code-workspace.ubt"
     template = f"{INTEGRATION}/Integration.code-workspace.tpl.json"
-    target = f"{INTEGRATION}/Integration.code-workspace"
+    Path(workspace).unlink(missing_ok=True)
-    expand_json_file(template, target, CONFIG)
+    Path(workspace_ubt).unlink(missing_ok=True)
+    shell(INTEGRATION, f'{UNREALENGINE}/GenerateProjectFiles.{BAT} -projectfiles -project="{INTEGRATION}/Integration.uproject" -game')
+    Path(workspace).rename(workspace_ubt)
+    expand_json_file(template, workspace, CONFIG)
 
 
 def build_clean():
+    """
+    This code is underdeveloped.
+    """
     shell(f"{INTEGRATION}/luprex", "make clean")
     shell(INTEGRATION, f"{UNREALENGINE}/Engine/Build/BatchFiles/{BUILD_BAT} -waitmutex IntegrationEditor {OS} DebugGame {INTEGRATION}/Integration.uproject -clean")
     Path(f"{INTEGRATION}/.vscode/compile_commands.json").unlink(missing_ok = True)
@@ -312,9 +337,6 @@ CONFIG = autodetect_system_config()
 store_system_config_in_globals(CONFIG)
 os.chdir(f"{INTEGRATION}/EnginePatches")
 
-if MODE == "experiment":
-    build_unrealbuildtool()
-
 if MODE == "all":
     generate_buildconfiguration_xml()
     generate_lpx_paths()
@@ -322,15 +344,12 @@ if MODE == "all":
     generate_integration_uproject()
     run_unrealengine_setup_bat()
     build_unrealbuildtool()
+    generate_integration_code_workspace()
 
 if MODE in ["all", "c++"]:
     build_luprex_and_integration()
     build_intellisense_database_for_clangd(MODE == "all")
 
-if MODE == "all":
-    generate_integration_code_workspace_ubt()
-    generate_integration_code_workspace()
-
 if MODE == "clean":
     build_clean()