diff --git a/test/internal/build_proxy_manifest/build_proxy_manifest_tests.py b/test/internal/build_proxy_manifest/build_proxy_manifest_tests.py index 3653e6ac02..f6c42a7d04 100644 --- a/test/internal/build_proxy_manifest/build_proxy_manifest_tests.py +++ b/test/internal/build_proxy_manifest/build_proxy_manifest_tests.py @@ -5,18 +5,26 @@ def _entry(configuration="Debug", arch="arm64"): + target_id = f"@@//app:app {configuration}-{arch}" return { "action": "build", "bazelLabel": "@@//app:app", "configuration": configuration, - "outputGroup": f"bp @@//app:app {configuration}-{arch}", + "indexOutputGroups": [f"bc {target_id}", f"bi {target_id}"], + "outputGroup": f"bp {target_id}", + "previewOutputGroups": [ + f"bc {target_id}", + f"bp {target_id}", + f"bl {target_id}", + ], "product": { "basename": "App.app", + "materialization": "copy_tree", "name": "App", "path": "bazel-out/App.app", "type": "com.apple.product-type.application", }, - "targetID": f"@@//app:app {configuration}-{arch}", + "targetID": target_id, "variant": { "arch": arch, "minimumOSVersion": "18.0", @@ -39,17 +47,22 @@ def test_assemble_is_versioned_and_byte_deterministic(self): [("a", 1, release), ("b", 1, debug)], bazel_path="/usr/local/bin/bazel", generator_label="@@//app:project", + project_container="App.xcodeproj", ) reverse = build_proxy_manifest_lib.assemble( [("b", 1, debug), ("a", 1, release)], bazel_path="/usr/local/bin/bazel", generator_label="@@//app:project", + project_container="App.xcodeproj", ) self.assertEqual(forward, reverse) manifest = json.loads(forward) - self.assertEqual(manifest["schemaVersion"], 1) - self.assertEqual(manifest["capabilities"], {"actions": ["build"]}) + self.assertEqual(manifest["schemaVersion"], 2) + self.assertEqual( + manifest["capabilities"], + {"actions": ["build", "clean", "indexbuild", "preview"]}, + ) self.assertEqual( manifest["ignoredXcodeTargetGUIDs"], ["FF0100000000000000000001"], @@ -57,11 +70,18 @@ def test_assemble_is_versioned_and_byte_deterministic(self): self.assertEqual( manifest["invocation"], { + "adapterPath": "rules_xcodeproj/bazel/generate_bazel_dependencies.sh", "bazelPath": "/usr/local/bin/bazel", "bazelrcPath": "rules_xcodeproj/bazel/xcodeproj.bazelrc", + "environmentKeys": build_proxy_manifest_lib.INVOCATION_ENVIRONMENT_KEYS, "generatorLabel": "@@//app:project", + "receiptSchemaVersion": 1, }, ) + self.assertEqual( + manifest["project"], + {"containerName": "App.xcodeproj", "identity": "@@//app:project"}, + ) self.assertEqual( [entry["configuration"] for entry in manifest["targets"]], ["Debug", "Release"], @@ -79,6 +99,7 @@ def test_missing_required_key_is_rejected(self): [("fragment", 1, _line(entry))], bazel_path="bazel", generator_label="@@//app:project", + project_container="App.xcodeproj", ) def test_unsupported_action_is_rejected(self): @@ -93,6 +114,7 @@ def test_unsupported_action_is_rejected(self): [("fragment", 1, _line(entry))], bazel_path="bazel", generator_label="@@//app:project", + project_container="App.xcodeproj", ) def test_duplicate_mapping_is_rejected(self): @@ -106,6 +128,7 @@ def test_duplicate_mapping_is_rejected(self): [("fragment", 1, entry), ("fragment", 2, entry)], bazel_path="bazel", generator_label="@@//app:project", + project_container="App.xcodeproj", ) def test_missing_generator_label_is_rejected(self): @@ -117,6 +140,7 @@ def test_missing_generator_label_is_rejected(self): [], bazel_path="bazel", generator_label="", + project_container="App.xcodeproj", ) def test_missing_bazel_path_is_rejected(self): @@ -128,6 +152,34 @@ def test_missing_bazel_path_is_rejected(self): [], bazel_path="", generator_label="@@//app:project", + project_container="App.xcodeproj", + ) + + def test_product_path_traversal_is_rejected(self): + entry = _entry() + entry["product"]["path"] = "bazel-out/../foreign/App.app" + + with self.assertRaisesRegex( + build_proxy_manifest_lib.ManifestError, + "product path must not contain traversal", + ): + build_proxy_manifest_lib.assemble( + [("fragment", 1, _line(entry))], + bazel_path="bazel", + generator_label="@@//app:project", + project_container="App.xcodeproj", + ) + + def test_project_container_must_be_a_xcodeproj_basename(self): + with self.assertRaisesRegex( + build_proxy_manifest_lib.ManifestError, + "project container must be a basename", + ): + build_proxy_manifest_lib.assemble( + [], + bazel_path="bazel", + generator_label="@@//app:project", + project_container="../App.xcodeproj", ) diff --git a/tools/generators/pbxnativetargets/src/Generator/BuildProxyManifestEntry.swift b/tools/generators/pbxnativetargets/src/Generator/BuildProxyManifestEntry.swift index 9213e85cee..e6a23c6b46 100644 --- a/tools/generators/pbxnativetargets/src/Generator/BuildProxyManifestEntry.swift +++ b/tools/generators/pbxnativetargets/src/Generator/BuildProxyManifestEntry.swift @@ -11,6 +11,7 @@ import ToolCommon struct BuildProxyManifestEntry: Codable, Equatable { struct Product: Codable, Equatable { let basename: String + let materialization: String let name: String let path: String? let type: String @@ -25,7 +26,9 @@ struct BuildProxyManifestEntry: Codable, Equatable { let action: String let bazelLabel: String let configuration: String + let indexOutputGroups: [String] let outputGroup: String + let previewOutputGroups: [String] let product: Product let targetID: String let variant: Variant @@ -56,6 +59,10 @@ Missing target arguments for build proxy manifest target ID \(targetID). let topLevelAttributes = topLevelTargetAttributes[targetID] let product = Product( basename: arguments.productBasename, + materialization: Self.materializationStrategy( + basename: arguments.productBasename, + path: topLevelAttributes?.outputsProductPath + ), name: arguments.productName, path: topLevelAttributes?.outputsProductPath, type: arguments.productType.identifier @@ -72,7 +79,13 @@ Missing target arguments for build proxy manifest target ID \(targetID). action: "build", bazelLabel: consolidationMapEntry.label.description, configuration: configuration, + indexOutputGroups: ["bc \(targetID.rawValue)", "bi \(targetID.rawValue)"], outputGroup: "bp \(targetID.rawValue)", + previewOutputGroups: [ + "bc \(targetID.rawValue)", + "bp \(targetID.rawValue)", + "bl \(targetID.rawValue)", + ], product: product, targetID: targetID.rawValue, variant: variant, @@ -86,6 +99,15 @@ Missing target arguments for build proxy manifest target ID \(targetID). return entries.sorted(by: stableLessThan) } + private static func materializationStrategy( + basename: String, + path: String? + ) -> String { + guard path != nil else { return "none" } + let treeExtensions = [".app", ".appex", ".bundle", ".framework", ".xctest"] + return treeExtensions.contains(where: basename.hasSuffix) ? "copy_tree" : "copy_file" + } + static func encodeJSONLines(_ entries: [Self]) throws -> Data { let encoder = JSONEncoder() encoder.outputFormatting = [.sortedKeys, .withoutEscapingSlashes] diff --git a/tools/generators/pbxnativetargets/test/Generator/BuildProxyManifestEntryTests.swift b/tools/generators/pbxnativetargets/test/Generator/BuildProxyManifestEntryTests.swift index 197d1e6483..e51901eb1e 100644 --- a/tools/generators/pbxnativetargets/test/Generator/BuildProxyManifestEntryTests.swift +++ b/tools/generators/pbxnativetargets/test/Generator/BuildProxyManifestEntryTests.swift @@ -8,9 +8,14 @@ class BuildProxyManifestEntryTests: XCTestCase { action: "build", bazelLabel: "@@//app:app", configuration: "Release", + indexOutputGroups: ["bc @@//app:app ios-opt", "bi @@//app:app ios-opt"], outputGroup: "bp @@//app:app ios-opt", + previewOutputGroups: [ + "bc @@//app:app ios-opt", "bp @@//app:app ios-opt", "bl @@//app:app ios-opt", + ], product: .init( basename: "App.app", + materialization: "copy_tree", name: "App", path: "bazel-out/App.app", type: "com.apple.product-type.application" @@ -27,9 +32,14 @@ class BuildProxyManifestEntryTests: XCTestCase { action: "build", bazelLabel: "@@//app:app", configuration: "Debug", + indexOutputGroups: ["bc @@//app:app ios-sim-dbg", "bi @@//app:app ios-sim-dbg"], outputGroup: "bp @@//app:app ios-sim-dbg", + previewOutputGroups: [ + "bc @@//app:app ios-sim-dbg", "bp @@//app:app ios-sim-dbg", "bl @@//app:app ios-sim-dbg", + ], product: .init( basename: "App.app", + materialization: "copy_tree", name: "App", path: "bazel-out/App.app", type: "com.apple.product-type.application" @@ -47,10 +57,19 @@ class BuildProxyManifestEntryTests: XCTestCase { let reverse = try BuildProxyManifestEntry.encodeJSONLines([second, first]) XCTAssertEqual(forward, reverse) + let objects = try forward.split(separator: 0x0A).map { line in + try XCTUnwrap( + JSONSerialization.jsonObject(with: Data(line)) as? [String: Any] + ) + } + XCTAssertEqual(objects.compactMap { $0["configuration"] as? String }, ["Debug", "Release"]) XCTAssertEqual( - String(decoding: forward, as: UTF8.self), - #"{"action":"build","bazelLabel":"@@//app:app","configuration":"Debug","outputGroup":"bp @@//app:app ios-sim-dbg","product":{"basename":"App.app","name":"App","path":"bazel-out/App.app","type":"com.apple.product-type.application"},"targetID":"@@//app:app ios-sim-dbg","variant":{"arch":"arm64","minimumOSVersion":"18.0","platform":"iphonesimulator"},"xcodeTargetGUID":"0000AAAAAAAA000000000001"}"# + "\n" + - #"{"action":"build","bazelLabel":"@@//app:app","configuration":"Release","outputGroup":"bp @@//app:app ios-opt","product":{"basename":"App.app","name":"App","path":"bazel-out/App.app","type":"com.apple.product-type.application"},"targetID":"@@//app:app ios-opt","variant":{"arch":"arm64","minimumOSVersion":"18.0","platform":"iphoneos"},"xcodeTargetGUID":"0000AAAAAAAA000000000001"}"# + "\n" + objects.first?["previewOutputGroups"] as? [String], + [ + "bc @@//app:app ios-sim-dbg", + "bp @@//app:app ios-sim-dbg", + "bl @@//app:app ios-sim-dbg", + ] ) } @@ -59,9 +78,14 @@ class BuildProxyManifestEntryTests: XCTestCase { action: "build", bazelLabel: "@@//lib", configuration: "Debug", + indexOutputGroups: ["bc @@//lib macos-dbg", "bi @@//lib macos-dbg"], outputGroup: "bp @@//lib macos-dbg", + previewOutputGroups: [ + "bc @@//lib macos-dbg", "bp @@//lib macos-dbg", "bl @@//lib macos-dbg", + ], product: .init( basename: "libLib.a", + materialization: "none", name: "Lib", path: nil, type: "com.apple.product-type.library.static" diff --git a/xcodeproj/internal/bazel_integration_files/BUILD b/xcodeproj/internal/bazel_integration_files/BUILD index 7410d13846..b2ce777b13 100644 --- a/xcodeproj/internal/bazel_integration_files/BUILD +++ b/xcodeproj/internal/bazel_integration_files/BUILD @@ -1,3 +1,5 @@ +load("@rules_python//python:defs.bzl", "py_library", "py_test") + _BASE_FILES = [ "calculate_output_groups.py", "copy_dsyms.sh", @@ -6,6 +8,31 @@ _BASE_FILES = [ "process_bazel_build_log.py", ] +_TEST_FILES = [ + "process_bazel_build_log_tests.py", + "write_build_proxy_invocation_receipt_tests.py", +] + +py_library( + name = "write_build_proxy_invocation_receipt_lib", + srcs = ["write_build_proxy_invocation_receipt.py"], +) + +py_test( + name = "write_build_proxy_invocation_receipt_tests", + srcs = ["write_build_proxy_invocation_receipt_tests.py"], + deps = [":write_build_proxy_invocation_receipt_lib"], +) + +py_test( + name = "process_bazel_build_log_tests", + srcs = [ + "process_bazel_build_log.py", + "process_bazel_build_log_tests.py", + ], + main = "process_bazel_build_log_tests.py", +) + filegroup( name = "bazel_integration_files", srcs = _BASE_FILES + [ @@ -14,7 +41,7 @@ filegroup( ":rsync_excludes", ] + glob( ["**/*"], - exclude = _BASE_FILES + [ + exclude = _BASE_FILES + _TEST_FILES + [ "**/*.bzl", "BUILD", ], diff --git a/xcodeproj/internal/bazel_integration_files/process_bazel_build_log.py b/xcodeproj/internal/bazel_integration_files/process_bazel_build_log.py index ff05b5a437..38afbbe5e2 100755 --- a/xcodeproj/internal/bazel_integration_files/process_bazel_build_log.py +++ b/xcodeproj/internal/bazel_integration_files/process_bazel_build_log.py @@ -2,10 +2,11 @@ import os import re +import signal import subprocess import sys -from typing import List -import signal +import threading +from typing import List, Optional STRIP_COLOR_RE = re.compile(r"\x1b\[[0-9;]{1,}[A-Za-z]") @@ -20,19 +21,56 @@ re.VERBOSE, ) + def _uppercase_first_letter(s: str) -> str: return s[:1].upper() + s[1:] + def _main(command: List[str]) -> None: + proxy_mode = bool(os.getenv("SWIFTBUILD_BAZEL_PROXY_REQUEST_DIR")) + cancel_grace_seconds = float( + os.getenv("SWIFTBUILD_BAZEL_PROXY_CANCEL_GRACE_SECONDS", "5.0") + ) + cancel_grace_seconds = min(max(cancel_grace_seconds, 0.1), 5.0) + process: Optional[subprocess.Popen[str]] = None + kill_timer: Optional[threading.Timer] = None + + def _signal_process_group(sig: signal.Signals) -> None: + if not process: + return + try: + if proxy_mode: + # The group can outlive its leader when a nested tool ignores + # SIGTERM, so escalation must still address the original PGID + # after the wrapper process has exited. + os.killpg(process.pid, sig) + elif process.poll() is None: + process.send_signal(sig) + except ProcessLookupError: + return + + def _force_kill() -> None: + _signal_process_group(signal.SIGKILL) + def _signal_handler(signum, frame): - """Print signal information and ignore the signal.""" + """Forward cancellation to Bazel and bound the shutdown grace period.""" + nonlocal kill_timer signal_name = signal.Signals(signum).name print(f"\nReceived signal {signal_name} ({signum})\n", file=sys.stderr) - return - - # Set up signal handler for SIGINT + if not proxy_mode: + return + _signal_process_group(signal.SIGTERM) + if kill_timer is None: + kill_timer = threading.Timer(cancel_grace_seconds, _force_kill) + kill_timer.daemon = True + kill_timer.start() + + # Preserve native integration behavior outside proxy mode. Proxy mode also + # owns SIGTERM so cancelling the Swift Build operation reaches Bazel. signal.signal(signal.SIGINT, _signal_handler) + if proxy_mode: + signal.signal(signal.SIGTERM, _signal_handler) srcroot = os.getenv("SRCROOT") if not srcroot: @@ -83,7 +121,11 @@ def _replacement(match: re.Match) -> str: return f"{prefix}/{message}" process = subprocess.Popen( - command, bufsize=1, stderr=subprocess.PIPE, universal_newlines=True + command, + bufsize=1, + stderr=subprocess.PIPE, + start_new_session=proxy_mode, + universal_newlines=True, ) assert process.stderr @@ -110,6 +152,9 @@ def _process_log_line(line: str): for line in process.stderr: _process_log_line(line) + if kill_timer is not None: + kill_timer.cancel() + # If the Bazel invocation failed and there was no formatted error found, # print a nicer error message instead of a cryptic in Xcode: # 'Command PhaseScriptExecution failed with a nonzero exit code' diff --git a/xcodeproj/internal/bazel_integration_files/process_bazel_build_log_tests.py b/xcodeproj/internal/bazel_integration_files/process_bazel_build_log_tests.py new file mode 100644 index 0000000000..777116b937 --- /dev/null +++ b/xcodeproj/internal/bazel_integration_files/process_bazel_build_log_tests.py @@ -0,0 +1,101 @@ +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import time +import unittest + + +class ProcessBazelBuildLogTests(unittest.TestCase): + def test_proxy_cancellation_terminates_the_bazel_process_group(self): + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + child_pid_path = root / "child.pid" + helper = ( + "import os,signal,time;" + f"open({str(child_pid_path)!r},'w').write(str(os.getpid()));" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "time.sleep(60)" + ) + environment = os.environ.copy() + environment.update( + { + "BAZEL_EXTERNAL": str(root / "external"), + "BAZEL_OUT": str(root / "bazel-out"), + "PROJECT_DIR": str(root), + "SRCROOT": str(root), + "SWIFTBUILD_BAZEL_PROXY_CANCEL_GRACE_SECONDS": "0.1", + "SWIFTBUILD_BAZEL_PROXY_REQUEST_DIR": str(root / "request"), + } + ) + wrapper = Path(__file__).with_name("process_bazel_build_log.py") + process = subprocess.Popen( + [sys.executable, str(wrapper), sys.executable, "-c", helper], + env=environment, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + ) + deadline = time.monotonic() + 5 + while not child_pid_path.exists() and time.monotonic() < deadline: + time.sleep(0.01) + self.assertTrue(child_pid_path.exists()) + child_pid = int(child_pid_path.read_text()) + + process.send_signal(signal.SIGTERM) + process.communicate(timeout=5) + self.assertNotEqual(process.returncode, 0) + with self.assertRaises(ProcessLookupError): + os.kill(child_pid, 0) + + def test_proxy_cancellation_kills_a_group_after_its_leader_exits(self): + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + child_pid_path = root / "child.pid" + child = ( + "import os,signal,time;" + "signal.signal(signal.SIGTERM,signal.SIG_IGN);" + "time.sleep(60)" + ) + leader = ( + "import subprocess,sys,time;" + f"child=subprocess.Popen([sys.executable,'-c',{child!r}]);" + f"open({str(child_pid_path)!r},'w').write(str(child.pid));" + "time.sleep(60)" + ) + environment = os.environ.copy() + environment.update( + { + "BAZEL_EXTERNAL": str(root / "external"), + "BAZEL_OUT": str(root / "bazel-out"), + "PROJECT_DIR": str(root), + "SRCROOT": str(root), + "SWIFTBUILD_BAZEL_PROXY_CANCEL_GRACE_SECONDS": "0.1", + "SWIFTBUILD_BAZEL_PROXY_REQUEST_DIR": str(root / "request"), + } + ) + wrapper = Path(__file__).with_name("process_bazel_build_log.py") + process = subprocess.Popen( + [sys.executable, str(wrapper), sys.executable, "-c", leader], + env=environment, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + ) + deadline = time.monotonic() + 5 + while not child_pid_path.exists() and time.monotonic() < deadline: + time.sleep(0.01) + self.assertTrue(child_pid_path.exists()) + child_pid = int(child_pid_path.read_text()) + + process.send_signal(signal.SIGTERM) + process.communicate(timeout=5) + self.assertNotEqual(process.returncode, 0) + with self.assertRaises(ProcessLookupError): + os.kill(child_pid, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/xcodeproj/internal/bazel_integration_files/write_build_proxy_invocation_receipt.py b/xcodeproj/internal/bazel_integration_files/write_build_proxy_invocation_receipt.py new file mode 100644 index 0000000000..d78e0dc690 --- /dev/null +++ b/xcodeproj/internal/bazel_integration_files/write_build_proxy_invocation_receipt.py @@ -0,0 +1,105 @@ +#!/usr/bin/python3 +"""Writes a redacted receipt for the exact generated Bazel invocation policy.""" + +import argparse +import json +import os +from pathlib import Path +import re +import tempfile + + +SENSITIVE_OPTION = re.compile( + r"(?:^|--|[;, ])(?:remote_header|bes_header|remote_cache_header|" + r"authorization|cookie|token|password)(?:=|:)|" + r"(?:^|[=;, ])(?:[A-Za-z0-9_]*(?:TOKEN|PASSWORD|SECRET|CREDENTIAL|API_KEY)" + r"[A-Za-z0-9_]*)=", + re.IGNORECASE, +) + + +def _pairs(values, name): + result = {} + for value in values: + key, separator, item = value.partition("=") + if not separator or not key: + raise ValueError(f"invalid {name} value: expected KEY=VALUE") + if key in result: + raise ValueError(f"duplicate {name} key: {key}") + result[key] = item + return result + + +def _validate_redaction(values): + for value in values: + if SENSITIVE_OPTION.search(value): + raise ValueError("credential-bearing options are forbidden in receipts") + + +def _write_private_json(path, value): + destination = path.expanduser().resolve() + destination.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{destination.name}.", + dir=destination.parent, + ) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + json.dump(value, stream, indent=2, sort_keys=True) + stream.write("\n") + os.replace(temporary_name, destination) + except BaseException: + try: + os.close(descriptor) + except OSError: + pass + try: + os.unlink(temporary_name) + except OSError: + pass + raise + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("output", type=Path) + parser.add_argument("--working-directory", required=True) + parser.add_argument("--command", required=True) + parser.add_argument("--startup-option", action="append", default=[]) + parser.add_argument("--bazelrc", action="append", default=[]) + parser.add_argument("--command-option", action="append", default=[]) + parser.add_argument("--target", action="append", default=[]) + parser.add_argument("--label", action="append", default=[]) + parser.add_argument("--output-group", action="append", default=[]) + parser.add_argument("--target-id", action="append", default=[]) + parser.add_argument("--environment-key", action="append", default=[]) + parser.add_argument("--mode", action="append", default=[]) + parser.add_argument("--materialization", action="append", default=[]) + parser.add_argument("--bep-path") + args = parser.parse_args() + + _validate_redaction( + args.startup_option + args.bazelrc + args.command_option + args.target + ) + receipt = { + "schemaVersion": 1, + "workingDirectory": args.working_directory, + "startupOptions": args.startup_option, + "bazelrcs": args.bazelrc, + "command": args.command, + "commandOptions": args.command_option, + "targets": sorted(set(args.target)), + "labels": sorted(set(args.label)), + "outputGroups": sorted(set(args.output_group)), + "targetIDs": sorted(set(args.target_id)), + "environmentKeys": sorted(set(args.environment_key)), + "modes": _pairs(args.mode, "mode"), + "materialization": _pairs(args.materialization, "materialization"), + "provenance": {"bepPath": args.bep_path} if args.bep_path else {}, + } + _write_private_json(args.output, receipt) + + +if __name__ == "__main__": + main() diff --git a/xcodeproj/internal/bazel_integration_files/write_build_proxy_invocation_receipt_tests.py b/xcodeproj/internal/bazel_integration_files/write_build_proxy_invocation_receipt_tests.py new file mode 100644 index 0000000000..a9e3d7f2a2 --- /dev/null +++ b/xcodeproj/internal/bazel_integration_files/write_build_proxy_invocation_receipt_tests.py @@ -0,0 +1,74 @@ +import json +import stat +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from xcodeproj.internal.bazel_integration_files import write_build_proxy_invocation_receipt + + +class WriteBuildProxyInvocationReceiptTests(unittest.TestCase): + def test_writes_redacted_private_receipt(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "receipt.json" + arguments = [ + "writer", + str(output), + "--working-directory=/workspace", + "--command=build", + "--startup-option=--output_base=/tmp/output", + "--bazelrc=/workspace/generated.bazelrc", + "--command-option=--config=_rules_xcodeproj_build", + "--target=//app:project", + "--label=//app:app", + "--output-group=bp //app:app ios-sim", + "--target-id=//app:app ios-sim", + "--environment-key=DEVELOPER_DIR", + "--mode=config=_rules_xcodeproj_build", + "--materialization=contract=manifest-v2", + "--bep-path=/private/build-event.jsonl", + ] + with mock.patch("sys.argv", arguments): + write_build_proxy_invocation_receipt.main() + + receipt = json.loads(output.read_text(encoding="utf-8")) + self.assertEqual(receipt["schemaVersion"], 1) + self.assertEqual(receipt["environmentKeys"], ["DEVELOPER_DIR"]) + self.assertNotIn("environment", receipt) + self.assertEqual(receipt["provenance"]["bepPath"], "/private/build-event.jsonl") + self.assertEqual(stat.S_IMODE(output.stat().st_mode), 0o600) + + def test_rejects_credential_bearing_option(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "receipt.json" + arguments = [ + "writer", + str(output), + "--working-directory=/workspace", + "--command=build", + "--command-option=--remote_header=authorization:secret", + ] + with mock.patch("sys.argv", arguments): + with self.assertRaisesRegex(ValueError, "credential-bearing"): + write_build_proxy_invocation_receipt.main() + self.assertFalse(output.exists()) + + def test_rejects_secret_repo_environment_option(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "receipt.json" + arguments = [ + "writer", + str(output), + "--working-directory=/workspace", + "--command=build", + "--command-option=--repo_env=PRIVATE_API_KEY=do-not-record", + ] + with mock.patch("sys.argv", arguments): + with self.assertRaisesRegex(ValueError, "credential-bearing"): + write_build_proxy_invocation_receipt.main() + self.assertFalse(output.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/xcodeproj/internal/build_proxy_manifest.bzl b/xcodeproj/internal/build_proxy_manifest.bzl index 7d72ab8f52..6aeb9ee346 100644 --- a/xcodeproj/internal/build_proxy_manifest.bzl +++ b/xcodeproj/internal/build_proxy_manifest.bzl @@ -7,8 +7,9 @@ def write_build_proxy_manifest( entries_files, generator_label, name, + project_container, tool): - """Assembles deterministic shard entries into the v1 proxy manifest. + """Assembles deterministic shard entries into the v2 proxy manifest. Args: actions: `ctx.actions`. @@ -16,6 +17,7 @@ def write_build_proxy_manifest( entries_files: JSON Lines manifest fragments from target shards. generator_label: The `xcodeproj` generator label the proxy must build. name: The generator target name. + project_container: Basename of the generated `.xcodeproj` bundle. tool: The executable that validates and assembles the manifest. Returns: @@ -29,6 +31,7 @@ def write_build_proxy_manifest( args.add(output) args.add(str(generator_label)) args.add(bazel_path) + args.add(project_container) args.add_all(entries_files) actions.run( diff --git a/xcodeproj/internal/build_proxy_manifest.py b/xcodeproj/internal/build_proxy_manifest.py index 43105bbc5e..d28584b205 100644 --- a/xcodeproj/internal/build_proxy_manifest.py +++ b/xcodeproj/internal/build_proxy_manifest.py @@ -12,6 +12,7 @@ def main() -> None: parser.add_argument("output", type=Path) parser.add_argument("generator_label") parser.add_argument("bazel_path") + parser.add_argument("project_container") parser.add_argument("fragments", nargs="*", type=Path) args = parser.parse_args() build_proxy_manifest_lib.write( @@ -19,6 +20,7 @@ def main() -> None: args.fragments, bazel_path=args.bazel_path, generator_label=args.generator_label, + project_container=args.project_container, ) diff --git a/xcodeproj/internal/build_proxy_manifest_lib.py b/xcodeproj/internal/build_proxy_manifest_lib.py index 1df27f9314..0511d782cf 100644 --- a/xcodeproj/internal/build_proxy_manifest_lib.py +++ b/xcodeproj/internal/build_proxy_manifest_lib.py @@ -1,31 +1,66 @@ -"""Deterministically assembles version 1 build-proxy manifest fragments.""" +"""Deterministically assembles version 2 build-proxy manifest fragments.""" import json from pathlib import Path from typing import Iterable, Mapping, Sequence -SCHEMA_VERSION = 1 -CAPABILITIES = {"actions": ["build"]} +SCHEMA_VERSION = 2 +CAPABILITIES = {"actions": ["build", "clean", "indexbuild", "preview"]} BAZELRC_PATH = "rules_xcodeproj/bazel/xcodeproj.bazelrc" +ADAPTER_PATH = "rules_xcodeproj/bazel/generate_bazel_dependencies.sh" BAZEL_DEPENDENCIES_TARGET_GUID = "FF0100000000000000000001" +RECEIPT_SCHEMA_VERSION = 1 +INVOCATION_ENVIRONMENT_KEYS = [ + "ACTION", + "BAZEL_CONFIG", + "BAZEL_EXTERNAL", + "BAZEL_INTEGRATION_DIR", + "BAZEL_OUT", + "BAZEL_OUTPUT_BASE", + "BAZEL_SEPARATE_INDEXBUILD_OUTPUT_BASE", + "BAZEL_SUPPRESS_COVERAGE_BUILD", + "CLANG_COVERAGE_MAPPING", + "COLOR_DIAGNOSTICS", + "DEVELOPER_DIR", + "ENABLE_ADDRESS_SANITIZER", + "ENABLE_PREVIEWS", + "ENABLE_THREAD_SANITIZER", + "ENABLE_UNDEFINED_BEHAVIOR_SANITIZER", + "HOME", + "IMPORT_INDEX_BUILD_INDEXSTORES", + "INDEX_DATA_STORE_DIR", + "INDEXING_PROJECT_DIR__NO", + "OBJROOT", + "PROJECT_DIR", + "RULES_XCODEPROJ_BUILD_MODE", + "SRCROOT", + "TERM", + "TOOLCHAINS", + "USER", + "XCODE_PRODUCT_BUILD_VERSION", + "XCODE_VERSION_ACTUAL", +] _REQUIRED_ENTRY_KEYS = { "action", "bazelLabel", "configuration", + "indexOutputGroups", "outputGroup", + "previewOutputGroups", "product", "targetID", "variant", "xcodeTargetGUID", } -_REQUIRED_PRODUCT_KEYS = {"basename", "name", "type"} +_REQUIRED_PRODUCT_KEYS = {"basename", "materialization", "name", "type"} _REQUIRED_VARIANT_KEYS = {"arch", "minimumOSVersion", "platform"} +_MATERIALIZATION_STRATEGIES = {"copy_file", "copy_tree", "none"} class ManifestError(ValueError): - """Raised when a generated manifest fragment violates the v1 contract.""" + """Raised when a generated manifest fragment violates the v2 contract.""" def _validate_entry(entry: object, source: str, line_number: int) -> Mapping: @@ -42,6 +77,25 @@ def _validate_entry(entry: object, source: str, line_number: int) -> Mapping: f"{source}:{line_number}: unsupported action: {entry['action']!r}" ) + target_id = entry["targetID"] + output_group = entry["outputGroup"] + if not isinstance(target_id, str) or not target_id: + raise ManifestError(f"{source}:{line_number}: targetID must be a string") + if output_group != f"bp {target_id}": + raise ManifestError( + f"{source}:{line_number}: outputGroup must identify targetID" + ) + for name, prefixes in [ + ("indexOutputGroups", ("bc ", "bi ")), + ("previewOutputGroups", ("bc ", "bp ", "bl ")), + ]: + values = entry[name] + expected = [f"{prefix}{target_id}" for prefix in prefixes] + if values != expected: + raise ManifestError( + f"{source}:{line_number}: {name} must identify targetID" + ) + product = entry["product"] if not isinstance(product, dict): raise ManifestError(f"{source}:{line_number}: product must be an object") @@ -51,6 +105,25 @@ def _validate_entry(entry: object, source: str, line_number: int) -> Mapping: f"{source}:{line_number}: product missing keys: " f"{', '.join(sorted(missing_product))}" ) + strategy = product["materialization"] + if strategy not in _MATERIALIZATION_STRATEGIES: + raise ManifestError( + f"{source}:{line_number}: unsupported materialization: {strategy!r}" + ) + product_path = product.get("path") + if strategy == "none" and product_path is not None: + raise ManifestError( + f"{source}:{line_number}: materialization 'none' must not declare a path" + ) + if strategy != "none": + if not isinstance(product_path, str) or not product_path.startswith("bazel-out/"): + raise ManifestError( + f"{source}:{line_number}: materializable product path must be under bazel-out/" + ) + if ".." in Path(product_path).parts: + raise ManifestError( + f"{source}:{line_number}: product path must not contain traversal" + ) variant = entry["variant"] if not isinstance(variant, dict): @@ -83,12 +156,17 @@ def assemble( *, bazel_path: str, generator_label: str, + project_container: str, ) -> bytes: - """Returns canonical v1 JSON bytes for named JSON Lines input.""" + """Returns canonical v2 JSON bytes for named JSON Lines input.""" if not generator_label: raise ManifestError("generator label must not be empty") if not bazel_path: raise ManifestError("Bazel path must not be empty") + if not project_container or Path(project_container).name != project_container: + raise ManifestError("project container must be a basename") + if not project_container.endswith(".xcodeproj"): + raise ManifestError("project container must end in .xcodeproj") entries = [] for source, line_number, line in fragment_lines: @@ -117,9 +195,16 @@ def assemble( "capabilities": CAPABILITIES, "ignoredXcodeTargetGUIDs": [BAZEL_DEPENDENCIES_TARGET_GUID], "invocation": { + "adapterPath": ADAPTER_PATH, "bazelPath": bazel_path, "bazelrcPath": BAZELRC_PATH, + "environmentKeys": INVOCATION_ENVIRONMENT_KEYS, "generatorLabel": generator_label, + "receiptSchemaVersion": RECEIPT_SCHEMA_VERSION, + }, + "project": { + "containerName": project_container, + "identity": generator_label, }, "schemaVersion": SCHEMA_VERSION, "targets": entries, @@ -140,6 +225,7 @@ def assemble_files( *, bazel_path: str, generator_label: str, + project_container: str, ) -> bytes: lines = [] for path in paths: @@ -152,6 +238,7 @@ def assemble_files( lines, bazel_path=bazel_path, generator_label=generator_label, + project_container=project_container, ) @@ -161,11 +248,13 @@ def write( *, bazel_path: str, generator_label: str, + project_container: str, ) -> None: output.write_bytes( assemble_files( fragments, bazel_path=bazel_path, generator_label=generator_label, + project_container=project_container, ) ) diff --git a/xcodeproj/internal/templates/bazel_build.sh b/xcodeproj/internal/templates/bazel_build.sh index 9b6f1bbf2a..52fb8f9cc4 100755 --- a/xcodeproj/internal/templates/bazel_build.sh +++ b/xcodeproj/internal/templates/bazel_build.sh @@ -86,7 +86,18 @@ for var in "${allowed_vars[@]}"; do fi done +readonly build_proxy_bazel_real="${BAZEL_REAL:-}" source "$BAZEL_INTEGRATION_DIR/bazel_env.sh" +if [[ + -n "${SWIFTBUILD_BAZEL_PROXY_REQUEST_DIR:-}" && + -n "$build_proxy_bazel_real" +]]; then + build_proxy_envs=() + for item in "${envs[@]}"; do + [[ "$item" != BAZEL_REAL=* ]] && build_proxy_envs+=("$item") + done + envs=("${build_proxy_envs[@]}" "BAZEL_REAL=$build_proxy_bazel_real") +fi bazel_cmd=( env -i @@ -157,20 +168,96 @@ fi # Build +if [[ -n "${SWIFTBUILD_BAZEL_PROXY_INVOCATION_RECEIPT:-}" ]]; then + receipt_args=( + "$SWIFTBUILD_BAZEL_PROXY_INVOCATION_RECEIPT" + --working-directory "$SRCROOT" + --command build + "--startup-option=--host_jvm_args=-Xdock:name=$DEVELOPER_DIR" + "--startup-option=--output_base=$output_base" + --target "%generator_label%" + --mode "action=$ACTION" + --mode "config=$config" + --mode "coverage=${CLANG_COVERAGE_MAPPING:-NO}" + --mode "previews=${ENABLE_PREVIEWS:-NO}" + --materialization "contract=manifest-v2" + ) + for bazelrc in "${bazelrcs[@]}"; do + receipt_args+=("--bazelrc=${bazelrc#--bazelrc=}") + done + for option in "${base_pre_config_flags[@]}" "${build_pre_config_flags[@]}"; do + if [[ "$option" != --build_event_json_file=* ]]; then + receipt_args+=("--command-option=$option") + fi + done + receipt_args+=("--command-option=--config=$config") + receipt_args+=("--command-option=--color=yes") + if [[ -n "${toolchain:-}" ]]; then + receipt_args+=("--command-option=--action_env=TOOLCHAINS=$toolchain") + fi + receipt_args+=("--command-option=$output_groups_flag") + for label in "${labels[@]:-}"; do + [[ -n "$label" ]] && receipt_args+=(--label "$label") + done + for output_group in "${output_groups[@]}"; do + receipt_args+=(--output-group "$output_group") + done + for target_id in "${target_ids[@]:-}"; do + [[ -n "$target_id" ]] && receipt_args+=(--target-id "$target_id") + done + for item in "${passthrough_env[@]}" "${envs[@]}"; do + receipt_args+=(--environment-key "${item%%=*}") + done + if [[ -n "${SWIFTBUILD_BAZEL_PROXY_BEP_PATH:-}" ]]; then + receipt_args+=(--bep-path "$SWIFTBUILD_BAZEL_PROXY_BEP_PATH") + fi + "$BAZEL_INTEGRATION_DIR/write_build_proxy_invocation_receipt.py" \ + "${receipt_args[@]}" +fi + echo "Starting Bazel build" -"$BAZEL_INTEGRATION_DIR/process_bazel_build_log.py" \ - "${bazel_cmd[@]}" \ - build \ - "${base_pre_config_flags[@]}" \ - ${build_pre_config_flags:+"${build_pre_config_flags[@]}"} \ - --config="$config" \ - --color=yes \ - ${toolchain:+--action_env=TOOLCHAINS="$toolchain"} \ - "$output_groups_flag" \ - "%generator_label%" \ - ${labels:+"--build_metadata=PATTERN=${labels[*]}"} \ - 2>&1 +build_log_cmd=( + "$BAZEL_INTEGRATION_DIR/process_bazel_build_log.py" + "${bazel_cmd[@]}" + build + "${base_pre_config_flags[@]}" + ${build_pre_config_flags:+"${build_pre_config_flags[@]}"} + --config="$config" + --color=yes +) +if [[ -n "${toolchain:-}" ]]; then + build_log_cmd+=("--action_env=TOOLCHAINS=$toolchain") +fi +build_log_cmd+=( + "$output_groups_flag" + "%generator_label%" +) +if [[ -n "${labels:-}" ]]; then + build_log_cmd+=("--build_metadata=PATTERN=${labels[*]}") +fi +readonly build_log_cmd + +if [[ -n "${SWIFTBUILD_BAZEL_PROXY_REQUEST_DIR:-}" ]]; then + "${build_log_cmd[@]}" 2>&1 & + readonly proxy_build_pid=$! + cancel_proxy_build() { + kill -TERM "$proxy_build_pid" 2>/dev/null || true + wait "$proxy_build_pid" 2>/dev/null || true + exit 143 + } + trap cancel_proxy_build TERM INT + set +e + wait "$proxy_build_pid" + proxy_build_status=$? + set -e + trap - TERM INT + if [[ $proxy_build_status -ne 0 ]]; then + exit "$proxy_build_status" + fi +else + "${build_log_cmd[@]}" 2>&1 +fi # Verify that we actually built what we requested diff --git a/xcodeproj/internal/templates/generate_bazel_dependencies.sh b/xcodeproj/internal/templates/generate_bazel_dependencies.sh index 247f57f17d..7fc289a4f7 100755 --- a/xcodeproj/internal/templates/generate_bazel_dependencies.sh +++ b/xcodeproj/internal/templates/generate_bazel_dependencies.sh @@ -6,7 +6,49 @@ cd "$SRCROOT" # Calculate Bazel `--output_groups` -if [ "$ACTION" == "indexbuild" ]; then +if [[ -n "${SWIFTBUILD_BAZEL_PROXY_REQUEST_DIR:-}" ]]; then + readonly request_dir="$SWIFTBUILD_BAZEL_PROXY_REQUEST_DIR" + if [[ ! -d "$request_dir" ]]; then + echo "error: Bazel build proxy request directory does not exist." >&2 + exit 1 + fi + + labels=() + while IFS= read -r value || [[ -n "$value" ]]; do + [[ -n "$value" ]] && labels+=("$value") + done < "$request_dir/labels" + + target_ids=() + while IFS= read -r value || [[ -n "$value" ]]; do + [[ -n "$value" ]] && target_ids+=("$value") + done < "$request_dir/target_ids" + + output_groups=("target_ids_list") + while IFS= read -r value || [[ -n "$value" ]]; do + [[ -n "$value" ]] && output_groups+=("$value") + done < "$request_dir/output_groups" + indexstores_filelists=() + if [[ "$ACTION" == "indexbuild" && "${IMPORT_INDEX_BUILD_INDEXSTORES:-NO}" == "YES" ]]; then + output_groups+=("index_import") + readonly targetid_regex='@{0,2}(.*)//(.*):(.*) ([^\ ]+)$' + for target_id in "${target_ids[@]}"; do + if [[ "$target_id" =~ $targetid_regex ]]; then + repo="${BASH_REMATCH[1]}" + if [[ "$repo" == "@" ]]; then + repo="" + fi + package="${BASH_REMATCH[2]}" + target="${BASH_REMATCH[3]}" + configuration="${BASH_REMATCH[4]}" + indexstores_filelists+=( + "$configuration/bin/${repo:+"external/$repo/"}$package/$target-bi.filelist" + ) + fi + done + elif [[ "$ACTION" != "indexbuild" ]]; then + output_groups=("index_import" "${output_groups[@]}") + fi +elif [ "$ACTION" == "indexbuild" ]; then echo >&2 "error: \`BazelDependencies\` should not run during Index Build." \ "Please file a bug report here:" \ "https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bug.md" @@ -78,7 +120,7 @@ else done readonly indexstores_filelists - if [ "${#output_groups[@]}" -eq 1 ]; then + if [ "${#labels_and_output_groups[@]}" -eq 0 ]; then echo "BazelDependencies invoked without any output groups set." \ "Exiting early." exit @@ -112,6 +154,11 @@ build_pre_config_flags=( # this once Bazel adds support for a Remote Output Service. "--remote_download_regex=.*\.indexstore/.*|.*\.(a|cfg|c|C|cc|cl|cpp|cu|cxx|c++|def|h|H|hh|hpp|hxx|h++|hmap|ilc|inc|inl|ipp|tcc|tlh|tli|tpp|m|modulemap|mm|pch|swift|swiftdoc|swiftmodule|swiftsourceinfo|yaml)$" ) +if [[ -n "${SWIFTBUILD_BAZEL_PROXY_BEP_PATH:-}" ]]; then + build_pre_config_flags+=( + "--build_event_json_file=$SWIFTBUILD_BAZEL_PROXY_BEP_PATH" + ) +fi apply_sanitizers=1 if [ "$ACTION" == "indexbuild" ]; then diff --git a/xcodeproj/internal/xcodeproj_rule.bzl b/xcodeproj/internal/xcodeproj_rule.bzl index c8d5b97ad0..fbe19eb77d 100644 --- a/xcodeproj/internal/xcodeproj_rule.bzl +++ b/xcodeproj/internal/xcodeproj_rule.bzl @@ -424,6 +424,7 @@ def _write_project_contents( entries_files = build_proxy_manifest_entries_files, generator_label = generator_label, name = name, + project_container = install_path, tool = build_proxy_manifest_generator, )