From 76101c9777389fb36da70437e9fb42157c8445cd Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 14 Nov 2025 14:22:52 -0800 Subject: [PATCH 01/51] chore: squash all commits on top of 3.35.7 --- .github/workflows/ci.yml | 43 +++ .github/workflows/shorebird_ci.yml | 60 ++++ DEPS | 9 +- engine/src/build/config/compiler/BUILD.gn | 10 +- engine/src/build/toolchain/win/BUILD.gn | 5 +- .../src/build/toolchain/win/tool_wrapper.py | 11 + engine/src/flutter/BUILD.gn | 27 +- engine/src/flutter/build/archives/BUILD.gn | 18 +- .../src/flutter/build/dart/tools/dart_pkg.py | 10 +- engine/src/flutter/common/config.gni | 8 + engine/src/flutter/lib/snapshot/BUILD.gn | 76 +++- engine/src/flutter/runtime/dart_snapshot.cc | 104 ++++-- engine/src/flutter/shell/common/BUILD.gn | 27 ++ engine/src/flutter/shell/common/shell.cc | 10 + .../flutter/shell/common/shorebird/BUILD.gn | 60 ++++ .../shell/common/shorebird/shorebird.cc | 335 ++++++++++++++++++ .../shell/common/shorebird/shorebird.h | 47 +++ .../common/shorebird/shorebird_unittests.cc | 21 ++ .../common/shorebird/snapshots_data_handle.cc | 144 ++++++++ .../common/shorebird/snapshots_data_handle.h | 48 +++ .../snapshots_data_handle_unittests.cc | 177 +++++++++ .../flutter/shell/platform/android/BUILD.gn | 20 +- .../platform/android/android_exports.lst | 12 + .../shell/platform/android/flutter_main.cc | 21 +- .../shell/platform/android/flutter_main.h | 3 + .../flutter/embedding/engine/FlutterJNI.java | 50 ++- .../shell/platform/darwin/ios/BUILD.gn | 10 + .../framework/Source/FlutterDartProject.mm | 33 ++ .../framework/Source/FlutterViewController.mm | 1 + .../shell/platform/darwin/macos/BUILD.gn | 2 + .../macos/framework/Source/FlutterEngine.mm | 58 ++- .../src/flutter/shell/platform/linux/BUILD.gn | 3 + .../flutter/shell/platform/linux/fl_engine.cc | 20 +- .../shell/platform/linux/fl_shorebird.cc | 56 +++ .../shell/platform/linux/fl_shorebird.h | 13 + .../flutter/shell/platform/windows/BUILD.gn | 8 + .../platform/windows/flutter_project_bundle.h | 4 + .../platform/windows/flutter_windows.dll.def | 17 + .../windows/flutter_windows_engine.cc | 150 ++++++++ engine/src/flutter/shell/testing/BUILD.gn | 3 +- .../flutter/sky/tools/create_ios_framework.py | 12 +- engine/src/flutter/testing/run_tests.py | 1 + .../flutter_tools/gradle/build.gradle.kts | 1 + .../flutter_tools/lib/src/base/build.dart | 24 +- .../lib/src/build_system/targets/assets.dart | 14 + .../lib/src/build_system/targets/common.dart | 35 ++ .../lib/src/build_system/targets/ios.dart | 6 + .../lib/src/build_system/targets/macos.dart | 6 + packages/flutter_tools/lib/src/cache.dart | 5 + .../lib/src/ios/application_package.dart | 12 + .../lib/src/shorebird/shorebird_yaml.dart | 64 ++++ packages/flutter_tools/lib/src/version.dart | 13 + .../lib/src/windows/build_windows.dart | 1 + .../hermetic/build_macos_test.dart | 41 +++ .../hermetic/build_windows_test.dart | 126 +++++-- .../build_system/targets/macos_test.dart | 182 ++++++++-- .../test/general.shard/cache_test.dart | 197 ++++++---- .../shorebird/shorebird_yaml_test.dart | 101 ++++++ packages/shorebird_tests/.gitignore | 3 + packages/shorebird_tests/README.md | 2 + .../shorebird_tests/analysis_options.yaml | 2 + packages/shorebird_tests/pubspec.yaml | 16 + .../shorebird_tests/test/android_test.dart | 92 +++++ packages/shorebird_tests/test/base_test.dart | 15 + packages/shorebird_tests/test/ios_test.dart | 91 +++++ .../shorebird_tests/test/shorebird_tests.dart | 281 +++++++++++++++ 66 files changed, 2898 insertions(+), 179 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/shorebird_ci.yml create mode 100644 engine/src/flutter/shell/common/shorebird/BUILD.gn create mode 100644 engine/src/flutter/shell/common/shorebird/shorebird.cc create mode 100644 engine/src/flutter/shell/common/shorebird/shorebird.h create mode 100644 engine/src/flutter/shell/common/shorebird/shorebird_unittests.cc create mode 100644 engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc create mode 100644 engine/src/flutter/shell/common/shorebird/snapshots_data_handle.h create mode 100644 engine/src/flutter/shell/common/shorebird/snapshots_data_handle_unittests.cc create mode 100644 engine/src/flutter/shell/platform/linux/fl_shorebird.cc create mode 100644 engine/src/flutter/shell/platform/linux/fl_shorebird.h create mode 100644 engine/src/flutter/shell/platform/windows/flutter_windows.dll.def create mode 100644 packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart create mode 100644 packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart create mode 100644 packages/shorebird_tests/.gitignore create mode 100644 packages/shorebird_tests/README.md create mode 100644 packages/shorebird_tests/analysis_options.yaml create mode 100644 packages/shorebird_tests/pubspec.yaml create mode 100644 packages/shorebird_tests/test/android_test.dart create mode 100644 packages/shorebird_tests/test/base_test.dart create mode 100644 packages/shorebird_tests/test/ios_test.dart create mode 100644 packages/shorebird_tests/test/shorebird_tests.dart diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000000..ade159ddabcb9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,43 @@ +name: ci + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + push: + branches: + - main + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + runs-on: ${{ matrix.os }} + + name: ๐Ÿงช Test + + env: + FLUTTER_STORAGE_BASE_URL: https://download.shorebird.dev + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + with: + # Fetch all branches and tags to ensure that Flutter can determine its version + fetch-depth: 0 + + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - name: ๐Ÿ“ฆ Install Dependencies + run: | + dart pub get -C ./dev/bots + dart pub get -C ./dev/tools + + - name: ๐Ÿงช Run Tests + run: dart ./dev/bots/test.dart diff --git a/.github/workflows/shorebird_ci.yml b/.github/workflows/shorebird_ci.yml new file mode 100644 index 0000000000000..ec147b4489255 --- /dev/null +++ b/.github/workflows/shorebird_ci.yml @@ -0,0 +1,60 @@ +name: shorebird_ci + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + push: + branches: + - shorebird/dev + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + runs-on: ${{ matrix.os }} + + name: ๐Ÿฆ Shorebird Test + + # TODO(eseidel): This is also set inside shorebird_tests, unclear if + # if it's needed here as well. + env: + FLUTTER_STORAGE_BASE_URL: https://download.shorebird.dev + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + with: + # Fetch all branches and tags to ensure that Flutter can determine its version + fetch-depth: 0 + + # TODO(eseidel): shorebird_tests seems to assume flutter is available + # yet it doesn't seem to set it up here? + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - uses: actions/setup-java@v4 + with: + distribution: "zulu" + java-version: "17" + + - name: ๐Ÿฆ Run Flutter Tools Tests + # TODO(eseidel): Find a nice way to run this on windows. + if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} + # TODO(eseidel): We can't run all flutter_tools tests until we make + # our changes not throw exceptions on missing shorebird.yaml. + # https://github.com/shorebirdtech/shorebird/issues/2392 + run: ../../bin/flutter test test/general.shard/shorebird + working-directory: packages/flutter_tools + + - name: ๐Ÿฆ Run Shorebird Tests + # TODO(felangel): These tests have a dependency on pkg:flutter_flavorizr which + # requires XCode -- therefore they don't work on Windows. + if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} + run: dart test + working-directory: packages/shorebird_tests diff --git a/DEPS b/DEPS index 0139ff7ebda07..8f74948904f13 100644 --- a/DEPS +++ b/DEPS @@ -16,6 +16,10 @@ vars = { 'skia_git': 'https://skia.googlesource.com', 'llvm_git': 'https://llvm.googlesource.com', 'skia_revision': '837be28dd21818d2169e1e6789e89e771a55f01d', + "dart_sdk_revision": "b65ce89c8057d6880e00693a7b0ecd7b9e5f61ca", + "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", + "updater_git": "https://github.com/shorebirdtech/updater.git", + "updater_rev": "76f005940db57c38b479cee858abc0cfbd12ac28", # WARNING: DO NOT EDIT canvaskit_cipd_instance MANUALLY # See `lib/web_ui/README.md` for how to roll CanvasKit to a new version. @@ -294,7 +298,7 @@ deps = { # Var('flutter_git') + '/third_party/protobuf-gn' + '@' + Var('dart_protobuf_gn_rev'), 'engine/src/flutter/third_party/dart': - Var('dart_git') + '/sdk.git' + '@' + Var('dart_revision'), + Var('dart_sdk_git') + '@' + Var('dart_sdk_revision'), # WARNING: Unused Dart dependencies in the list below till "WARNING:" marker are removed automatically - see create_updated_flutter_deps.py. @@ -491,6 +495,9 @@ deps = { 'engine/src/flutter/third_party/ocmock': Var('flutter_git') + '/third_party/ocmock' + '@' + Var('ocmock_rev'), + 'engine/src/flutter/third_party/updater': + Var('updater_git') + '@' + Var('updater_rev'), + 'engine/src/flutter/third_party/libjpeg-turbo/src': Var('flutter_git') + '/third_party/libjpeg-turbo' + '@' + '0fb821f3b2e570b2783a94ccd9a2fb1f4916ae9f', diff --git a/engine/src/build/config/compiler/BUILD.gn b/engine/src/build/config/compiler/BUILD.gn index 2d5e0e0ccbcef..70ebcf699bbbb 100644 --- a/engine/src/build/config/compiler/BUILD.gn +++ b/engine/src/build/config/compiler/BUILD.gn @@ -443,7 +443,10 @@ config("compiler") { # Example PR: https://github.com/dart-lang/native/pull/1615 ldflags += [ "-Wl,--no-undefined", - "-Wl,--exclude-libs,ALL", + + # TODO: Terrible hack, but otherwise libupdater.a symbols can't + # be exported from libflutter.so, even when added to android_exports.lst. + # "-Wl,--exclude-libs,ALL", # Enable identical code folding to reduce size. "-Wl,--icf=all", @@ -646,6 +649,8 @@ config("runtime_library") { ldflags += [ "-Wl,--warn-shared-textrel" ] libs += [ + # Rust requires libunwind. + "unwind", "c", "dl", "m", @@ -660,6 +665,9 @@ config("runtime_library") { } else if (current_cpu == "x86") { current_android_cpu = "i686" } + # libunwind.a is located in the respective android cpu subdirectories. + # The clang version needs to match the version in the lib_dirs line above. + lib_dirs += [ "${android_toolchain_root}/lib/clang/18/lib/linux/${current_android_cpu}/" ] libs += [ "clang_rt.builtins-${current_android_cpu}-android" ] } diff --git a/engine/src/build/toolchain/win/BUILD.gn b/engine/src/build/toolchain/win/BUILD.gn index 45a98b1ecd64b..b5073fb22277b 100644 --- a/engine/src/build/toolchain/win/BUILD.gn +++ b/engine/src/build/toolchain/win/BUILD.gn @@ -204,8 +204,11 @@ template("msvc_toolchain") { expname = "${dllname}.exp" pdbname = "${dllname}.pdb" rspfile = "${dllname}.rsp" + # .def files are used to export symbols from the DLL. This arg will be + # removed by the python tool wrapper if the .def file doesn't exist. + deffile = "${dllname}.def" - link_command = "\"$python_path\" $tool_wrapper_path link-wrapper $env False link.exe /nologo /IMPLIB:$libname /DLL /OUT:$dllname /PDB:${dllname}.pdb @$rspfile" + link_command = "\"$python_path\" $tool_wrapper_path link-wrapper $env False link.exe /nologo /IMPLIB:$libname /DLL /OUT:$dllname /PDB:${dllname}.pdb /DEF:$deffile @$rspfile" # TODO(brettw) support manifests #manifest_command = "\"$python_path\" $tool_wrapper_path manifest-wrapper $env mt.exe -nologo -manifest $manifests -out:${dllname}.manifest" diff --git a/engine/src/build/toolchain/win/tool_wrapper.py b/engine/src/build/toolchain/win/tool_wrapper.py index b4fc8485ffa17..7866c6122d832 100644 --- a/engine/src/build/toolchain/win/tool_wrapper.py +++ b/engine/src/build/toolchain/win/tool_wrapper.py @@ -121,6 +121,17 @@ def ExecLinkWrapper(self, arch, use_separate_mspdbsrv, *args): self._UseSeparateMspdbsrv(env, args) if sys.platform == 'win32': args = list(args) # *args is a tuple by default, which is read-only. + + # Remove the /DEF arg if not provided. We would ideally be able to do this + # in build\toolchain\win\BUILD.gn, but there doesn't seem to be a way to + # conditionally add args to the command line based on whether a file exists + # or not, so we do it here instead. + def_arg_prefix = "/DEF:" + for arg in args: + if arg.startswith(def_arg_prefix): + def_file = arg[len(def_arg_prefix):] + if not os.path.exists(def_file): + args.remove(arg) args[0] = args[0].replace('/', '\\') # https://docs.python.org/2/library/subprocess.html: # "On Unix with shell=True [...] if args is a sequence, the first item diff --git a/engine/src/flutter/BUILD.gn b/engine/src/flutter/BUILD.gn index ec7b55d24f152..c8cbc6803dbe9 100644 --- a/engine/src/flutter/BUILD.gn +++ b/engine/src/flutter/BUILD.gn @@ -118,6 +118,9 @@ group("flutter") { # path_ops "//flutter/tools/path_ops", + + # Built alongside gen_snapshot arm64 targets. + "$dart_src/runtime/bin:analyze_snapshot", ] if (host_os == "linux" || host_os == "mac") { @@ -127,13 +130,6 @@ group("flutter") { ] } - if (host_os == "linux") { - public_deps += [ - # Built alongside gen_snapshot for 64 bit targets - "$dart_src/runtime/bin:analyze_snapshot", - ] - } - if (full_dart_sdk) { public_deps += [ "//flutter/web_sdk" ] } @@ -214,6 +210,7 @@ group("unittests") { "//flutter/runtime:runtime_unittests", "//flutter/shell/common:shell_unittests", "//flutter/shell/geometry:geometry_unittests", + "//flutter/shell/common/shorebird:shorebird_unittests", "//flutter/shell/platform/embedder:embedder_a11y_unittests", "//flutter/shell/platform/embedder:embedder_proctable_unittests", "//flutter/shell/platform/embedder:embedder_unittests", @@ -344,3 +341,19 @@ if (host_os == "win") { outputs = [ "$root_build_dir/gen_snapshot/gen_snapshot.exe" ] } } + +# A top-level target for analyze_snapshot, modeled after the gen_snapshot +# target above. +if (host_os == "win") { + _analyze_snapshot_target = + "$dart_src/runtime/bin:analyze_snapshot($host_toolchain)" + + copy("analyze_snapshot") { + deps = [ _analyze_snapshot_target ] + + analyze_snapshot_out_dir = + get_label_info(_analyze_snapshot_target, "root_out_dir") + sources = [ "$analyze_snapshot_out_dir/analyze_snapshot.exe" ] + outputs = [ "$root_build_dir/analyze_snapshot/analyze_snapshot.exe" ] + } +} diff --git a/engine/src/flutter/build/archives/BUILD.gn b/engine/src/flutter/build/archives/BUILD.gn index 4f902010d21dc..55700433541db 100644 --- a/engine/src/flutter/build/archives/BUILD.gn +++ b/engine/src/flutter/build/archives/BUILD.gn @@ -45,6 +45,7 @@ generated_file("artifacts_entitlement_config") { if (build_engine_artifacts) { zip_bundle("artifacts") { deps = [ + "$dart_src/runtime/bin:analyze_snapshot", "$dart_src/runtime/bin:gen_snapshot", "//flutter/flutter_frontend_server:frontend_server", "//flutter/impeller/compiler:impellerc", @@ -142,6 +143,10 @@ if (build_engine_artifacts) { if (host_os == "mac") { deps += [ ":artifacts_entitlement_config" ] files += [ + { + source = "$root_out_dir/analyze_snapshot$exe" + destination = "analyze_snapshot$exe" + }, { source = "$target_gen_dir/entitlements.txt" destination = "entitlements.txt" @@ -320,14 +325,25 @@ if (is_mac) { } if (host_os == "win") { + # This rule archives both gen_snapshot *and* analyze_snapshot. The name is + # misleading. We (shorebird) have updated this rule to include + # analyze_snapshot but did not update the name because it is referenced + # elsewhere in the tooling. zip_bundle("archive_win_gen_snapshot") { - deps = [ "//flutter:gen_snapshot" ] + deps = [ + "//flutter:analyze_snapshot", + "//flutter:gen_snapshot", + ] output = "$full_target_platform_name-$flutter_runtime_mode/windows-x64.zip" files = [ { source = "$root_out_dir/gen_snapshot/gen_snapshot.exe" destination = "gen_snapshot.exe" }, + { + source = "$root_out_dir/analyze_snapshot/analyze_snapshot.exe" + destination = "analyze_snapshot.exe" + }, ] } } diff --git a/engine/src/flutter/build/dart/tools/dart_pkg.py b/engine/src/flutter/build/dart/tools/dart_pkg.py index c60e3e02431e4..5b3e0a8b5f3ee 100755 --- a/engine/src/flutter/build/dart/tools/dart_pkg.py +++ b/engine/src/flutter/build/dart/tools/dart_pkg.py @@ -163,7 +163,10 @@ def main(): for source in args.package_sources: relative_source = os.path.relpath(source, common_source_prefix) target = os.path.join(target_dir, relative_source) - copy(source, target) + try: + copy(source, target) + except shutil.SameFileError: + pass # Copy sdk-ext sources into pkg directory sdk_ext_dir = os.path.join(target_dir, 'sdk_ext') @@ -179,7 +182,10 @@ def main(): for source in args.sdk_ext_files: relative_source = os.path.relpath(source, common_source_prefix) target = os.path.join(sdk_ext_dir, relative_source) - copy(source, target) + try: + copy(source, target) + except shutil.SameFileError: + pass # Write stamp file. with open(args.stamp_file, 'w'): diff --git a/engine/src/flutter/common/config.gni b/engine/src/flutter/common/config.gni index 318d305bd37fc..35ec1c7e058dc 100644 --- a/engine/src/flutter/common/config.gni +++ b/engine/src/flutter/common/config.gni @@ -67,6 +67,14 @@ if (slimpeller) { feature_defines_list += [ "SLIMPELLER=1" ] } +if (is_android || is_ios || is_linux || is_mac || is_win) { + feature_defines_list += [ "SHOREBIRD_PLATFORM_SUPPORTED=1" ] +} + +if (is_ios) { + feature_defines_list += [ "SHOREBIRD_USE_INTERPRETER=1" ] +} + if (is_ios || is_mac) { flutter_cflags_objc = [ "-Werror=overriding-method-mismatch", diff --git a/engine/src/flutter/lib/snapshot/BUILD.gn b/engine/src/flutter/lib/snapshot/BUILD.gn index e4b52cac21985..e495383162a89 100644 --- a/engine/src/flutter/lib/snapshot/BUILD.gn +++ b/engine/src/flutter/lib/snapshot/BUILD.gn @@ -35,7 +35,10 @@ group("generate_snapshot_bins") { if (host_os == "mac" && (target_os == "mac" || target_os == "ios" || target_os == "android")) { # For macOS target builds: needed for both target CPUs (arm64, x64). - public_deps += [ ":create_macos_gen_snapshots" ] + public_deps += [ + ":create_macos_analyze_snapshots", + ":create_macos_gen_snapshots", + ] } else if (host_os == "mac" && (target_cpu == "arm" || target_cpu == "arm64")) { # For iOS, Android target builds: all AOT target CPUs are arm/arm64. @@ -46,9 +49,11 @@ group("generate_snapshot_bins") { public_deps = [ "$dart_src/runtime/bin:gen_snapshot($host_toolchain)" ] } - # Build analyze_snapshot for 64-bit target CPUs. - if (host_os == "linux" && (target_cpu == "x64" || target_cpu == "arm64" || - target_cpu == "riscv64")) { + # Build analyze_snapshot for 64-bit target CPUs on linux. + # Or always targeting arm64 for Shorebird builds. + if ((host_os == "linux" && + (target_cpu == "x64" || target_cpu == "riscv64")) || + target_cpu == "arm64") { public_deps += [ "$dart_src/runtime/bin:analyze_snapshot($host_toolchain)" ] } } @@ -257,6 +262,69 @@ if (host_os == "mac" && ":create_macos_gen_snapshot_x64${gen_snapshot_suffix}", ] } + + # Added by shorebird. + # analyze_snapshot targets below were copied from the gen_snapshot targets + # above to allow us to include analyze_snapshot in the artifacts generated + # for create_ios_framework.py. + template("build_mac_analyze_snapshot") { + assert(defined(invoker.host_arch)) + host_cpu = invoker.host_arch + + build_toolchain = "//build/toolchain/mac:clang_$host_cpu" + analyze_snapshot_target_name = "analyze_snapshot" + + # At this point, the gen_snapshot equivalent changes + # gen_ snapshot_target_name to "gen_snapshot_host_targeting_host". There is + # no equivalent for analyze_snapshot, so we don't do that here. + # + # It's unclear whether we need to do so now, but we didn't previously, so + # we're not doing it now until we have a reason to. + + analyze_snapshot_target = + "$dart_src/runtime/bin:$analyze_snapshot_target_name($build_toolchain)" + + copy(target_name) { + # The toolchain-specific output directory. For cross-compiles, this is a + # clang-x64 or clang-arm64 subdirectory of the top-level build directory. + output_dir = get_label_info(analyze_snapshot_target, "root_out_dir") + + sources = [ "${output_dir}/${analyze_snapshot_target_name}" ] + outputs = [ + "${root_out_dir}/artifacts_$host_cpu/analyze_snapshot_${target_cpu}", + ] + deps = [ analyze_snapshot_target ] + } + } + + build_mac_analyze_snapshot( + "create_macos_analyze_snapshot_arm64_${target_cpu}") { + host_arch = "arm64" + } + + build_mac_analyze_snapshot( + "create_macos_analyze_snapshot_x64_${target_cpu}") { + host_arch = "x64" + } + + action("create_macos_analyze_snapshots") { + script = "//flutter/sky/tools/create_macos_binary.py" + outputs = [ "${root_out_dir}/analyze_snapshot_${target_cpu}" ] + args = [ + "--in-arm64", + rebase_path( + "${root_out_dir}/artifacts_arm64/analyze_snapshot_${target_cpu}"), + "--in-x64", + rebase_path( + "${root_out_dir}/artifacts_x64/analyze_snapshot_${target_cpu}"), + "--out", + rebase_path("${root_out_dir}/analyze_snapshot_${target_cpu}"), + ] + deps = [ + ":create_macos_analyze_snapshot_arm64_${target_cpu}", + ":create_macos_analyze_snapshot_x64_${target_cpu}", + ] + } } source_set("snapshot") { diff --git a/engine/src/flutter/runtime/dart_snapshot.cc b/engine/src/flutter/runtime/dart_snapshot.cc index 198a2e75a7edc..9ab19ef4b7a67 100644 --- a/engine/src/flutter/runtime/dart_snapshot.cc +++ b/engine/src/flutter/runtime/dart_snapshot.cc @@ -56,33 +56,93 @@ static std::shared_ptr SearchMapping( const std::vector& native_library_paths, const char* native_library_symbol_name, bool is_executable) { - // Ask the embedder. There is no fallback as we expect the embedders (via - // their embedding APIs) to just specify the mappings directly. - if (embedder_mapping_callback) { - // Note that mapping will be nullptr if the mapping callback returns an - // invalid mapping. If all the other methods for resolving the data also - // fail, the engine will stop with accompanying error logs. - if (auto mapping = embedder_mapping_callback()) { - return mapping; +#if SHOREBIRD_USE_INTERPRETER + // Detect when we're trying to load a Shorebird patch. + auto patch_path = native_library_path.front(); + bool is_patch = patch_path.find(".vmcode") != std::string::npos; + if (is_patch) { + // We use this terrible hack to load in the patch and then extract the + // symbols from it when the path is not App.framework/App but rather + // foo.vmcode, etc. We read the symbols into static variables, but then I + // believe we need to hold onto the ELF itself, otherwise the symbols + // become invalid. + // "leaked_elf" is meant to indicate that we're not freeing the ELF. + static Dart_LoadedElf* leaked_elf = nullptr; + // The VM Snapshot is identical for all binaries produced by a given version + // of Dart. Our linker checks this and will fail to link if ever the VM + // snapshot changes. + const uint8_t* ignored_vm_data = nullptr; + const uint8_t* ignored_vm_instrs = nullptr; + static const uint8_t* isolate_data = nullptr; + static const uint8_t* isolate_instrs = nullptr; + if (leaked_elf == nullptr) { + const char* error = nullptr; + // vmcode files are elf files prefixed with a shorebird linker header. + auto elf_mapping = GetFileMapping(patch_path, false /* executable */); + int elf_file_offset = Shorebird_ReadLinkHeader(elf_mapping->GetMapping(), + elf_mapping->GetSize()); + + leaked_elf = Dart_LoadELF(patch_path.c_str(), elf_file_offset, &error, + &ignored_vm_data, &ignored_vm_instrs, + &isolate_data, &isolate_instrs, + /* load as read-only, not rx */ false); + if (leaked_elf != nullptr) { + FML_LOG(INFO) << "Loaded ELF"; + } else { + FML_LOG(FATAL) << "Failed to load ELF at " << patch_path + << " error: " << error; + abort(); + } } - } - // Attempt to open file at path specified. - if (!file_path.empty()) { - if (auto file_mapping = GetFileMapping(file_path, is_executable)) { - return file_mapping; + FML_LOG(INFO) << "Loading symbol from ELF " << native_library_symbol_name; + + if (native_library_symbol_name == DartSnapshot::kIsolateDataSymbol) { + return std::make_unique(isolate_data, 0, + nullptr, true); + } else if (native_library_symbol_name == + DartSnapshot::kIsolateInstructionsSymbol) { + return std::make_unique(isolate_instrs, 0, + nullptr, true); + } + // Fall through to normal lookups for VM data and instructions. + // This fallthrough depends on the fact that NativeLibrary below can't + // read the ELF out of our .vmcode files. + } else { + // Only try to open the file if we're not loading a patch. +#endif + + // Ask the embedder. There is no fallback as we expect the embedders (via + // their embedding APIs) to just specify the mappings directly. + if (embedder_mapping_callback) { + // Note that mapping will be nullptr if the mapping callback returns an + // invalid mapping. If all the other methods for resolving the data also + // fail, the engine will stop with accompanying error logs. + if (auto mapping = embedder_mapping_callback()) { + return mapping; + } } - } - // Look in application specified native library if specified. - for (const std::string& path : native_library_paths) { - auto native_library = fml::NativeLibrary::Create(path.c_str()); - auto symbol_mapping = std::make_unique( - native_library, native_library_symbol_name); - if (symbol_mapping->GetMapping() != nullptr) { - return symbol_mapping; + // Attempt to open file at path specified. + if (!file_path.empty()) { + if (auto file_mapping = GetFileMapping(file_path, is_executable)) { + return file_mapping; + } } - } + + // Look in application specified native library if specified. + for (const std::string& path : native_library_paths) { + auto native_library = fml::NativeLibrary::Create(path.c_str()); + auto symbol_mapping = std::make_unique( + native_library, native_library_symbol_name); + if (symbol_mapping->GetMapping() != nullptr) { + return symbol_mapping; + } + } + +#if SHOREBIRD_USE_INTERPRETER + } // !is_patch +#endif // Look inside the currently loaded process. { diff --git a/engine/src/flutter/shell/common/BUILD.gn b/engine/src/flutter/shell/common/BUILD.gn index e1aba738f17be..bbe42f5a987f8 100644 --- a/engine/src/flutter/shell/common/BUILD.gn +++ b/engine/src/flutter/shell/common/BUILD.gn @@ -158,6 +158,8 @@ source_set("common") { "//flutter/skia", ] + include_dirs = [ "//flutter/updater" ] + if (impeller_supports_rendering) { sources += [ "snapshot_controller_impeller.cc", @@ -166,6 +168,31 @@ source_set("common") { deps += [ "//flutter/impeller" ] } + + # Needed to compile flutter_tester for macOS. + if (host_os == "mac" && target_os == "mac") { + if (target_cpu == "arm64") { + libs = [ "//flutter/third_party/updater/target/aarch64-apple-darwin/release/libupdater.a" ] + } else if (target_cpu == "x64") { + libs = [ "//flutter/third_party/updater/target/x86_64-apple-darwin/release/libupdater.a" ] + } + } + + # Needed to compile flutter_tester for Windows. + if (host_os == "win" && target_os == "win") { + if (target_cpu == "x64") { + libs = [ + "userenv.lib", + "//flutter/third_party/updater/target/x86_64-pc-windows-msvc/release/updater.lib", + ] + } + } + + if (host_os == "linux" && target_os == "linux") { + if (target_cpu == "x64") { + libs = [ "//flutter/third_party/updater/target/x86_64-unknown-linux-gnu/release/libupdater.a" ] + } + } } # These are in their own source_set to avoid a dependency cycle with //common/graphics diff --git a/engine/src/flutter/shell/common/shell.cc b/engine/src/flutter/shell/common/shell.cc index 762663b061d1c..fac4ec6336fbe 100644 --- a/engine/src/flutter/shell/common/shell.cc +++ b/engine/src/flutter/shell/common/shell.cc @@ -47,6 +47,8 @@ #include "third_party/skia/include/core/SkGraphics.h" #include "third_party/tonic/common/log.h" +#include "third_party/updater/library/include/updater.h" + namespace flutter { constexpr char kSkiaChannel[] = "flutter/skia"; @@ -522,6 +524,14 @@ Shell::Shell(DartVMRef vm, is_gpu_disabled_sync_switch_(new fml::SyncSwitch(is_gpu_disabled)), weak_factory_gpu_(nullptr), weak_factory_(this) { + // FIXME: This is probably the wrong place to hook into. +#if SHOREBIRD_PLATFORM_SUPPORTED + if (!vm_) { + shorebird_report_launch_failure(); + } else { + shorebird_report_launch_success(); + } +#endif FML_CHECK(!settings.enable_software_rendering || !settings.enable_impeller) << "Software rendering is incompatible with Impeller."; if (!settings.enable_impeller && settings.warn_on_impeller_opt_out) { diff --git a/engine/src/flutter/shell/common/shorebird/BUILD.gn b/engine/src/flutter/shell/common/shorebird/BUILD.gn new file mode 100644 index 0000000000000..2c7def6991542 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/BUILD.gn @@ -0,0 +1,60 @@ +import("//flutter/common/config.gni") +import("//flutter/testing/testing.gni") + +source_set("snapshots_data_handle") { + sources = [ + "snapshots_data_handle.cc", + "snapshots_data_handle.h", + ] + + deps = [ + "//flutter/fml", + "//flutter/runtime", + "//flutter/runtime:libdart", + "//flutter/shell/common", + ] +} + +source_set("shorebird") { + sources = [ + "shorebird.cc", + "shorebird.h", + ] + + deps = [ + ":snapshots_data_handle", + "//flutter/fml", + "//flutter/runtime", + "//flutter/runtime:libdart", + "//flutter/shell/common", + "//flutter/shell/platform/embedder:embedder_headers", + ] + + include_dirs = [ "//flutter/updater" ] +} + +if (enable_unittests) { + test_fixtures("shorebird_fixtures") { + fixtures = [] + } + + executable("shorebird_unittests") { + testonly = true + + sources = [ + "shorebird_unittests.cc", + "snapshots_data_handle_unittests.cc", + ] + + # This only includes snapshots_data_handle and not shorebird because + # shorebird fails to link due to a missing updater lib. + deps = [ + ":shorebird", + ":shorebird_fixtures", + ":snapshots_data_handle", + "//flutter/runtime", + "//flutter/testing", + "//flutter/testing:fixture_test", + ] + } +} diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc new file mode 100644 index 0000000000000..a2b2f382820a1 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -0,0 +1,335 @@ + +#include "flutter/shell/common/shorebird/shorebird.h" + +#include +#include +#include +#include +#include + +#include "flutter/fml/command_line.h" +#include "flutter/fml/file.h" +#include "flutter/fml/macros.h" +#include "flutter/fml/mapping.h" +#include "flutter/fml/message_loop.h" +#include "flutter/fml/native_library.h" +#include "flutter/fml/paths.h" +#include "flutter/lib/ui/plugins/callback_cache.h" +#include "flutter/runtime/dart_snapshot.h" +#include "flutter/runtime/dart_vm.h" +#include "flutter/shell/common/shell.h" +#include "flutter/shell/common/shorebird/snapshots_data_handle.h" +#include "flutter/shell/common/switches.h" +#include "fml/logging.h" +#include "shell/platform/embedder/embedder.h" +#include "third_party/dart/runtime/include/dart_tools_api.h" + +#include "third_party/updater/library/include/updater.h" + +// Namespaced to avoid Google style warnings. +namespace flutter { + +// Old Android versions (e.g. the v16 ndk Flutter uses) don't always include a +// getauxval symbol, but the Rust ring crate assumes it exists: +// https://github.com/briansmith/ring/blob/fa25bf3a7403c9fe6458cb87bd8427be41225ca2/src/cpu/arm.rs#L22 +// It uses it to determine if the CPU supports AES instructions. +// Making this a weak symbol allows the linker to use a real version instead +// if it can find one. +// BoringSSL just reads from procfs instead, which is what we would do if +// we needed to implement this ourselves. Implementation looks straightforward: +// https://lwn.net/Articles/519085/ +// https://github.com/google/boringssl/blob/6ab4f0ae7f2db96d240eb61a5a8b4724e5a09b2f/crypto/cpu_arm_linux.c +#if defined(__ANDROID__) && defined(__arm__) +extern "C" __attribute__((weak)) unsigned long getauxval(unsigned long type) { + return 0; +} +#endif + +// TODO(eseidel): I believe we need to leak these or we'll sometimes crash +// when using the base snapshot in mixed mode. This likely will not play +// nicely with multi-engine support and will need to be refactored. +static fml::RefPtr vm_snapshot; +static fml::RefPtr isolate_snapshot; + +void SetBaseSnapshot(Settings& settings) { + // These mappings happen to be to static data in the App.framework, but + // we still need to seem to hold onto the DartSnapshot objects to keep + // the mappings alive. + vm_snapshot = DartSnapshot::VMSnapshotFromSettings(settings); + isolate_snapshot = DartSnapshot::IsolateSnapshotFromSettings(settings); + Shorebird_SetBaseSnapshots(isolate_snapshot->GetDataMapping(), + isolate_snapshot->GetInstructionsMapping(), + vm_snapshot->GetDataMapping(), + vm_snapshot->GetInstructionsMapping()); +} + +class FileCallbacksImpl { + public: + static void* Open(); + static uintptr_t Read(void* file, uint8_t* buffer, uintptr_t length); + static int64_t Seek(void* file, int64_t offset, int32_t whence); + static void Close(void* file); +}; + +FileCallbacks ShorebirdFileCallbacks() { + return { + .open = FileCallbacksImpl::Open, + .read = FileCallbacksImpl::Read, + .seek = FileCallbacksImpl::Seek, + .close = FileCallbacksImpl::Close, + }; +} + +// Given the contents of a yaml file, return the given value if it exists, +// otherwise return an empty string. +// Does not support nested keys. +std::string GetValueFromYaml(const std::string& yaml, const std::string& key) { + std::stringstream ss(yaml); + std::string line; + std::string prefix = key + ":"; + while (std::getline(ss, line, '\n')) { + if (line.find(prefix) != std::string::npos) { + auto ret = line.substr(line.find(prefix) + prefix.size()); + + // Remove leading and trailing spaces + while (!ret.empty() && std::isspace(ret.front())) { + ret.erase(0, 1); + } + while (!ret.empty() && std::isspace(ret.back())) { + ret.pop_back(); + } + return ret; + } + } + return ""; +} + +// FIXME: consolidate this with the other ConfigureShorebird +bool ConfigureShorebird(const ShorebirdConfigArgs& args, + std::string& patch_path) { + patch_path = fml::PathToUtf8(args.release_app_library_path); + auto shorebird_updater_dir_name = "shorebird_updater"; + + // Parse app id from shorebird.yaml + std::string app_id = GetValueFromYaml(args.shorebird_yaml, "app_id"); + if (app_id.empty()) { + FML_LOG(ERROR) << "Shorebird updater: appid not found in shorebird.yaml"; + return false; + } + + auto code_cache_dir = fml::paths::JoinPaths( + {std::move(args.code_cache_path), shorebird_updater_dir_name, app_id}); + auto app_storage_dir = fml::paths::JoinPaths( + {std::move(args.app_storage_path), shorebird_updater_dir_name, app_id}); + + fml::CreateDirectory(fml::paths::GetCachesDirectory(), + {shorebird_updater_dir_name}, + fml::FilePermission::kReadWrite); + + bool init_result; + // Using a block to make AppParameters lifetime explicit. + { + AppParameters app_parameters; + // Combine version and version_code into a single string. + // We could also pass these separately through to the updater if needed. + auto release_version = args.release_version.version; + if (!args.release_version.build_number.empty()) { + release_version += "+" + args.release_version.build_number; + } + + app_parameters.release_version = release_version.c_str(); + app_parameters.code_cache_dir = code_cache_dir.c_str(); + app_parameters.app_storage_dir = app_storage_dir.c_str(); + + // https://stackoverflow.com/questions/26032039/convert-vectorstring-into-char-c + std::vector c_paths{}; + c_paths.push_back(args.release_app_library_path.c_str()); + // Do not modify application_library_path or c_strings will invalidate. + + app_parameters.original_libapp_paths = c_paths.data(); + app_parameters.original_libapp_paths_size = c_paths.size(); + + // shorebird_init copies from app_parameters and shorebirdYaml. + init_result = shorebird_init(&app_parameters, ShorebirdFileCallbacks(), + args.shorebird_yaml.c_str()); + } + + // We've decided not to support synchronous updates on launch for now. + // It's a terrible user experience (having the app hang on launch) and + // instead we will provide examples of how to build a custom update UI + // within Dart, including updating as part of login, etc. + // https://github.com/shorebirdtech/shorebird/issues/950 + + FML_LOG(INFO) << "Checking for active patch"; + shorebird_validate_next_boot_patch(); + char* c_active_path = shorebird_next_boot_patch_path(); + if (c_active_path != NULL) { + patch_path = c_active_path; + shorebird_free_string(c_active_path); + FML_LOG(INFO) << "Shorebird updater: patch path: " << patch_path; + } else { + FML_LOG(INFO) << "Shorebird updater: no active patch."; + } + + // We are careful only to report a launch start in the case where it's the + // first time we've configured shorebird this process. Otherwise we could end + // up in a case where we report a launch start, but never a completion (e.g. + // from package:flutter_work_manager which sometimes creates a FlutterEngine + // (and thus configures shorebird) but never runs it. The proper fix for this + // is probably to move the launch_start() call to be later in the lifecycle + // (when the snapshot is loaded and run, rather than when FlutterEngine is + // initialized). This "hack" will still have a problem where FlutterEngine is + // initialized but never run before the app is quit, could still cause us to + // suddenly mark-bad a patch that was never actually attempted to launch. + if (!init_result) { + return false; + } + + // Once start_update_thread is called, the next_boot_patch* functions may + // change their return values if the shorebird_report_launch_failed + // function is called. + shorebird_report_launch_start(); + + if (shorebird_should_auto_update()) { + FML_LOG(INFO) << "Starting Shorebird update"; + shorebird_start_update_thread(); + } else { + FML_LOG(INFO) + << "Shorebird auto_update disabled, not checking for updates."; + } + + return true; +} + +void ConfigureShorebird(std::string code_cache_path, + std::string app_storage_path, + Settings& settings, + const std::string& shorebird_yaml, + const std::string& version, + const std::string& version_code) { + // If you are crashing here, you probably are running Shorebird in a Debug + // config, where the AOT snapshot won't be linked into the process, and thus + // lookups will fail. Change your Scheme to Release to fix: + // https://github.com/flutter/flutter/wiki/Debugging-the-engine#debugging-ios-builds-with-xcode + FML_CHECK(DartSnapshot::VMSnapshotFromSettings(settings)) + << "XCode Scheme must be set to Release to use Shorebird"; + + auto shorebird_updater_dir_name = "shorebird_updater"; + + auto code_cache_dir = fml::paths::JoinPaths( + {std::move(code_cache_path), shorebird_updater_dir_name}); + auto app_storage_dir = fml::paths::JoinPaths( + {std::move(app_storage_path), shorebird_updater_dir_name}); + + fml::CreateDirectory(fml::paths::GetCachesDirectory(), + {shorebird_updater_dir_name}, + fml::FilePermission::kReadWrite); + + bool init_result; + // Using a block to make AppParameters lifetime explicit. + { + AppParameters app_parameters; + // Combine version and version_code into a single string. + // We could also pass these separately through to the updater if needed. + auto release_version = version + "+" + version_code; + app_parameters.release_version = release_version.c_str(); + app_parameters.code_cache_dir = code_cache_dir.c_str(); + app_parameters.app_storage_dir = app_storage_dir.c_str(); + + // https://stackoverflow.com/questions/26032039/convert-vectorstring-into-char-c + std::vector c_paths{}; + for (const auto& string : settings.application_library_path) { + c_paths.push_back(string.c_str()); + } + // Do not modify application_library_path or c_strings will invalidate. + + app_parameters.original_libapp_paths = c_paths.data(); + app_parameters.original_libapp_paths_size = c_paths.size(); + + // shorebird_init copies from app_parameters and shorebirdYaml. + init_result = shorebird_init(&app_parameters, ShorebirdFileCallbacks(), + shorebird_yaml.c_str()); + } + + // We've decided not to support synchronous updates on launch for now. + // It's a terrible user experience (having the app hang on launch) and + // instead we will provide examples of how to build a custom update UI + // within Dart, including updating as part of login, etc. + // https://github.com/shorebirdtech/shorebird/issues/950 + + // We only set the base snapshot on iOS for now. +#if SHOREBIRD_USE_INTERPRETER + SetBaseSnapshot(settings); +#endif + + shorebird_validate_next_boot_patch(); + char* c_active_path = shorebird_next_boot_patch_path(); + if (c_active_path != NULL) { + std::string active_path = c_active_path; + shorebird_free_string(c_active_path); + FML_LOG(INFO) << "Shorebird updater: active path: " << active_path; + +#if SHOREBIRD_USE_INTERPRETER + // On iOS we add the patch to the front of the list instead of clearing + // the list, to allow dart_snapshot.cc to still find the base snapshot + // for the vm isolate. + settings.application_library_path.insert( + settings.application_library_path.begin(), active_path); +#else + settings.application_library_path.clear(); + settings.application_library_path.emplace_back(active_path); +#endif + } else { + FML_LOG(INFO) << "Shorebird updater: no active patch."; + } + + // We are careful only to report a launch start in the case where it's the + // first time we've configured shorebird this process. Otherwise we could end + // up in a case where we report a launch start, but never a completion (e.g. + // from package:flutter_work_manager which sometimes creates a FlutterEngine + // (and thus configures shorebird) but never runs it. The proper fix for this + // is probably to move the launch_start() call to be later in the lifecycle + // (when the snapshot is loaded and run, rather than when FlutterEngine is + // initialized). This "hack" will still have a problem where FlutterEngine is + // initialized but never run before the app is quit, could still cause us to + // suddenly mark-bad a patch that was never actually attempted to launch. + if (!init_result) { + return; + } + + // Once start_update_thread is called, the next_boot_patch* functions may + // change their return values if the shorebird_report_launch_failed + // function is called. + shorebird_report_launch_start(); + + if (shorebird_should_auto_update()) { + FML_LOG(INFO) << "Starting Shorebird update"; + shorebird_start_update_thread(); + } else { + FML_LOG(INFO) + << "Shorebird auto_update disabled, not checking for updates."; + } +} + +void* FileCallbacksImpl::Open() { + return SnapshotsDataHandle::createForSnapshots(*vm_snapshot, + *isolate_snapshot) + .release(); +} + +uintptr_t FileCallbacksImpl::Read(void* file, + uint8_t* buffer, + uintptr_t length) { + return reinterpret_cast(file)->Read(buffer, length); +} + +int64_t FileCallbacksImpl::Seek(void* file, int64_t offset, int32_t whence) { + // Currently we only support blob handles. + return reinterpret_cast(file)->Seek(offset, whence); +} + +void FileCallbacksImpl::Close(void* file) { + delete reinterpret_cast(file); +} + +} // namespace flutter \ No newline at end of file diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.h b/engine/src/flutter/shell/common/shorebird/shorebird.h new file mode 100644 index 0000000000000..c6873c5014a00 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/shorebird.h @@ -0,0 +1,47 @@ +#ifndef FLUTTER_SHELL_COMMON_SHOREBIRD_SHOREBIRD_H_ +#define FLUTTER_SHELL_COMMON_SHOREBIRD_SHOREBIRD_H_ + +#include "flutter/common/settings.h" +#include "shell/platform/embedder/embedder.h" + +namespace flutter { + +struct ReleaseVersion { + std::string version; + std::string build_number; +}; + +struct ShorebirdConfigArgs { + std::string code_cache_path; + std::string app_storage_path; + std::string release_app_library_path; + std::string shorebird_yaml; + ReleaseVersion release_version; + + ShorebirdConfigArgs(std::string code_cache_path, + std::string app_storage_path, + std::string release_app_library_path, + std::string shorebird_yaml, + ReleaseVersion release_version) + : code_cache_path(code_cache_path), + app_storage_path(app_storage_path), + release_app_library_path(release_app_library_path), + shorebird_yaml(shorebird_yaml), + release_version(release_version) {} +}; + +bool ConfigureShorebird(const ShorebirdConfigArgs& args, + std::string& patch_path); + +void ConfigureShorebird(std::string code_cache_path, + std::string app_storage_path, + Settings& settings, + const std::string& shorebird_yaml, + const std::string& version, + const std::string& version_code); + +std::string GetValueFromYaml(const std::string& yaml, const std::string& key); + +} // namespace flutter + +#endif // FLUTTER_SHELL_COMMON_SHOREBIRD_SHOREBIRD_H_ diff --git a/engine/src/flutter/shell/common/shorebird/shorebird_unittests.cc b/engine/src/flutter/shell/common/shorebird/shorebird_unittests.cc new file mode 100644 index 0000000000000..7c108ac1e6231 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/shorebird_unittests.cc @@ -0,0 +1,21 @@ +#include "flutter/shell/common/shorebird/shorebird.h" + +#include "gtest/gtest.h" + +namespace flutter { +namespace testing { +TEST(Shorebird, GetValueFromYamlValueExists) { + std::string yaml = "appid: com.example.app\nversion: 1.0.0\n"; + std::string key = "appid"; + std::string value = GetValueFromYaml(yaml, key); + EXPECT_EQ(value, "com.example.app"); +} + +TEST(Shorebird, GetValueFromYamlValueDoesNotExist) { + std::string yaml = "appid: com.example.app\nversion: 1.0.0\n"; + std::string key = "appid2"; + std::string value = GetValueFromYaml(yaml, key); + EXPECT_EQ(value, ""); +} +} // namespace testing +} // namespace flutter \ No newline at end of file diff --git a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc new file mode 100644 index 0000000000000..0c6c5a45450a1 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc @@ -0,0 +1,144 @@ +#include "flutter/shell/common/shorebird/snapshots_data_handle.h" + +#include "third_party/dart/runtime/include/dart_native_api.h" + +namespace flutter { + +static std::unique_ptr DataMapping(const DartSnapshot& snapshot) { + auto ptr = snapshot.GetDataMapping(); + return std::make_unique(ptr, + Dart_SnapshotDataSize(ptr)); +} + +static std::unique_ptr InstructionsMapping( + const DartSnapshot& snapshot) { + auto ptr = snapshot.GetInstructionsMapping(); + return std::make_unique(ptr, + Dart_SnapshotInstrSize(ptr)); +} + +// The size of the snapshot data is the sum of the sizes of the blobs. +size_t SnapshotsDataHandle::FullSize() const { + size_t size = 0; + for (const auto& blob : blobs_) { + size += blob->GetSize(); + } + return size; +} + +// The offset into the snapshots data blobs as though they were a single +// contiguous buffer. +size_t SnapshotsDataHandle::AbsoluteOffsetForIndex(BlobsIndex index) { + if (index.blob >= blobs_.size()) { + FML_LOG(WARNING) << "Blob index " << index.blob + << " is larger than the number of blobs (" << blobs_.size() + << "). Returning full size (" << FullSize() << ")"; + return FullSize(); + } + if (index.offset > blobs_[index.blob]->GetSize()) { + FML_LOG(WARNING) << "Offset for blob " << index.blob << " (" << index.offset + << ") is larger than the blob size (" + << blobs_[index.blob]->GetSize() + << "). Returning index start of next blob"; + return AbsoluteOffsetForIndex({index.blob + 1, 0}); + } + size_t offset = 0; + for (size_t i = 0; i < index.blob; i++) { + offset += blobs_[i]->GetSize(); + } + offset += index.offset; + return offset; +} + +BlobsIndex SnapshotsDataHandle::IndexForAbsoluteOffset(int64_t offset, + BlobsIndex start_index) { + size_t start_offset = AbsoluteOffsetForIndex(start_index); + if (offset < 0) { + if ((size_t)abs(offset) > start_offset) { + FML_LOG(WARNING) + << "Offset is before the beginning of SnapshotsData. Returning 0, 0"; + return {0, 0}; + } + } else if (offset + start_offset >= FullSize()) { + FML_LOG(WARNING) << "Target offset is past the end of SnapshotsData (" + << offset + start_offset << ", blobs size:" << FullSize() + << "). Returning last blob index and offset"; + return {blobs_.size(), blobs_.back()->GetSize()}; + } + + size_t dest_offset = start_offset + offset; + BlobsIndex index = {0, 0}; + for (const auto& blob : blobs_) { + if (dest_offset < blob->GetSize()) { + // The remaining offset is within this blob. + index.offset = dest_offset; + break; + } + + index.blob++; + dest_offset -= blob->GetSize(); + } + return index; +} + +std::unique_ptr SnapshotsDataHandle::createForSnapshots( + const DartSnapshot& vm_snapshot, + const DartSnapshot& isolate_snapshot) { + // This needs to match the order in which the blobs are written out in + // analyze_snapshot --dump_blobs + std::vector> blobs; + blobs.push_back(DataMapping(vm_snapshot)); + blobs.push_back(DataMapping(isolate_snapshot)); + blobs.push_back(InstructionsMapping(vm_snapshot)); + blobs.push_back(InstructionsMapping(isolate_snapshot)); + return std::make_unique(std::move(blobs)); +} + +uintptr_t SnapshotsDataHandle::Read(uint8_t* buffer, uintptr_t length) { + uintptr_t bytes_read = 0; + // Copy current blob from current offset and possibly into the next blob + // until we have read length bytes. + while (bytes_read < length) { + if (current_index_.blob >= blobs_.size()) { + // We have read all blobs. + break; + } + intptr_t remaining_blob_length = + blobs_[current_index_.blob]->GetSize() - current_index_.offset; + if (remaining_blob_length <= 0) { + // We have read all bytes in this blob. + current_index_.blob++; + current_index_.offset = 0; + continue; + } + intptr_t bytes_to_read = fmin(length - bytes_read, remaining_blob_length); + memcpy(buffer + bytes_read, + blobs_[current_index_.blob]->GetMapping() + current_index_.offset, + bytes_to_read); + bytes_read += bytes_to_read; + current_index_.offset += bytes_to_read; + } + + return bytes_read; +} + +int64_t SnapshotsDataHandle::Seek(int64_t offset, int32_t whence) { + BlobsIndex start_index; + switch (whence) { + case SEEK_CUR: + start_index = current_index_; + break; + case SEEK_SET: + start_index = {0, 0}; + break; + case SEEK_END: + start_index = {blobs_.size(), blobs_.back()->GetSize()}; + break; + default: + FML_CHECK(false) << "Unrecognized whence value in Seek: " << whence; + } + current_index_ = IndexForAbsoluteOffset(offset, start_index); + return current_index_.offset; +} + +} // namespace flutter \ No newline at end of file diff --git a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.h b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.h new file mode 100644 index 0000000000000..50c4b8179b412 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.h @@ -0,0 +1,48 @@ +#ifndef FLUTTER_SHELL_COMMON_SHOREBIRD_SNAPSHOTS_DATA_HANDLE_H_ +#define FLUTTER_SHELL_COMMON_SHOREBIRD_SNAPSHOTS_DATA_HANDLE_H_ + +#include +#include "flutter/fml/file.h" +#include "flutter/runtime/dart_snapshot.h" +#include "third_party/dart/runtime/include/dart_tools_api.h" + +namespace flutter { + +// An offset into an indexed collection of buffers. blob is the index of the +// buffer, and offset is the offset into that buffer. +struct BlobsIndex { + size_t blob; + size_t offset; +}; + +// Implements a POSIX file I/O interface which allows us to provide the four +// data blobs of a Dart snapshot (vm_data, vm_instructions, isolate_data, +// isolate_instructions) to Rust as though it were a single piece of memory. +class SnapshotsDataHandle { + public: + // This would ideally be private, but we need to be able to call it from the + // static createForSnapshots method. + explicit SnapshotsDataHandle(std::vector> blobs) + : blobs_(std::move(blobs)) {} + + static std::unique_ptr createForSnapshots( + const DartSnapshot& vm_snapshot, + const DartSnapshot& isolate_snapshot); + + uintptr_t Read(uint8_t* buffer, uintptr_t length); + int64_t Seek(int64_t offset, int32_t whence); + + // The sum of all the blobs' sizes. + size_t FullSize() const; + + private: + size_t AbsoluteOffsetForIndex(BlobsIndex index); + BlobsIndex IndexForAbsoluteOffset(int64_t offset, BlobsIndex startIndex); + + BlobsIndex current_index_ = {0, 0}; + std::vector> blobs_; +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_COMMON_SHOREBIRD_SNAPSHOTS_DATA_HANDLE_H_ diff --git a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle_unittests.cc b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle_unittests.cc new file mode 100644 index 0000000000000..2acf44a7b8973 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle_unittests.cc @@ -0,0 +1,177 @@ +#include +#include +#include + +#include "flutter/shell/common/shorebird/snapshots_data_handle.h" + +#include "flutter/fml/mapping.h" +#include "flutter/runtime/dart_snapshot.h" +#include "flutter/testing/testing.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "testing/fixture_test.h" + +namespace flutter { +namespace testing { + +std::unique_ptr MakeHandle( + std::vector& blobs) { + // Map the strings into non-owned mappings: + std::vector> mappings = {}; + for (auto& blob : blobs) { + std::unique_ptr mapping = + std::make_unique( + reinterpret_cast(blob.data()), blob.size()); + mappings.push_back(std::move(mapping)); + } + auto handle = + std::make_unique(std::move(mappings)); + return handle; +} + +TEST(SnapshotsDataHandle, Read) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 12; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + blobs_handle->Read(buffer, 6); + + EXPECT_EQ(buffer[0], 'a'); + EXPECT_EQ(buffer[1], 'b'); + EXPECT_EQ(buffer[2], 'c'); + EXPECT_EQ(buffer[3], 'd'); + EXPECT_EQ(buffer[4], 'e'); + EXPECT_EQ(buffer[5], 'f'); + + // Only the first 6 bytes should have been read. + EXPECT_EQ(buffer[6], 0); +} + +TEST(SnapshotsDataHandle, ReadAfterSeekWithPositiveOffset) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + blobs_handle->Seek(4, SEEK_CUR); + blobs_handle->Read(buffer, 6); + + EXPECT_EQ(buffer[0], 'e'); + EXPECT_EQ(buffer[1], 'f'); + EXPECT_EQ(buffer[2], 'g'); + EXPECT_EQ(buffer[3], 'h'); + EXPECT_EQ(buffer[4], 'i'); + EXPECT_EQ(buffer[5], 'j'); + + // Only the first 6 bytes should have been read. + EXPECT_EQ(buffer[6], 0); +} + +TEST(SnapshotsDataHandle, ReadAfterSeekWithNegativeOffset) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + blobs_handle->Read(buffer, 5); + EXPECT_EQ(buffer[0], 'a'); + EXPECT_EQ(buffer[1], 'b'); + EXPECT_EQ(buffer[2], 'c'); + EXPECT_EQ(buffer[3], 'd'); + EXPECT_EQ(buffer[4], 'e'); + EXPECT_EQ(buffer[5], 0); + + // Reset buffer + std::fill(buffer, buffer + buffer_size, 0); + + // Read 5, seeked back 4, should start reading at offset 1 ('b') + blobs_handle->Seek(-4, SEEK_CUR); + blobs_handle->Read(buffer, 6); + + EXPECT_EQ(buffer[0], 'b'); + EXPECT_EQ(buffer[1], 'c'); + EXPECT_EQ(buffer[2], 'd'); + EXPECT_EQ(buffer[3], 'e'); + EXPECT_EQ(buffer[4], 'f'); + EXPECT_EQ(buffer[5], 'g'); + EXPECT_EQ(buffer[6], 0); +} + +TEST(SnapshotsDataHandle, SeekPastEnd) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + // Seek 1 past the end + blobs_handle->Seek(blobs_handle->FullSize() + 1, SEEK_CUR); + + // Seek back 2 bytes and read 2 bytes + blobs_handle->Seek(-2, SEEK_CUR); + blobs_handle->Read(buffer, 2); + + EXPECT_EQ(buffer[0], 'k'); + EXPECT_EQ(buffer[1], 'l'); +} + +TEST(SnapshotsDataHandle, SeekBeforeBeginning) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + // Seek before the start of the blobs and read the first 2 bytes. + blobs_handle->Seek(-2, SEEK_CUR); + blobs_handle->Read(buffer, 2); + + EXPECT_EQ(buffer[0], 'a'); + EXPECT_EQ(buffer[1], 'b'); +} + +TEST(SnapshotsDataHandle, SeekFromBeginning) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + // Seek 10 bytes from current (the beginning) + blobs_handle->Seek(10, SEEK_CUR); + + // Seek 2 bytes from the beginning and read 2 bytes + blobs_handle->Seek(2, SEEK_SET); + blobs_handle->Read(buffer, 2); + + EXPECT_EQ(buffer[0], 'c'); + EXPECT_EQ(buffer[1], 'd'); +} + +TEST(SnapshotsDataHandle, SeekFromEnd) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + // Seek 2 bytes from the end and read 2 bytes + blobs_handle->Seek(-2, SEEK_END); + blobs_handle->Read(buffer, 2); + + EXPECT_EQ(buffer[0], 'k'); + EXPECT_EQ(buffer[1], 'l'); +} + +} // namespace testing +} // namespace flutter diff --git a/engine/src/flutter/shell/platform/android/BUILD.gn b/engine/src/flutter/shell/platform/android/BUILD.gn index 90af9ceb17ac2..11b208db449e5 100644 --- a/engine/src/flutter/shell/platform/android/BUILD.gn +++ b/engine/src/flutter/shell/platform/android/BUILD.gn @@ -171,6 +171,7 @@ source_set("flutter_shell_native_src") { "//flutter/runtime", "//flutter/runtime:libdart", "//flutter/shell/common", + "//flutter/shell/common/shorebird", "//flutter/shell/platform/android/context", "//flutter/shell/platform/android/external_view_embedder", "//flutter/shell/platform/android/jni", @@ -184,6 +185,8 @@ source_set("flutter_shell_native_src") { public_configs = [ "//flutter:config" ] + include_dirs = [ "//flutter/updater" ] + defines = [] libs = [ @@ -191,6 +194,17 @@ source_set("flutter_shell_native_src") { "EGL", "GLESv2", ] + if (target_cpu == "arm") { + libs += [ "//flutter/third_party/updater/target/armv7-linux-androideabi/release/libupdater.a" ] + } else if (target_cpu == "arm64") { + libs += [ "//flutter/third_party/updater/target/aarch64-linux-android/release/libupdater.a" ] + } else if (target_cpu == "x64") { + libs += [ "//flutter/third_party/updater/target/x86_64-linux-android/release/libupdater.a" ] + } else if (target_cpu == "x86") { + libs += [ "//flutter/third_party/updater/target/i686-linux-android/release/libupdater.a" ] + } else { + assert(false, "Unsupported target_cpu") + } } action("gen_android_build_config_java") { @@ -809,8 +823,10 @@ if (target_cpu != "x86") { } } -if (host_os == "linux" && - (target_cpu == "x64" || target_cpu == "arm64" || target_cpu == "riscv64")) { +# Build analyze_snapshot for 64-bit target CPUs on linux. +# Or always targeting arm64 for Shorebird builds. +if ((host_os == "linux" && (target_cpu == "x64" || target_cpu == "riscv64")) || + target_cpu == "arm64") { zip_bundle("analyze_snapshot") { deps = [ "$dart_src/runtime/bin:analyze_snapshot($host_toolchain)" ] diff --git a/engine/src/flutter/shell/platform/android/android_exports.lst b/engine/src/flutter/shell/platform/android/android_exports.lst index 198bff773dd74..9b924e7738cd4 100644 --- a/engine/src/flutter/shell/platform/android/android_exports.lst +++ b/engine/src/flutter/shell/platform/android/android_exports.lst @@ -11,6 +11,18 @@ _binary_icudtl_dat_size; InternalFlutterGpu*; kInternalFlutterGpu*; + shorebird_init; + shorebird_active_path; + shorebird_active_patch_number; + shorebird_free_string; + shorebird_free_update_result; + shorebird_check_for_downloadable_update; + shorebird_check_for_update; + shorebird_update; + shorebird_update_with_result; + shorebird_next_boot_patch_number; + shorebird_current_boot_patch_number; + shorebird_validate_next_boot_patch; local: *; }; diff --git a/engine/src/flutter/shell/platform/android/flutter_main.cc b/engine/src/flutter/shell/platform/android/flutter_main.cc index d14ac40029645..d881ca39efd3f 100644 --- a/engine/src/flutter/shell/platform/android/flutter_main.cc +++ b/engine/src/flutter/shell/platform/android/flutter_main.cc @@ -20,6 +20,7 @@ #include "flutter/fml/platform/android/paths_android.h" #include "flutter/lib/ui/plugins/callback_cache.h" #include "flutter/runtime/dart_vm.h" +#include "flutter/shell/common/shorebird/shorebird.h" #include "flutter/shell/common/switches.h" #include "flutter/shell/platform/android/android_context_vk_impeller.h" #include "flutter/shell/platform/android/android_rendering_selector.h" @@ -29,6 +30,8 @@ #include "impeller/toolkit/android/proc_table.h" #include "txt/platform.h" +#include "third_party/updater/library/include/updater.h" + namespace flutter { constexpr int kMinimumAndroidApiLevelForImpeller = 29; @@ -93,6 +96,9 @@ void FlutterMain::Init(JNIEnv* env, jstring kernelPath, jstring appStoragePath, jstring engineCachesPath, + jstring shorebirdYaml, + jstring version, + jstring versionCode, jlong initTimeMillis, jint api_level) { std::vector args; @@ -151,8 +157,18 @@ void FlutterMain::Init(JNIEnv* env, flutter::DartCallbackCache::SetCachePath( fml::jni::JavaStringToString(env, appStoragePath)); - fml::paths::InitializeAndroidCachesPath( - fml::jni::JavaStringToString(env, engineCachesPath)); + auto code_cache_path = fml::jni::JavaStringToString(env, engineCachesPath); + auto app_storage_path = fml::jni::JavaStringToString(env, appStoragePath); + fml::paths::InitializeAndroidCachesPath(code_cache_path); + +#if FLUTTER_RELEASE + std::string shorebird_yaml = fml::jni::JavaStringToString(env, shorebirdYaml); + std::string version_string = fml::jni::JavaStringToString(env, version); + std::string version_code_string = + fml::jni::JavaStringToString(env, versionCode); + ConfigureShorebird(code_cache_path, app_storage_path, settings, + shorebird_yaml, version_string, version_code_string); +#endif flutter::DartCallbackCache::LoadCacheFromDisk(); @@ -245,6 +261,7 @@ bool FlutterMain::Register(JNIEnv* env) { { .name = "nativeInit", .signature = "(Landroid/content/Context;[Ljava/lang/String;Ljava/" + "lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/" "lang/String;Ljava/lang/String;Ljava/lang/String;JI)V", .fnPtr = reinterpret_cast(&Init), }, diff --git a/engine/src/flutter/shell/platform/android/flutter_main.h b/engine/src/flutter/shell/platform/android/flutter_main.h index dda959801bc32..cf03f889ec30e 100644 --- a/engine/src/flutter/shell/platform/android/flutter_main.h +++ b/engine/src/flutter/shell/platform/android/flutter_main.h @@ -44,6 +44,9 @@ class FlutterMain { jstring kernelPath, jstring appStoragePath, jstring engineCachesPath, + jstring shorebirdYaml, + jstring version, + jstring versionCode, jlong initTimeMillis, jint api_level); diff --git a/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java b/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java index 14ffc8d541abf..ad7428e74f557 100644 --- a/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java +++ b/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java @@ -8,6 +8,8 @@ import android.annotation.SuppressLint; import android.content.Context; +import android.content.pm.PackageInfo; +import android.content.pm.PackageManager; import android.content.res.AssetManager; import android.graphics.Bitmap; import android.graphics.SurfaceTexture; @@ -42,6 +44,10 @@ import io.flutter.view.AccessibilityBridge; import io.flutter.view.FlutterCallbackInformation; import io.flutter.view.TextureRegistry; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.lang.ref.WeakReference; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -178,6 +184,9 @@ private static native void nativeInit( @Nullable String bundlePath, @NonNull String appStoragePath, @NonNull String engineCachesPath, + @Nullable String shorebirdYaml, + @Nullable String version, + @Nullable String versionCode, long initTimeMillis, int apiLevel); @@ -206,8 +215,47 @@ public void init( Log.w(TAG, "FlutterJNI.init called more than once"); } + String version = null; + String versionCode = null; + try { + PackageInfo packageInfo = + context.getPackageManager().getPackageInfo(context.getPackageName(), 0); + version = packageInfo.versionName; + if (Build.VERSION.SDK_INT >= API_LEVELS.API_28) { + versionCode = String.valueOf(packageInfo.getLongVersionCode()); + } else { + versionCode = String.valueOf(packageInfo.versionCode); + } + } catch (PackageManager.NameNotFoundException e) { + Log.e(TAG, "Failed to read app version. Shorebird updater can't run.", e); + } + + String shorebirdYaml = null; + try { + InputStream yaml = context.getAssets().open("flutter_assets/shorebird.yaml"); + BufferedReader r = new BufferedReader(new InputStreamReader(yaml)); + StringBuilder total = new StringBuilder(); + for (String line; (line = r.readLine()) != null; ) { + total.append(line).append('\n'); + } + shorebirdYaml = total.toString(); + Log.d(TAG, "shorebird.yaml: " + shorebirdYaml); + } catch (IOException e) { + Log.e(TAG, "Failed to load shorebird.yaml", e); + Log.e(TAG, "Did you remember to include shorebird.yaml in your pubspec.yaml's assets?"); + } + FlutterJNI.nativeInit( - context, args, bundlePath, appStoragePath, engineCachesPath, initTimeMillis, apiLevel); + context, + args, + bundlePath, + appStoragePath, + engineCachesPath, + shorebirdYaml, + version, + versionCode, + initTimeMillis, + apiLevel); FlutterJNI.initCalled = true; } diff --git a/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn b/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn index 911ee431e2516..fd167854107b1 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn +++ b/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn @@ -75,6 +75,14 @@ source_set("flutter_framework_source") { "//build/config/ios:ios_application_extension", ] + if (target_cpu == "arm64") { + libs = [ "//flutter/third_party/updater/target/aarch64-apple-ios/release/libupdater.a" ] + } else if (target_cpu == "x64") { + libs = [ "//flutter/third_party/updater/target/x86_64-apple-ios/release/libupdater.a" ] + } else { + assert(false, "Unsupported target_cpu") + } + sources = [ "framework/Source/FlutterAppDelegate.mm", "framework/Source/FlutterAppDelegate_Internal.h", @@ -194,12 +202,14 @@ source_set("flutter_framework_source") { deps = [ ":ios_gpu_configuration", + "$dart_src/runtime/bin:elf_loader", "//flutter/common", "//flutter/common/graphics", "//flutter/fml", "//flutter/lib/ui", "//flutter/runtime", "//flutter/shell/common", + "//flutter/shell/common/shorebird", "//flutter/shell/platform/common:common_cpp_input", "//flutter/shell/platform/darwin/common", "//flutter/shell/platform/darwin/common:framework_common", diff --git a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterDartProject.mm b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterDartProject.mm index 31a49a856a4a5..45660d1cd637d 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterDartProject.mm +++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterDartProject.mm @@ -13,6 +13,8 @@ #include "flutter/common/constants.h" #include "flutter/fml/build_config.h" +#include "flutter/fml/paths.h" +#include "flutter/shell/common/shorebird/shorebird.h" #include "flutter/shell/common/switches.h" #import "flutter/shell/platform/darwin/common/InternalFlutterSwiftCommon/InternalFlutterSwiftCommon.h" #include "flutter/shell/platform/darwin/common/command_line.h" @@ -92,10 +94,12 @@ static BOOL DoesHardwareSupportWideGamut() { } if (flutter::DartVM::IsRunningPrecompiledCode()) { + NSLog(@"SANITY CHECK: Running precompiled code."); if (hasExplicitBundle) { NSString* executablePath = bundle.executablePath; if ([[NSFileManager defaultManager] fileExistsAtPath:executablePath]) { settings.application_library_paths.push_back(executablePath.UTF8String); + NSLog(@"Using precompiled library from %@", executablePath); } } @@ -107,6 +111,7 @@ static BOOL DoesHardwareSupportWideGamut() { NSString* executablePath = [NSBundle bundleWithPath:libraryPath].executablePath; if (executablePath.length > 0) { settings.application_library_paths.push_back(executablePath.UTF8String); + NSLog(@"Using library from %@", libraryPath); } } } @@ -121,6 +126,7 @@ static BOOL DoesHardwareSupportWideGamut() { [NSBundle bundleWithPath:applicationFrameworkPath].executablePath; if (executablePath.length > 0) { settings.application_library_paths.push_back(executablePath.UTF8String); + NSLog(@"Using App.framework from %@", applicationFrameworkPath); } } } @@ -152,6 +158,33 @@ static BOOL DoesHardwareSupportWideGamut() { } } + NSString* assetsPath = [NSString stringWithUTF8String:settings.assets_path.c_str()]; + NSLog(@"ASSET PATH %@", assetsPath); + + // FIXME: This may not be the correct path (e.g., should it include the organization id?) + // See + // https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html#//apple_ref/doc/uid/TP40010672-CH2-SW13 + // /private/var/mobile/Containers/Data/Application/264477BF-6E38-47C9-AAD9-532BB842F197/Library/Application + // Support/shorebird/shorebird_updater + std::string cache_path = + fml::paths::JoinPaths({getenv("HOME"), "Library/Application Support/shorebird"}); + NSURL* shorebirdYamlPath = [NSURL URLWithString:@"shorebird.yaml" + relativeToURL:[NSURL fileURLWithPath:assetsPath]]; + NSString* appVersion = [mainBundle objectForInfoDictionaryKey:@"CFBundleShortVersionString"]; + NSString* appBuildNumber = [mainBundle objectForInfoDictionaryKey:@"CFBundleVersion"]; + NSString* shorebirdYamlContents = [NSString stringWithContentsOfURL:shorebirdYamlPath + encoding:NSUTF8StringEncoding + error:nil]; + if (shorebirdYamlContents != nil) { + // Note: we intentionally pass cache_path twice. We provide two different directories + // to ConfigureShorebird because Android differentiates between data that persists + // between releases and data that does not. iOS does not make this distinction. + flutter::ConfigureShorebird(cache_path, cache_path, settings, shorebirdYamlContents.UTF8String, + appVersion.UTF8String, appBuildNumber.UTF8String); + } else { + NSLog(@"Failed to find shorebird.yaml, not starting updater."); + } + // Domain network configuration // Disabled in https://github.com/flutter/flutter/issues/72723. // Re-enable in https://github.com/flutter/flutter/issues/54448. diff --git a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm index 74fea45026de4..841bb7c60e6fc 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm +++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm @@ -39,6 +39,7 @@ #import "flutter/third_party/spring_animation/spring_animation.h" FLUTTER_ASSERT_ARC +#import static constexpr int kMicrosecondsPerSecond = 1000 * 1000; static constexpr CGFloat kScrollViewContentSize = 2.0; diff --git a/engine/src/flutter/shell/platform/darwin/macos/BUILD.gn b/engine/src/flutter/shell/platform/darwin/macos/BUILD.gn index bc69c85f797a0..6c5e1f4eb928b 100644 --- a/engine/src/flutter/shell/platform/darwin/macos/BUILD.gn +++ b/engine/src/flutter/shell/platform/darwin/macos/BUILD.gn @@ -144,12 +144,14 @@ source_set("flutter_framework_source") { ":macos_gpu_configuration", "//flutter/flow", "//flutter/fml", + "//flutter/shell/common/shorebird", "//flutter/shell/platform/common:common_cpp_accessibility", "//flutter/shell/platform/common:common_cpp_core", "//flutter/shell/platform/common:common_cpp_enums", "//flutter/shell/platform/common:common_cpp_input", "//flutter/shell/platform/common:common_cpp_isolate_scope", "//flutter/shell/platform/common:common_cpp_switches", + "//flutter/shell/platform/darwin/common", "//flutter/shell/platform/darwin/common:availability_version_check", "//flutter/shell/platform/darwin/common:framework_common", "//flutter/shell/platform/darwin/graphics", diff --git a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm index 8ba35a05761e3..a514611719a56 100644 --- a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm +++ b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm @@ -11,6 +11,8 @@ #include #include "flutter/common/constants.h" +#include "flutter/fml/paths.h" +#include "flutter/shell/common/shorebird/shorebird.h" #include "flutter/shell/platform/common/app_lifecycle_state.h" #include "flutter/shell/platform/common/engine_switches.h" #include "flutter/shell/platform/embedder/embedder.h" @@ -656,6 +658,40 @@ - (void)onFocusChangeRequest:(const FlutterViewFocusChangeRequest*)request { } } +- (BOOL)configureShorebird:(NSString**)patchPath { + NSLog(@"[shorebird] setting up non-linker shorebird"); + NSString* bundlePath = + [[NSBundle bundleWithURL:[NSBundle.mainBundle.privateFrameworksURL + URLByAppendingPathComponent:@"App.framework"]] bundlePath]; + bundlePath = [bundlePath stringByAppendingString:@"/App"]; + NSString* assetsPath = _project.assetsPath; + NSURL* shorebirdYamlPath = [NSURL URLWithString:@"shorebird.yaml" + relativeToURL:[NSURL fileURLWithPath:assetsPath]]; + NSString* shorebirdYamlContents = [NSString stringWithContentsOfURL:shorebirdYamlPath + encoding:NSUTF8StringEncoding + error:nil]; + NSString* appVersion = + [NSBundle.mainBundle objectForInfoDictionaryKey:@"CFBundleShortVersionString"]; + NSString* appBuildNumber = [NSBundle.mainBundle objectForInfoDictionaryKey:@"CFBundleVersion"]; + std::string cache_path = + fml::paths::JoinPaths({getenv("HOME"), "Library", "Application Support", "shorebird"}); + flutter::ReleaseVersion release_version = {appVersion.UTF8String, appBuildNumber.UTF8String}; + flutter::ShorebirdConfigArgs shorebird_args(cache_path, cache_path, bundlePath.UTF8String, + shorebirdYamlContents.UTF8String, release_version); + NSLog(@"[shorebird] calling ConfigureShorebird"); + std::string patch_path; + auto res = flutter::ConfigureShorebird(shorebird_args, patch_path); + if (!res) { + NSLog(@"[shorebird] ConfigureShorebird failed"); + return NO; + } + + NSLog(@"[shorebird] ConfigureShorebird success!"); + *patchPath = [NSString stringWithUTF8String:patch_path.c_str()]; + NSLog(@"[shorebird] patchPath: %@", *patchPath); + return YES; +} + - (BOOL)runWithEntrypoint:(NSString*)entrypoint { if (self.running) { return NO; @@ -753,7 +789,19 @@ - (BOOL)runWithEntrypoint:(NSString*)entrypoint { }; flutterArguments.custom_task_runners = &custom_task_runners; - [self loadAOTData:_project.assetsPath]; + NSString* elfPath; + BOOL configureShorebirdRes = [self configureShorebird:&elfPath]; + if (!configureShorebirdRes) { + // No patch exists, or we failed to configure shorebird. This is a fallback. + // Upstream, this code lives in -(void)loadAOTData:. + // + // This is the location where the test fixture places the snapshot file. + // For applications built by Flutter tool, this is in "App.framework". + elfPath = [NSString pathWithComponents:@[ _project.assetsPath, @"app_elf_snapshot.so" ]]; + } + + [self loadAOTData:elfPath]; + if (_aotData) { flutterArguments.aot_data = _aotData; } @@ -777,6 +825,7 @@ - (BOOL)runWithEntrypoint:(NSString*)entrypoint { }; FlutterRendererConfig rendererConfig = [_renderer createRendererConfig]; + FlutterEngineResult result = _embedderAPI.Initialize( FLUTTER_ENGINE_VERSION, &rendererConfig, &flutterArguments, (__bridge void*)(self), &_engine); if (result != kSuccess) { @@ -806,7 +855,7 @@ - (BOOL)runWithEntrypoint:(NSString*)entrypoint { return YES; } -- (void)loadAOTData:(NSString*)assetsDir { +- (void)loadAOTData:(NSString*)elfPath { if (!_embedderAPI.RunsAOTCompiledDartCode()) { return; } @@ -814,11 +863,8 @@ - (void)loadAOTData:(NSString*)assetsDir { BOOL isDirOut = false; // required for NSFileManager fileExistsAtPath. NSFileManager* fileManager = [NSFileManager defaultManager]; - // This is the location where the test fixture places the snapshot file. - // For applications built by Flutter tool, this is in "App.framework". - NSString* elfPath = [NSString pathWithComponents:@[ assetsDir, @"app_elf_snapshot.so" ]]; - if (![fileManager fileExistsAtPath:elfPath isDirectory:&isDirOut]) { + FML_LOG(INFO) << "in loadAOTData, elfPath does not exist: " << elfPath.UTF8String; return; } diff --git a/engine/src/flutter/shell/platform/linux/BUILD.gn b/engine/src/flutter/shell/platform/linux/BUILD.gn index b9b1fb6657b11..2039603cb0330 100644 --- a/engine/src/flutter/shell/platform/linux/BUILD.gn +++ b/engine/src/flutter/shell/platform/linux/BUILD.gn @@ -156,6 +156,7 @@ source_set("flutter_linux_sources") { "fl_settings_channel.cc", "fl_settings_handler.cc", "fl_settings_portal.cc", + "fl_shorebird.cc", "fl_socket_accessible.cc", "fl_standard_message_codec.cc", "fl_standard_method_codec.cc", @@ -184,12 +185,14 @@ source_set("flutter_linux_sources") { deps = [ "$dart_src/runtime:dart_api", "//flutter/fml", + "//flutter/shell/common/shorebird", "//flutter/shell/platform/common:common_cpp_enums", "//flutter/shell/platform/common:common_cpp_input", "//flutter/shell/platform/common:common_cpp_isolate_scope", "//flutter/shell/platform/common:common_cpp_switches", "//flutter/shell/platform/embedder:embedder_headers", "//flutter/third_party/rapidjson", + "//flutter/third_party/tonic", ] } diff --git a/engine/src/flutter/shell/platform/linux/fl_engine.cc b/engine/src/flutter/shell/platform/linux/fl_engine.cc index c1205d9974536..abb1701c3a95d 100644 --- a/engine/src/flutter/shell/platform/linux/fl_engine.cc +++ b/engine/src/flutter/shell/platform/linux/fl_engine.cc @@ -10,6 +10,7 @@ #include #include "flutter/common/constants.h" +#include "flutter/fml/logging.h" #include "flutter/shell/platform/common/engine_switches.h" #include "flutter/shell/platform/embedder/embedder.h" #include "flutter/shell/platform/linux/fl_accessibility_handler.h" @@ -24,6 +25,7 @@ #include "flutter/shell/platform/linux/fl_platform_handler.h" #include "flutter/shell/platform/linux/fl_plugin_registrar_private.h" #include "flutter/shell/platform/linux/fl_settings_handler.h" +#include "flutter/shell/platform/linux/fl_shorebird.h" #include "flutter/shell/platform/linux/fl_texture_gl_private.h" #include "flutter/shell/platform/linux/fl_texture_registrar_private.h" #include "flutter/shell/platform/linux/public/flutter_linux/fl_plugin_registry.h" @@ -781,6 +783,9 @@ gboolean fl_engine_start(FlEngine* self, GError** error) { g_autoptr(GPtrArray) command_line_args = g_ptr_array_new_with_free_func(g_free); + // FlutterProjectArgs expects a full argv, so when processing it for flags + // the first item is treated as the executable and ignored. Add a dummy + // value so that all switches are used. g_ptr_array_insert(command_line_args, 0, g_strdup("flutter")); for (const auto& env_switch : flutter::GetSwitchesFromEnvironment()) { g_ptr_array_add(command_line_args, g_strdup(env_switch.c_str())); @@ -818,9 +823,22 @@ gboolean fl_engine_start(FlEngine* self, GError** error) { args.compositor = &compositor; if (self->embedder_api.RunsAOTCompiledDartCode()) { + // This struct contains raw C strings and needs to have its lifetime scoped + // to this block. FlutterEngineAOTDataSource source = {}; source.type = kFlutterEngineAOTDataSourceTypeElfPath; - source.elf_path = fl_dart_project_get_aot_library_path(self->project); + std::string patch_path; + auto setup_shorebird_result = + flutter::SetUpShorebird(args.assets_path, patch_path); + if (setup_shorebird_result) { + // If we have a patch installed, we replace the default AOT library path + // with the patch path here. + source.elf_path = patch_path.c_str(); + } else { + FML_LOG(ERROR) << "Failed to configure Shorebird."; + source.elf_path = fl_dart_project_get_aot_library_path(self->project); + } + if (self->embedder_api.CreateAOTData(&source, &self->aot_data) != kSuccess) { g_set_error(error, fl_engine_error_quark(), FL_ENGINE_ERROR_FAILED, diff --git a/engine/src/flutter/shell/platform/linux/fl_shorebird.cc b/engine/src/flutter/shell/platform/linux/fl_shorebird.cc new file mode 100644 index 0000000000000..eb913ad720c61 --- /dev/null +++ b/engine/src/flutter/shell/platform/linux/fl_shorebird.cc @@ -0,0 +1,56 @@ +#include "flutter/shell/platform/linux/fl_shorebird.h" + +#include +#include +#include + +#include "flutter/fml/file.h" +#include "flutter/fml/logging.h" +#include "flutter/fml/paths.h" +#include "flutter/shell/common/shorebird/shorebird.h" +#include "rapidjson/document.h" +#include "third_party/tonic/filesystem/filesystem/file.h" + +// Namespaced to avoid Google style warnings. +namespace flutter { + +gboolean SetUpShorebird(const char* assets_path, std::string& patch_path) { + auto shorebird_yaml_path = + fml::paths::JoinPaths({assets_path, "shorebird.yaml"}); + std::string shorebird_yaml_contents(""); + if (!filesystem::ReadFileToString(shorebird_yaml_path, + &shorebird_yaml_contents)) { + FML_LOG(ERROR) << "Failed to read shorebird.yaml."; + return false; + } + + std::string code_cache_path = + fml::paths::JoinPaths({g_get_home_dir(), ".shorebird_cache"}); + auto executable_location = fml::paths::GetExecutableDirectoryPath().second; + auto app_path = + fml::paths::JoinPaths({executable_location, "lib", "libapp.so"}); + auto version_json_path = fml::paths::JoinPaths({assets_path, "version.json"}); + std::ifstream input(version_json_path); + if (!input) { + return false; + } + std::string json_contents{std::istreambuf_iterator(input), + std::istreambuf_iterator()}; + + rapidjson::Document json_doc; + json_doc.Parse(json_contents.c_str()); + if (json_doc.HasParseError()) { + // Could not parse version file, aborting. + return false; + } + + const auto version_map = json_doc.GetObject(); + ReleaseVersion release_version{version_map["version"].GetString(), + version_map["build_number"].GetString()}; + + ShorebirdConfigArgs shorebird_args(code_cache_path, code_cache_path, app_path, + shorebird_yaml_contents, release_version); + return ConfigureShorebird(shorebird_args, patch_path); +} + +} // namespace flutter diff --git a/engine/src/flutter/shell/platform/linux/fl_shorebird.h b/engine/src/flutter/shell/platform/linux/fl_shorebird.h new file mode 100644 index 0000000000000..e38965776ac97 --- /dev/null +++ b/engine/src/flutter/shell/platform/linux/fl_shorebird.h @@ -0,0 +1,13 @@ +#ifndef FLUTTER_SHELL_PLATFORM_LINUX_FL_SHOREBIRD_H_ +#define FLUTTER_SHELL_PLATFORM_LINUX_FL_SHOREBIRD_H_ + +#include +#include + +namespace flutter { + +gboolean SetUpShorebird(const char* assets_path, std::string& patch_path); + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_LINUX_FL_SHOREBIRD_H_ diff --git a/engine/src/flutter/shell/platform/windows/BUILD.gn b/engine/src/flutter/shell/platform/windows/BUILD.gn index 6c547936faa0c..4aeda70af9bd2 100644 --- a/engine/src/flutter/shell/platform/windows/BUILD.gn +++ b/engine/src/flutter/shell/platform/windows/BUILD.gn @@ -169,6 +169,7 @@ source_set("flutter_windows_source") { ":flutter_windows_headers", "//flutter/fml", "//flutter/impeller/renderer/backend/gles", + "//flutter/shell/common/shorebird:shorebird", "//flutter/shell/geometry", "//flutter/shell/platform/common:common_cpp", "//flutter/shell/platform/common:common_cpp_input", @@ -184,6 +185,7 @@ source_set("flutter_windows_source") { "//flutter/third_party/angle:libEGL_static", "//flutter/third_party/angle:libGLESv2_static", "//flutter/third_party/rapidjson", + "//flutter/third_party/tonic", ] } @@ -195,6 +197,11 @@ copy("publish_headers_windows") { deps = [ "//flutter/shell/platform/common:publish_headers" ] } +copy("updater_exports_windows") { + sources = [ "flutter_windows.dll.def" ] + outputs = [ "$root_out_dir/{{source_file_part}}" ] +} + shared_library("flutter_windows") { deps = [ ":flutter_windows_source" ] @@ -310,6 +317,7 @@ group("windows") { deps = [ ":flutter_windows", ":publish_headers_windows", + ":updater_exports_windows", "//flutter/shell/platform/windows/client_wrapper:publish_wrapper_windows", ] diff --git a/engine/src/flutter/shell/platform/windows/flutter_project_bundle.h b/engine/src/flutter/shell/platform/windows/flutter_project_bundle.h index 4291206c920e0..e95f4780d88b7 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_project_bundle.h +++ b/engine/src/flutter/shell/platform/windows/flutter_project_bundle.h @@ -56,6 +56,10 @@ class FlutterProjectBundle { // Sets engine switches. void SetSwitches(const std::vector& switches); + void SetAotLibraryPath(const std::filesystem::path& aot_library_path) { + aot_library_path_ = aot_library_path; + } + // Attempts to load AOT data for this bundle. The returned data must be // retained until any engine instance it is passed to has been shut down. // diff --git a/engine/src/flutter/shell/platform/windows/flutter_windows.dll.def b/engine/src/flutter/shell/platform/windows/flutter_windows.dll.def new file mode 100644 index 0000000000000..bcaa785977259 --- /dev/null +++ b/engine/src/flutter/shell/platform/windows/flutter_windows.dll.def @@ -0,0 +1,17 @@ +EXPORTS + shorebird_check_for_downloadable_update = shorebird_check_for_downloadable_update + shorebird_check_for_update = shorebird_check_for_update + shorebird_current_boot_patch_number = shorebird_current_boot_patch_number + shorebird_free_string = shorebird_free_string + shorebird_free_update_result = shorebird_free_update_result + shorebird_init = shorebird_init + shorebird_next_boot_patch_number = shorebird_next_boot_patch_number + shorebird_next_boot_patch_path = shorebird_next_boot_patch_path + shorebird_validate_next_boot_patch = shorebird_validate_next_boot_patch + shorebird_report_launch_failure = shorebird_report_launch_failure + shorebird_report_launch_start = shorebird_report_launch_start + shorebird_report_launch_success = shorebird_report_launch_success + shorebird_should_auto_update = shorebird_should_auto_update + shorebird_start_update_thread = shorebird_start_update_thread + shorebird_update = shorebird_update + shorebird_update_with_result = shorebird_update_with_result \ No newline at end of file diff --git a/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc b/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc index 70d7364d56e64..32a6a675e1cf2 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc +++ b/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc @@ -5,15 +5,20 @@ #include "flutter/shell/platform/windows/flutter_windows_engine.h" #include +#include +#include +#include #include #include #include +#include #include "flutter/fml/logging.h" #include "flutter/fml/paths.h" #include "flutter/fml/platform/win/wstring_conversion.h" #include "flutter/fml/synchronization/waitable_event.h" +#include "flutter/shell/common/shorebird/shorebird.h" #include "flutter/shell/platform/common/client_wrapper/binary_messenger_impl.h" #include "flutter/shell/platform/common/client_wrapper/include/flutter/standard_message_codec.h" #include "flutter/shell/platform/common/path_utils.h" @@ -29,6 +34,7 @@ #include "flutter/shell/platform/windows/window_manager.h" #include "flutter/third_party/accessibility/ax/ax_node.h" #include "shell/platform/windows/flutter_project_bundle.h" +#include "third_party/tonic/filesystem/filesystem/file.h" // winbase.h defines GetCurrentTime as a macro. #undef GetCurrentTime @@ -269,13 +275,148 @@ bool FlutterWindowsEngine::Run() { return Run(""); } +int GetReleaseVersionAndBuildNumber(ReleaseVersion* release_version) { + char module_path[MAX_PATH]; + // Get the full path of the currently running executable. The return value is + // the size of the string that was copied to the buffer, with -1 indicating + // failure. + if (GetModuleFileNameA(NULL, module_path, MAX_PATH) == -1) { + return -1; + } + + // Get the size of the version information + DWORD handle = -1; + DWORD version_info_size = GetFileVersionInfoSizeA(module_path, &handle); + if (version_info_size == -1) { + return -1; + } + + // Allocate memory for version info + std::unique_ptr version_data(new char[version_info_size]); + if (!GetFileVersionInfoA(module_path, handle, version_info_size, + version_data.get())) { + return -1; + } + + // Adopted from + // https://learn.microsoft.com/en-us/windows/win32/api/winver/nf-winver-verqueryvaluea + // Get the translation table + struct LANGANDCODEPAGE { + WORD wLanguage; + WORD wCodePage; + }* lpTranslate; + + UINT cbTranslate = 0; + if (!VerQueryValueA(version_data.get(), "\\VarFileInfo\\Translation", + (LPVOID*)&lpTranslate, &cbTranslate)) { + FML_LOG(ERROR) << "Error: Unable to get translation info."; + return -1; + } + + // Construct the query string using the first translation found + char subBlock[64]; + sprintf_s(subBlock, "\\StringFileInfo\\%04x%04x\\ProductVersion", + lpTranslate[0].wLanguage, lpTranslate[0].wCodePage); + + LPSTR versionString = nullptr; + UINT size = 0; + if (!VerQueryValueA(version_data.get(), subBlock, (LPVOID*)&versionString, + &size)) { + return -1; + } + + if (!versionString) { + return -1; + } + + // The version string is in the format of "1.0.0+1", with the label ("+1") + // being optional. + auto version = std::string(versionString); + auto plusPos = version.find("+"); + if (plusPos != std::string::npos) { + auto semVer = version.substr(0, plusPos); + auto patch = version.substr(plusPos + 1, version.length()); + release_version->version = semVer; + release_version->build_number = patch; + } else { + release_version->version = version; + } + + return kSuccess; +} + +bool GetLocalAppDataPath(std::string& outPath) { + PWSTR path = nullptr; + HRESULT result = SHGetKnownFolderPath(FOLDERID_LocalAppData, 0, NULL, &path); + if (!SUCCEEDED(result)) { + return false; + } + + std::wstring widePath(path); + std::string localAppDataPath(widePath.begin(), widePath.end()); + // The calling process is responsible for freeing this resource + // https://learn.microsoft.com/en-us/windows/win32/api/shlobj_core/nf-shlobj_core-shgetknownfolderpath + CoTaskMemFree(path); + outPath = localAppDataPath; + return true; +} + +bool SetUpShorebird(std::string assets_path_string, std::string& patch_path) { + auto shorebird_yaml_path = + fml::paths::JoinPaths({assets_path_string, "shorebird.yaml"}); + std::string shorebird_yaml_contents(""); + if (!filesystem::ReadFileToString(shorebird_yaml_path, + &shorebird_yaml_contents)) { + FML_LOG(ERROR) << "Failed to read shorebird.yaml."; + return false; + } + + std::string code_cache_path; + if (!GetLocalAppDataPath(code_cache_path)) { + FML_LOG(ERROR) << "Failed to retrieve the local AppData directory."; + return false; + } + + auto executable_location = fml::paths::GetExecutableDirectoryPath().second; + auto app_path = + fml::paths::JoinPaths({executable_location, "data", "app.so"}); + ReleaseVersion release_version; + auto release_version_result = + GetReleaseVersionAndBuildNumber(&release_version); + if (release_version_result != kSuccess) { + FML_LOG(ERROR) + << "Failed to retrieve the release version and build number."; + return false; + } + + ShorebirdConfigArgs shorebird_args(code_cache_path, code_cache_path, app_path, + shorebird_yaml_contents, release_version); + return ConfigureShorebird(shorebird_args, patch_path); +} + bool FlutterWindowsEngine::Run(std::string_view entrypoint) { + std::string assets_path_string = project_->assets_path().u8string(); + std::string icu_path_string = project_->icu_path().u8string(); + if (!project_->HasValidPaths()) { FML_LOG(ERROR) << "Missing or unresolvable paths to assets."; return false; } std::string assets_path_string = fml::PathToUtf8(project_->assets_path()); std::string icu_path_string = fml::PathToUtf8(project_->icu_path()); + + std::string patch_path; + auto setup_shorebird_result = SetUpShorebird(assets_path_string, patch_path); + if (setup_shorebird_result) { + // If we have a patch installed, we replace the default AOT library path + // with the patch path here. + FML_LOG(INFO) << "Setting project patch path: " << patch_path; + project_->SetAotLibraryPath(patch_path); + } else { + FML_LOG(ERROR) << "Failed to configure Shorebird."; + } + + // This loads AOT data from the project_'s aot_library_path_. if (embedder_api_.RunsAOTCompiledDartCode()) { aot_data_ = project_->LoadAotData(embedder_api_); if (!aot_data_) { @@ -406,6 +547,15 @@ bool FlutterWindowsEngine::Run(std::string_view entrypoint) { host->root_isolate_create_callback_(); } }; + // Copied from shell\platform\darwin\macos\framework\Source\FlutterEngine.mm + // Writes log messages to stdout. + args.log_message_callback = [](const char* tag, const char* message, + void* user_data) { + if (tag && tag[0]) { + std::cout << tag << ": "; + } + std::cout << message << std::endl; + }; args.channel_update_callback = [](const FlutterChannelUpdate* update, void* user_data) { auto host = static_cast(user_data); diff --git a/engine/src/flutter/shell/testing/BUILD.gn b/engine/src/flutter/shell/testing/BUILD.gn index 79dd5084ffee2..496843c1b6469 100644 --- a/engine/src/flutter/shell/testing/BUILD.gn +++ b/engine/src/flutter/shell/testing/BUILD.gn @@ -41,7 +41,8 @@ executable("testing") { deps = [ "$dart_src/runtime:libdart_jit", - "$dart_src/runtime/bin:common_embedder_dart_io", + "$dart_src/runtime/bin:dart_io_api", + "$dart_src/runtime/bin:elf_loader", "//flutter/assets", "//flutter/common", "//flutter/flow", diff --git a/engine/src/flutter/sky/tools/create_ios_framework.py b/engine/src/flutter/sky/tools/create_ios_framework.py index d87e0f91bff49..0ead267e9609e 100644 --- a/engine/src/flutter/sky/tools/create_ios_framework.py +++ b/engine/src/flutter/sky/tools/create_ios_framework.py @@ -19,7 +19,8 @@ def main(): parser = argparse.ArgumentParser( description=( 'Creates Flutter.framework, Flutter.xcframework and ' - 'copies architecture-dependent gen_snapshot binaries to output dir' + 'copies architecture-dependent analyze_snapshot and gen_snapshot ' + 'binaries to output dir' ) ) @@ -89,13 +90,17 @@ def main(): '%s_extension_safe' % simulator_x64_out_dir, '%s_extension_safe' % simulator_arm64_out_dir ) - # Copy gen_snapshot binary to destination directory. + # Copy analyze_snapshot and gen_snapshot binaries to destination directory. if arm64_out_dir: gen_snapshot = os.path.join(arm64_out_dir, 'universal', 'gen_snapshot_arm64') + analyze_snapshot = os.path.join(arm64_out_dir, 'analyze_snapshot_arm64') sky_utils.copy_binary(gen_snapshot, os.path.join(dst, 'gen_snapshot_arm64')) + sky_utils.copy_binary(analyze_snapshot, os.path.join(dst, 'analyze_snapshot_arm64')) if x64_out_dir: gen_snapshot = os.path.join(x64_out_dir, 'universal', 'gen_snapshot_x64') + analyze_snapshot = os.path.join(x64_out_dir, 'analyze_snapshot_x64') sky_utils.copy_binary(gen_snapshot, os.path.join(dst, 'gen_snapshot_x64')) + sky_utils.copy_binary(analyze_snapshot, os.path.join(dst, 'analyze_snapshot_x64')) zip_archive(dst, args) return 0 @@ -177,7 +182,7 @@ def zip_archive(dst, args): # See: https://github.com/flutter/flutter/blob/62382c7b83a16b3f48dc06c19a47f6b8667005a5/dev/bots/suite_runners/run_verify_binaries_codesigned_tests.dart#L82-L130 # Binaries that must be codesigned and require entitlements for particular APIs. - with_entitlements = ['gen_snapshot_arm64'] + with_entitlements = ['analyze_snapshot_arm64', 'gen_snapshot_arm64'] with_entitlements_file = os.path.join(dst, 'entitlements.txt') sky_utils.write_codesign_config(with_entitlements_file, with_entitlements) @@ -211,6 +216,7 @@ def zip_archive(dst, args): # pylint: enable=line-too-long zip_contents = [ + 'analyze_snapshot_arm64', 'gen_snapshot_arm64', 'Flutter.xcframework', 'entitlements.txt', diff --git a/engine/src/flutter/testing/run_tests.py b/engine/src/flutter/testing/run_tests.py index 3fa7a2c39d0e0..e5972c413462a 100755 --- a/engine/src/flutter/testing/run_tests.py +++ b/engine/src/flutter/testing/run_tests.py @@ -484,6 +484,7 @@ def make_test( make_test('platform_view_android_delegate_unittests'), # https://github.com/flutter/flutter/issues/36295 make_test('shell_unittests'), + make_test('shorebird_unittests'), ] if is_windows(): diff --git a/packages/flutter_tools/gradle/build.gradle.kts b/packages/flutter_tools/gradle/build.gradle.kts index 31fc5cbaedc50..84a0f8a6e2ca8 100644 --- a/packages/flutter_tools/gradle/build.gradle.kts +++ b/packages/flutter_tools/gradle/build.gradle.kts @@ -64,6 +64,7 @@ dependencies { // * ndkVersion in FlutterExtension in packages/flutter_tools/gradle/src/main/kotlin/FlutterExtension.kt compileOnly("com.android.tools.build:gradle:8.11.1") + implementation("org.yaml:snakeyaml:2.0") testImplementation(kotlin("test")) testImplementation("com.android.tools.build:gradle:8.11.1") testImplementation("org.mockito:mockito-core:5.8.0") diff --git a/packages/flutter_tools/lib/src/base/build.dart b/packages/flutter_tools/lib/src/base/build.dart index bb967d7111be4..008b93da357d2 100644 --- a/packages/flutter_tools/lib/src/base/build.dart +++ b/packages/flutter_tools/lib/src/base/build.dart @@ -8,7 +8,6 @@ import '../artifacts.dart'; import '../build_info.dart'; import '../darwin/darwin.dart'; import '../macos/xcode.dart'; - import 'file_system.dart'; import 'logger.dart'; import 'process.dart'; @@ -130,7 +129,28 @@ class AOTSnapshotter { final Directory outputDir = _fileSystem.directory(outputPath); outputDir.createSync(recursive: true); - final genSnapshotArgs = ['--deterministic']; + // Currently we only use the linker on iOS, but we will eventually split out + // the concept of "optimizes patch snapshot" from "uses linker" and probably + // only uses the linker on iOS, but optimize patch snapshots everywhere. + // TODO(eseidel): TargetPlatform.darwin doesn't use the linker. + bool usesLinker = (platform == TargetPlatform.ios || platform == TargetPlatform.darwin); + final dumpLinkInfoArgs = [ + // Shorebird dumps the class table information during snapshot compilation which is later used during linking. + '--print_class_table_link_debug_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.class_table.json')}', + '--print_class_table_link_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.ct.link')}', + '--print_field_table_link_debug_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.field_table.json')}', + '--print_field_table_link_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.ft.link')}', + '--print_dispatch_table_link_debug_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.dispatch_table.json')}', + '--print_dispatch_table_link_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.dt.link')}', + ]; + + final genSnapshotArgs = [ + // Shorebird uses --deterministic to improve snapshot stability and increase linking. + '--deterministic', + // Only save LinkInfo if we're using the linker. + if (usesLinker) + ...dumpLinkInfoArgs, + ]; final bool targetingApplePlatform = platform == TargetPlatform.ios || platform == TargetPlatform.darwin; diff --git a/packages/flutter_tools/lib/src/build_system/targets/assets.dart b/packages/flutter_tools/lib/src/build_system/targets/assets.dart index f9ebbae2045b5..409f37f994ca9 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/assets.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/assets.dart @@ -13,6 +13,7 @@ import '../../dart/package_map.dart'; import '../../devfs.dart'; import '../../flutter_manifest.dart'; import '../../isolated/native_assets/dart_hook_result.dart'; +import '../../shorebird/shorebird_yaml.dart'; import '../build_system.dart'; import '../depfile.dart'; import '../exceptions.dart'; @@ -162,6 +163,19 @@ Future copyAssets( } if (doCopy) { await (content.file as File).copy(file.path); + if (file.basename == 'shorebird.yaml') { + try { + updateShorebirdYaml( + environment.defines[kFlavor], + file.path, + environment: globals.platform.environment, + ); + } on Exception catch (error) { + throw Exception( + 'Failed to generate shorebird configuration. Error: $error', + ); + } + } } } else { await file.writeAsBytes(await entry.value.content.contentsAsBytes()); diff --git a/packages/flutter_tools/lib/src/build_system/targets/common.dart b/packages/flutter_tools/lib/src/build_system/targets/common.dart index 63eb1c736e825..e67e646bebf1b 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/common.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/common.dart @@ -504,3 +504,38 @@ abstract final class Lipo { } } } + +/// For managing the supplementary linking files for Shorebird. +abstract final class LinkSupplement { + static Future create( + Environment environment, { + required String inputBuildDir, + required String outputBuildDir, + }) async { + // If the shorebird directory exists, delete it first. + final Directory shorebirdDir = environment.fileSystem.directory( + environment.fileSystem.path.join(outputBuildDir, 'shorebird'), + ); + if (shorebirdDir.existsSync()) { + shorebirdDir.deleteSync(recursive: true); + } + + void maybeCopy(String name) { + final File file = environment.fileSystem.file( + environment.fileSystem.path.join(inputBuildDir, name), + ); + if (file.existsSync()) { + file.copySync(environment.fileSystem.path.join(shorebirdDir.path, name)); + } + } + + // Copy the link information (generated by gen_snapshot) + // into the shorebird directory. + maybeCopy('App.ct.link'); + maybeCopy('App.class_table.json'); + maybeCopy('App.dt.link'); + maybeCopy('App.dispatch_table.json'); + maybeCopy('App.ft.link'); + maybeCopy('App.field_table.json'); + } +} diff --git a/packages/flutter_tools/lib/src/build_system/targets/ios.dart b/packages/flutter_tools/lib/src/build_system/targets/ios.dart index a6116632fd46c..9a91c94beeece 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/ios.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/ios.dart @@ -143,6 +143,12 @@ abstract class AotAssemblyBase extends Target { // Don't fail if the dSYM wasn't created (i.e. during a debug build). skipMissingInputs: true, ); + + await LinkSupplement.create( + environment, + inputBuildDir: buildOutputPath, + outputBuildDir: getIosBuildDirectory(), + ); } } diff --git a/packages/flutter_tools/lib/src/build_system/targets/macos.dart b/packages/flutter_tools/lib/src/build_system/targets/macos.dart index dbffaa465c994..c0c382c934f48 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/macos.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/macos.dart @@ -344,6 +344,12 @@ class CompileMacOSFramework extends Target { // Don't fail if the dSYM wasn't created (i.e. during a debug build). skipMissingInputs: true, ); + + await LinkSupplement.create( + environment, + inputBuildDir: buildOutputPath, + outputBuildDir: getMacOSBuildDirectory(), + ); } @override diff --git a/packages/flutter_tools/lib/src/cache.dart b/packages/flutter_tools/lib/src/cache.dart index 275fb098f4f83..59d16d8983ca9 100644 --- a/packages/flutter_tools/lib/src/cache.dart +++ b/packages/flutter_tools/lib/src/cache.dart @@ -34,6 +34,7 @@ import 'base/user_messages.dart'; import 'convert.dart'; import 'features.dart'; +const kShorebirdStorageUrl = 'https://download.shorebird.dev'; const kFlutterRootEnvironmentVariableName = 'FLUTTER_ROOT'; // should point to //flutter/ (root of flutter/flutter repo) const kFlutterEngineEnvironmentVariableName = @@ -532,6 +533,10 @@ class Cache { ? 'https://storage.googleapis.com' : 'https://storage.googleapis.com/$storageRealm'; } + // Shorebird's artifact proxy is a trusted source. + if (overrideUrl == kShorebirdStorageUrl) { + return overrideUrl; + } // verify that this is a valid URI. overrideUrl = storageRealm.isEmpty ? overrideUrl : '$overrideUrl/$storageRealm'; try { diff --git a/packages/flutter_tools/lib/src/ios/application_package.dart b/packages/flutter_tools/lib/src/ios/application_package.dart index f1c8f44849548..e8613d4d7a8bd 100644 --- a/packages/flutter_tools/lib/src/ios/application_package.dart +++ b/packages/flutter_tools/lib/src/ios/application_package.dart @@ -130,6 +130,18 @@ class BuildableIOSApp extends IOSApp { @override String? get name => _appProductName; + String get shorebirdYamlPath => + globals.fs.path.join( + archiveBundleOutputPath, + 'Products', + 'Applications', + _appProductName != null ? '$_appProductName.app' : 'Runner.app', + 'Frameworks', + 'App.framework', + 'flutter_assets', + 'shorebird.yaml', + ); + @override String get simulatorBundlePath => _buildAppPath(XcodeSdk.IPhoneSimulator.platformName); diff --git a/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart b/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart new file mode 100644 index 0000000000000..9e40f392b7eb8 --- /dev/null +++ b/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart @@ -0,0 +1,64 @@ +// Copyright 2024 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:yaml/yaml.dart'; +import 'package:yaml_edit/yaml_edit.dart'; + +import '../base/file_system.dart'; +import '../globals.dart' as globals; + +void updateShorebirdYaml(String? flavor, String shorebirdYamlPath, {required Map environment}) { + final File shorebirdYaml = globals.fs.file(shorebirdYamlPath); + if (!shorebirdYaml.existsSync()) { + throw Exception('shorebird.yaml not found at $shorebirdYamlPath'); + } + final YamlDocument input = loadYamlDocument(shorebirdYaml.readAsStringSync()); + final YamlMap yamlMap = input.contents as YamlMap; + final Map compiled = compileShorebirdYaml(yamlMap, flavor: flavor, environment: environment); + // Currently we write out over the same yaml file, we should fix this to + // write to a new .json file instead and avoid naming confusion between the + // input and compiled files. + final YamlEditor yamlEditor = YamlEditor(''); + yamlEditor.update([], compiled); + shorebirdYaml.writeAsStringSync(yamlEditor.toString(), flush: true); +} + +String appIdForFlavor(YamlMap yamlMap, {required String? flavor}) { + if (flavor == null || flavor.isEmpty) { + final String? defaultAppId = yamlMap['app_id'] as String?; + if (defaultAppId == null || defaultAppId.isEmpty) { + throw Exception('Cannot find "app_id" in shorebird.yaml'); + } + return defaultAppId; + } + + final YamlMap? yamlFlavors = yamlMap['flavors'] as YamlMap?; + if (yamlFlavors == null) { + throw Exception('Cannot find "flavors" in shorebird.yaml.'); + } + final String? flavorAppId = yamlFlavors[flavor] as String?; + if (flavorAppId == null || flavorAppId.isEmpty) { + throw Exception('Cannot find "app_id" for $flavor in shorebird.yaml'); + } + return flavorAppId; +} + +Map compileShorebirdYaml(YamlMap yamlMap, {required String? flavor, required Map environment}) { + final String appId = appIdForFlavor(yamlMap, flavor: flavor); + final Map compiled = { + 'app_id': appId, + }; + void copyIfSet(String key) { + if (yamlMap[key] != null) { + compiled[key] = yamlMap[key]; + } + } + copyIfSet('base_url'); + copyIfSet('auto_update'); + final String? shorebirdPublicKeyEnvVar = environment['SHOREBIRD_PUBLIC_KEY']; + if (shorebirdPublicKeyEnvVar != null) { + compiled['patch_public_key'] = shorebirdPublicKeyEnvVar; + } + return compiled; +} diff --git a/packages/flutter_tools/lib/src/version.dart b/packages/flutter_tools/lib/src/version.dart index 0eb275a7bed11..d7a4a765b38d9 100644 --- a/packages/flutter_tools/lib/src/version.dart +++ b/packages/flutter_tools/lib/src/version.dart @@ -1049,6 +1049,19 @@ class GitTagVersion { } } + // Check if running on a Shorebird release branch. + final String shorebirdFlutterReleases = _runGit( + 'git for-each-ref --contains $gitRef --format %(refname:short) refs/remotes/origin/flutter_release/*', + processUtils, + workingDirectory, + ).trim(); + final String? shorebirdFlutterVersion = LineSplitter.split( + shorebirdFlutterReleases, + ).map((e) => e.replaceFirst('origin/flutter_release/', '')).toList().firstOrNull; + if (shorebirdFlutterVersion != null) { + return parse(shorebirdFlutterVersion); + } + // If we don't exist in a tag, use git to find the latest tag. return _useNewestTagAndCommitsPastFallback( git: git, diff --git a/packages/flutter_tools/lib/src/windows/build_windows.dart b/packages/flutter_tools/lib/src/windows/build_windows.dart index 44d8992e29e37..61a53a904979e 100644 --- a/packages/flutter_tools/lib/src/windows/build_windows.dart +++ b/packages/flutter_tools/lib/src/windows/build_windows.dart @@ -21,6 +21,7 @@ import '../flutter_plugins.dart'; import '../globals.dart' as globals; import '../migrations/cmake_custom_command_migration.dart'; import '../migrations/cmake_native_assets_migration.dart'; +import '../shorebird/shorebird_yaml.dart'; import 'migrations/build_architecture_migration.dart'; import 'migrations/show_window_migration.dart'; import 'migrations/version_migration.dart'; diff --git a/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart index 7c3845044399b..adafb265253c6 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart @@ -73,6 +73,14 @@ final macosPlatformCustomEnv = FakePlatform( environment: {'FLUTTER_ROOT': '/', 'HOME': '/'}, ); +final Platform macosPlatformWithShorebirdPublicKey = FakePlatform( + operatingSystem: 'macos', + environment: { + 'FLUTTER_ROOT': '/', + 'HOME': '/', + 'SHOREBIRD_PUBLIC_KEY': 'my_public_key', + } +); final Platform notMacosPlatform = FakePlatform(environment: {'FLUTTER_ROOT': '/'}); void main() { @@ -1123,4 +1131,37 @@ STDERR STUFF OperatingSystemUtils: () => FakeOperatingSystemUtils(hostPlatform: HostPlatform.darwin_x64), }, ); + + testUsingContext('macOS build outputs path and size when successful', + () async { + final BuildCommand command = BuildCommand( + artifacts: artifacts, + androidSdk: FakeAndroidSdk(), + buildSystem: TestBuildSystem.all(BuildResult(success: true)), + fileSystem: MemoryFileSystem.test(), + processUtils: processUtils, + logger: BufferLogger.test(), + osUtils: FakeOperatingSystemUtils(), + ); + createMinimalMockProjectFiles(); + final File shorebirdYamlFile = fileSystem.file( + 'build/macos/Build/Products/Release/example.app/Contents/Frameworks/App.framework/Resources/flutter_assets/shorebird.yaml', + ) + ..createSync(recursive: true) + ..writeAsStringSync('app_id: my-app-id'); + + await createTestCommandRunner(command) + .run(const ['build', 'macos', '--no-pub']); + + final String updatedYaml = shorebirdYamlFile.readAsStringSync(); + expect(updatedYaml, contains('app_id: my-app-id')); + expect(updatedYaml, contains('patch_public_key: my_public_key')); + }, overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => FakeProcessManager.list([ + setUpFakeXcodeBuildHandler('Release'), + ]), + Platform: () => macosPlatformWithShorebirdPublicKey, + FeatureFlags: () => TestFeatureFlags(isMacOSEnabled: true), + }); } diff --git a/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart index 11c574670713a..25da553b819e3 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart @@ -38,6 +38,13 @@ final Platform windowsPlatform = FakePlatform( 'USERPROFILE': '/', }, ); +final Platform windowsPlatformWithPublicKey = + FakePlatform(operatingSystem: 'windows', environment: { + 'PROGRAMFILES(X86)': r'C:\Program Files (x86)\', + 'FLUTTER_ROOT': flutterRoot, + 'USERPROFILE': '/', + 'SHOREBIRD_PUBLIC_KEY': 'my_public_key', +}); final Platform notWindowsPlatform = FakePlatform( environment: {'FLUTTER_ROOT': flutterRoot}, ); @@ -115,7 +122,9 @@ void main() { ...['--target', 'INSTALL'], if (verbose) '--verbose', ], - environment: {if (verbose) 'VERBOSE_SCRIPT_LOGGING': 'true'}, + environment: { + if (verbose) 'VERBOSE_SCRIPT_LOGGING': 'true' + }, onRun: onRun, stdout: stdout, ); @@ -131,7 +140,8 @@ void main() { setUpMockProjectFilesForBuild(); expect( - createTestCommandRunner(command).run(const ['windows', '--no-pub']), + createTestCommandRunner(command) + .run(const ['windows', '--no-pub']), throwsToolExit(), ); }, @@ -154,10 +164,10 @@ void main() { setUpMockCoreProjectFiles(); expect( - createTestCommandRunner(command).run(const ['windows', '--no-pub']), + createTestCommandRunner(command) + .run(const ['windows', '--no-pub']), throwsToolExit( - message: - 'No Windows desktop project configured. See ' + message: 'No Windows desktop project configured. See ' 'https://flutter.dev/to/add-desktop-support ' 'to learn about adding Windows support to a project.', ), @@ -182,8 +192,10 @@ void main() { setUpMockProjectFilesForBuild(); expect( - createTestCommandRunner(command).run(const ['windows', '--no-pub']), - throwsToolExit(message: '"build windows" only supported on Windows hosts.'), + createTestCommandRunner(command) + .run(const ['windows', '--no-pub']), + throwsToolExit( + message: '"build windows" only supported on Windows hosts.'), ); }, overrides: { @@ -205,7 +217,8 @@ void main() { setUpMockProjectFilesForBuild(); expect( - createTestCommandRunner(command).run(const ['windows', '--no-pub']), + createTestCommandRunner(command) + .run(const ['windows', '--no-pub']), throwsToolExit( message: '"build windows" is not currently supported. To enable, run "flutter config --enable-windows-desktop".', @@ -235,7 +248,8 @@ void main() { buildCommand('Release', stdout: 'STDOUT STUFF'), ]); - await createTestCommandRunner(command).run(const ['windows', '--no-pub']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub']); expect(testLogger.statusText, isNot(contains('STDOUT STUFF'))); expect(testLogger.traceText, contains('STDOUT STUFF')); }, @@ -262,7 +276,8 @@ void main() { buildCommand('Release'), ]); - await createTestCommandRunner(command).run(const ['windows', '--no-pub']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub']); expect( analyticsTimingEventExists( @@ -332,7 +347,8 @@ C:\foo\windows\x64\runner\main.cpp(17,1): error C2065: 'Baz': undeclared identif buildCommand('Release', stdout: stdout), ]); - await createTestCommandRunner(command).run(const ['windows', '--no-pub']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub']); // Just the warnings and errors should be surfaced. expect(testLogger.errorText, r''' C:\foo\windows\x64\runner\main.cpp(18): error C2220: the following warning is treated as an error [C:\foo\build\windows\x64\runner\test.vcxproj] @@ -365,7 +381,8 @@ C:\foo\windows\x64\runner\main.cpp(17,1): error C2065: 'Baz': undeclared identif buildCommand('Release', verbose: true, stdout: 'STDOUT STUFF'), ]); - await createTestCommandRunner(command).run(const ['windows', '--no-pub', '-v']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub', '-v']); expect(testLogger.statusText, contains('STDOUT STUFF')); expect(testLogger.traceText, isNot(contains('STDOUT STUFF'))); }, @@ -485,7 +502,8 @@ if %errorlevel% neq 0 goto :VCEnd assembleProject.createSync(recursive: true); assembleProject.writeAsStringSync(fakeBadProjectContent); - await createTestCommandRunner(command).run(const ['windows', '--no-pub']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub']); final List projectLines = assembleProject.readAsLinesSync(); @@ -645,7 +663,8 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, ).run(const ['windows', '--release', '--no-pub']); - expect(testLogger.statusText, contains(r'โœ“ Built build\windows\x64\runner\Release')); + expect(testLogger.statusText, + contains(r'โœ“ Built build\windows\x64\runner\Release')); }, overrides: { FileSystem: () => fileSystem, @@ -702,7 +721,8 @@ if %errorlevel% neq 0 goto :VCEnd buildCommand('Release'), ]); - await createTestCommandRunner(command).run(const ['windows', '--no-pub']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub']); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -750,7 +770,12 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, - ).run(const ['windows', '--no-pub', '--build-name=1.2.3', '--build-number=4']); + ).run(const [ + 'windows', + '--no-pub', + '--build-name=1.2.3', + '--build-number=4' + ]); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -906,7 +931,12 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, - ).run(const ['windows', '--no-pub', '--build-name=1.2.3', '--build-number=4']); + ).run(const [ + 'windows', + '--no-pub', + '--build-name=1.2.3', + '--build-number=4' + ]); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -954,7 +984,12 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, - ).run(const ['windows', '--no-pub', '--build-name=1.2.3', '--build-number=hello']); + ).run(const [ + 'windows', + '--no-pub', + '--build-name=1.2.3', + '--build-number=hello' + ]); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -1011,7 +1046,12 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, - ).run(const ['windows', '--no-pub', '--build-name=1.2.3', '--build-number=4.5']); + ).run(const [ + 'windows', + '--no-pub', + '--build-name=1.2.3', + '--build-number=4.5' + ]); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -1131,14 +1171,16 @@ if %errorlevel% neq 0 goto :VCEnd contains('A summary of your Windows bundle analysis can be found at'), ); expect(testLogger.statusText, contains('dart devtools --appSizeBase=')); - expect(fakeAnalytics.sentEvents, contains(Event.codeSizeAnalysis(platform: 'windows'))); + expect(fakeAnalytics.sentEvents, + contains(Event.codeSizeAnalysis(platform: 'windows'))); }, overrides: { FeatureFlags: () => TestFeatureFlags(isWindowsEnabled: true), FileSystem: () => fileSystem, ProcessManager: () => processManager, Platform: () => windowsPlatform, - FileSystemUtils: () => FileSystemUtils(fileSystem: fileSystem, platform: windowsPlatform), + FileSystemUtils: () => + FileSystemUtils(fileSystem: fileSystem, platform: windowsPlatform), Analytics: () => fakeAnalytics, }, ); @@ -1155,12 +1197,14 @@ if %errorlevel% neq 0 goto :VCEnd logger: BufferLogger.test(), operatingSystemUtils: FakeOperatingSystemUtils(), )..visualStudioOverride = fakeVisualStudio; - fileSystem.currentDirectory = fileSystem.directory("test_'path")..createSync(); + fileSystem.currentDirectory = fileSystem.directory("test_'path") + ..createSync(); final String absPath = fileSystem.currentDirectory.absolute.path; setUpMockCoreProjectFiles(); expect( - createTestCommandRunner(command).run(const ['windows', '--no-pub']), + createTestCommandRunner(command) + .run(const ['windows', '--no-pub']), throwsToolExit( message: 'Path $absPath contains invalid characters in "\'#!\$^&*=|,;<>?". ' @@ -1199,7 +1243,8 @@ No file or variants found for asset: images/a_dot_burr.jpeg. buildCommand('Release', stdout: stdout), ]); - await createTestCommandRunner(command).run(const ['windows', '--no-pub']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub']); // Just the warnings and errors should be surfaced. expect(testLogger.errorText, r''' Error detected in pubspec.yaml: @@ -1213,6 +1258,39 @@ No file or variants found for asset: images/a_dot_burr.jpeg. FeatureFlags: () => TestFeatureFlags(isWindowsEnabled: true), }, ); + + testUsingContext( + 'shorebird.yaml is updated when SHOREBIRD_PUBLIC_KEY env var is set', + () async { + final FakeVisualStudio fakeVisualStudio = FakeVisualStudio(); + final BuildWindowsCommand command = BuildWindowsCommand( + logger: BufferLogger.test(), + operatingSystemUtils: FakeOperatingSystemUtils()) + ..visualStudioOverride = fakeVisualStudio; + setUpMockProjectFilesForBuild(); + final File shorebirdYamlFile = fileSystem.file( + r'build\windows\x64\runner\Release\data\flutter_assets\shorebird.yaml', + ) + ..createSync(recursive: true) + ..writeAsStringSync('app_id: my-app-id'); + + processManager = FakeProcessManager.list([ + cmakeGenerationCommand(), + buildCommand('Release'), + ]); + + await createTestCommandRunner(command) + .run(const ['windows', '--release', '--no-pub']); + + final String updatedYaml = shorebirdYamlFile.readAsStringSync(); + expect(updatedYaml, contains('app_id: my-app-id')); + expect(updatedYaml, contains('patch_public_key: my_public_key')); + }, overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => processManager, + Platform: () => windowsPlatformWithPublicKey, + FeatureFlags: () => TestFeatureFlags(isWindowsEnabled: true), + }); } class FakeVisualStudio extends Fake implements VisualStudio { diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart index aa31e06bfcddd..4ada1d72a0f8e 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart @@ -142,7 +142,14 @@ void main() { ); lipoExtractX86_64Command = FakeCommand( - command: ['lipo', '-output', binary.path, '-extract', 'x86_64', binary.path], + command: [ + 'lipo', + '-output', + binary.path, + '-extract', + 'x86_64', + binary.path + ], ); }); @@ -229,7 +236,8 @@ void main() { isException.having( (Exception exception) => exception.toString(), 'description', - contains('FlutterMacOS.framework/Versions/A/FlutterMacOS does not exist, cannot thin'), + contains( + 'FlutterMacOS.framework/Versions/A/FlutterMacOS does not exist, cannot thin'), ), ), ); @@ -249,7 +257,13 @@ void main() { copyFrameworkCommand, lipoInfoFatCommand, FakeCommand( - command: ['lipo', binary.path, '-verify_arch', 'arm64', 'x86_64'], + command: [ + 'lipo', + binary.path, + '-verify_arch', + 'arm64', + 'x86_64' + ], exitCode: 1, ), ]); @@ -285,7 +299,8 @@ void main() { expect( logger.traceText, - contains('Skipping lipo for non-fat file /FlutterMacOS.framework/Versions/A/FlutterMacOS'), + contains( + 'Skipping lipo for non-fat file /FlutterMacOS.framework/Versions/A/FlutterMacOS'), ); }); @@ -313,7 +328,8 @@ void main() { lipoVerifyX86_64Command, ]); - await const ReleaseUnpackMacOS().build(environment..defines[kBuildMode] = 'release'); + await const ReleaseUnpackMacOS() + .build(environment..defines[kBuildMode] = 'release'); expect(processManager, hasNoRemainingExpectations); }, @@ -335,7 +351,8 @@ void main() { copyFrameworkDsymCommand, ]); - await const ReleaseUnpackMacOS().build(environment..defines[kBuildMode] = 'release'); + await const ReleaseUnpackMacOS() + .build(environment..defines[kBuildMode] = 'release'); expect(processManager, hasNoRemainingExpectations); }, @@ -372,7 +389,8 @@ void main() { ]); await expectLater( - const ReleaseUnpackMacOS().build(environment..defines[kBuildMode] = 'release'), + const ReleaseUnpackMacOS() + .build(environment..defines[kBuildMode] = 'release'), throwsA( isException.having( (Exception exception) => exception.toString(), @@ -393,15 +411,19 @@ void main() { () async { fileSystem .directory( - artifacts.getArtifactPath(Artifact.flutterMacOSFramework, mode: BuildMode.debug), + artifacts.getArtifactPath(Artifact.flutterMacOSFramework, + mode: BuildMode.debug), ) .createSync(); - final String inputKernel = fileSystem.path.join(environment.buildDir.path, 'app.dill'); + final String inputKernel = + fileSystem.path.join(environment.buildDir.path, 'app.dill'); fileSystem.file(inputKernel) ..createSync(recursive: true) ..writeAsStringSync('testing'); - expect(() async => const DebugMacOSBundleFlutterAssets().build(environment), throwsException); + expect( + () async => const DebugMacOSBundleFlutterAssets().build(environment), + throwsException); }, overrides: { FileSystem: () => fileSystem, @@ -414,7 +436,8 @@ void main() { () async { fileSystem .directory( - artifacts.getArtifactPath(Artifact.flutterMacOSFramework, mode: BuildMode.debug), + artifacts.getArtifactPath(Artifact.flutterMacOSFramework, + mode: BuildMode.debug), ) .createSync(); fileSystem @@ -447,20 +470,25 @@ void main() { expect( fileSystem - .file('App.framework/Versions/A/Resources/flutter_assets/kernel_blob.bin') + .file( + 'App.framework/Versions/A/Resources/flutter_assets/kernel_blob.bin') .readAsStringSync(), 'testing', ); expect( - fileSystem.file('App.framework/Versions/A/Resources/Info.plist').readAsStringSync(), + fileSystem + .file('App.framework/Versions/A/Resources/Info.plist') + .readAsStringSync(), contains('io.flutter.flutter.app'), ); expect( - fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/vm_snapshot_data'), + fileSystem.file( + 'App.framework/Versions/A/Resources/flutter_assets/vm_snapshot_data'), exists, ); expect( - fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/isolate_snapshot_data'), + fileSystem.file( + 'App.framework/Versions/A/Resources/flutter_assets/isolate_snapshot_data'), exists, ); }, @@ -563,23 +591,30 @@ void main() { fileSystem .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') .createSync(recursive: true); - fileSystem.file('${environment.buildDir.path}/App.framework/App').createSync(recursive: true); - fileSystem.file('${environment.buildDir.path}/native_assets.json').createSync(); + fileSystem + .file('${environment.buildDir.path}/App.framework/App') + .createSync(recursive: true); + fileSystem + .file('${environment.buildDir.path}/native_assets.json') + .createSync(); await const ProfileMacOSBundleFlutterAssets().build( environment..defines[kBuildMode] = 'profile', ); expect( - fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/kernel_blob.bin'), + fileSystem.file( + 'App.framework/Versions/A/Resources/flutter_assets/kernel_blob.bin'), isNot(exists), ); expect( - fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/vm_snapshot_data'), + fileSystem.file( + 'App.framework/Versions/A/Resources/flutter_assets/vm_snapshot_data'), isNot(exists), ); expect( - fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/isolate_snapshot_data'), + fileSystem.file( + 'App.framework/Versions/A/Resources/flutter_assets/isolate_snapshot_data'), isNot(exists), ); }, @@ -598,17 +633,23 @@ void main() { fileSystem .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') .createSync(recursive: true); - fileSystem.file('${environment.buildDir.path}/App.framework/App').createSync(recursive: true); fileSystem - .file('${environment.buildDir.path}/App.framework.dSYM/Contents/Resources/DWARF/App') + .file('${environment.buildDir.path}/App.framework/App') + .createSync(recursive: true); + fileSystem + .file( + '${environment.buildDir.path}/App.framework.dSYM/Contents/Resources/DWARF/App') .createSync(recursive: true); - fileSystem.file('${environment.buildDir.path}/native_assets.json').createSync(); + fileSystem + .file('${environment.buildDir.path}/native_assets.json') + .createSync(); await const ReleaseMacOSBundleFlutterAssets().build( environment..defines[kBuildMode] = 'release', ); - expect(fileSystem.file('App.framework.dSYM/Contents/Resources/DWARF/App'), exists); + expect(fileSystem.file('App.framework.dSYM/Contents/Resources/DWARF/App'), + exists); }, overrides: { FileSystem: () => fileSystem, @@ -625,17 +666,20 @@ void main() { fileSystem .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') .createSync(recursive: true); - final File inputFramework = - fileSystem.file(fileSystem.path.join(environment.buildDir.path, 'App.framework', 'App')) - ..createSync(recursive: true) - ..writeAsStringSync('ABC'); - fileSystem.file(environment.buildDir.childFile('native_assets.json')).createSync(); + final File inputFramework = fileSystem.file(fileSystem.path + .join(environment.buildDir.path, 'App.framework', 'App')) + ..createSync(recursive: true) + ..writeAsStringSync('ABC'); + fileSystem + .file(environment.buildDir.childFile('native_assets.json')) + .createSync(); await const ProfileMacOSBundleFlutterAssets().build( environment..defines[kBuildMode] = 'profile', ); final File outputFramework = fileSystem.file( - fileSystem.path.join(environment.outputDir.path, 'App.framework', 'App'), + fileSystem.path + .join(environment.outputDir.path, 'App.framework', 'App'), ); expect(outputFramework.readAsStringSync(), 'ABC'); @@ -666,9 +710,12 @@ void main() { .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') .createSync(recursive: true); fileSystem - .file(fileSystem.path.join(environment.buildDir.path, 'App.framework', 'App')) + .file(fileSystem.path + .join(environment.buildDir.path, 'App.framework', 'App')) .createSync(recursive: true); - fileSystem.file(environment.buildDir.childFile('native_assets.json')).createSync(); + fileSystem + .file(environment.buildDir.childFile('native_assets.json')) + .createSync(); await const ReleaseMacOSBundleFlutterAssets().build(environment); expect( @@ -702,7 +749,8 @@ void main() { expect( fakeAnalytics.sentEvents, contains( - Event.appleUsageEvent(workflow: 'assemble', parameter: 'macos-archive', result: 'fail'), + Event.appleUsageEvent( + workflow: 'assemble', parameter: 'macos-archive', result: 'fail'), ), ); }, @@ -739,7 +787,10 @@ void main() { '-install_name', '@rpath/App.framework/App', '-o', - environment.buildDir.childDirectory('App.framework').childFile('App').path, + environment.buildDir + .childDirectory('App.framework') + .childFile('App') + .path, ], ), ); @@ -753,6 +804,54 @@ void main() { }, ); + testUsingContext( + 'ReleaseMacOSBundleFlutterAssets updates shorebird.yaml if present', + () async { + environment.defines[kBuildMode] = 'release'; + environment.defines[kXcodeAction] = 'install'; + environment.defines[kFlavor] = 'internal'; + + fileSystem + .file('bin/cache/artifacts/engine/darwin-x64/vm_isolate_snapshot.bin') + .createSync(recursive: true); + fileSystem + .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') + .createSync(recursive: true); + fileSystem + .file(fileSystem.path + .join(environment.buildDir.path, 'App.framework', 'App')) + .createSync(recursive: true); + final String shorebirdYamlPath = fileSystem.path.join( + environment.buildDir.path, + 'App.framework', + 'Versions', + 'A', + 'Resources', + 'flutter_assets', + 'shorebird.yaml', + ); + fileSystem.file(fileSystem.path + .join(environment.buildDir.path, 'App.framework', 'App')) + ..createSync(recursive: true) + ..writeAsStringSync(''' +# Some other text that should be removed +app_id: base-app-id +flavors: + internal: internal-app-id + stable: stable-app-id +'''); + + await const ReleaseMacOSBundleFlutterAssets().build(environment); + + expect(fileSystem.file(shorebirdYamlPath).readAsStringSync(), + 'app_id: internal-app-id'); + }, + overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => processManager, + }, + ); + testUsingContext( 'DebugMacOSFramework creates universal binary', () async { @@ -782,7 +881,10 @@ void main() { '-install_name', '@rpath/App.framework/App', '-o', - environment.buildDir.childDirectory('App.framework').childFile('App').path, + environment.buildDir + .childDirectory('App.framework') + .childFile('App') + .path, ], ), ); @@ -951,14 +1053,18 @@ void main() { command: [ 'lipo', environment.buildDir - .childFile('arm64/App.framework.dSYM/Contents/Resources/DWARF/App') + .childFile( + 'arm64/App.framework.dSYM/Contents/Resources/DWARF/App') .path, environment.buildDir - .childFile('x86_64/App.framework.dSYM/Contents/Resources/DWARF/App') + .childFile( + 'x86_64/App.framework.dSYM/Contents/Resources/DWARF/App') .path, '-create', '-output', - environment.buildDir.childFile('App.framework.dSYM/Contents/Resources/DWARF/App').path, + environment.buildDir + .childFile('App.framework.dSYM/Contents/Resources/DWARF/App') + .path, ], ), ]); diff --git a/packages/flutter_tools/test/general.shard/cache_test.dart b/packages/flutter_tools/test/general.shard/cache_test.dart index 4a6152961b800..08bd0a02c62d9 100644 --- a/packages/flutter_tools/test/general.shard/cache_test.dart +++ b/packages/flutter_tools/test/general.shard/cache_test.dart @@ -108,7 +108,8 @@ void main() { final Directory artifactDir = fileSystem.systemTempDirectory.createTempSync( 'flutter_cache_test_artifact.', ); - final Directory downloadDir = fileSystem.systemTempDirectory.createTempSync( + final Directory downloadDir = + fileSystem.systemTempDirectory.createTempSync( 'flutter_cache_test_download.', ); final Cache cache = FakeSecondaryCache() @@ -131,7 +132,8 @@ void main() { final Directory artifactDir = fileSystem.systemTempDirectory.createTempSync( 'flutter_cache_test_artifact.', ); - final Directory downloadDir = fileSystem.systemTempDirectory.createTempSync( + final Directory downloadDir = + fileSystem.systemTempDirectory.createTempSync( 'flutter_cache_test_download.', ); final Cache cache = FakeSecondaryCache() @@ -143,7 +145,8 @@ void main() { final artifact = FakeSimpleArtifact(cache); await artifact.update(FakeArtifactUpdater(), logger, fileSystem, FakeOperatingSystemUtils()); - expect(logger.warningText, contains('No known version for the artifact name "fake"')); + expect(logger.warningText, + contains('No known version for the artifact name "fake"')); }); testWithoutContext( @@ -156,16 +159,20 @@ void main() { fileSystem.path.join('artifacts', 'gradle_wrapper'), ); fileSystem - .file(fileSystem.path.join(directory.path, 'gradle', 'wrapper', 'gradle-wrapper.jar')) + .file(fileSystem.path.join( + directory.path, 'gradle', 'wrapper', 'gradle-wrapper.jar')) .createSync(recursive: true); expect(gradleWrapper.isUpToDateInner(fileSystem), false); }, ); - testWithoutContext('Gradle wrapper will delete .properties/NOTICES if they exist', () async { + testWithoutContext( + 'Gradle wrapper will delete .properties/NOTICES if they exist', + () async { final FileSystem fileSystem = MemoryFileSystem.test(); - final Directory artifactDir = fileSystem.systemTempDirectory.createTempSync( + final Directory artifactDir = + fileSystem.systemTempDirectory.createTempSync( 'flutter_cache_test_artifact.', ); final cache = FakeSecondaryCache() @@ -180,12 +187,15 @@ void main() { ); final gradleWrapper = GradleWrapper(cache); final File propertiesFile = fileSystem.file( - fileSystem.path.join(artifactDir.path, 'gradle', 'wrapper', 'gradle-wrapper.properties'), + fileSystem.path.join( + artifactDir.path, 'gradle', 'wrapper', 'gradle-wrapper.properties'), )..createSync(recursive: true); - final File noticeFile = fileSystem.file(fileSystem.path.join(artifactDir.path, 'NOTICE')) + final File noticeFile = fileSystem + .file(fileSystem.path.join(artifactDir.path, 'NOTICE')) ..createSync(recursive: true); - await gradleWrapper.updateInner(FakeArtifactUpdater(), fileSystem, operatingSystemUtils); + await gradleWrapper.updateInner( + FakeArtifactUpdater(), fileSystem, operatingSystemUtils); expect(propertiesFile, isNot(exists)); expect(noticeFile, isNot(exists)); @@ -201,7 +211,8 @@ void main() { fileSystem.path.join('artifacts', 'gradle_wrapper'), ); fileSystem - .file(fileSystem.path.join(directory.path, 'gradle', 'wrapper', 'gradle-wrapper.jar')) + .file(fileSystem.path.join( + directory.path, 'gradle', 'wrapper', 'gradle-wrapper.jar')) .createSync(recursive: true); fileSystem .file(fileSystem.path.join(directory.path, 'gradlew')) @@ -214,9 +225,12 @@ void main() { }, ); - testWithoutContext('should not be up to date, if some cached artifact is not', () async { - final CachedArtifact artifact1 = FakeSecondaryCachedArtifact()..upToDate = true; - final CachedArtifact artifact2 = FakeSecondaryCachedArtifact()..upToDate = false; + testWithoutContext( + 'should not be up to date, if some cached artifact is not', () async { + final CachedArtifact artifact1 = FakeSecondaryCachedArtifact() + ..upToDate = true; + final CachedArtifact artifact2 = FakeSecondaryCachedArtifact() + ..upToDate = false; final FileSystem fileSystem = MemoryFileSystem.test(); final cache = Cache.test( @@ -252,7 +266,8 @@ void main() { processManager: FakeProcessManager.any(), ); - await cache.updateAll({DevelopmentArtifact.universal}); + await cache + .updateAll({DevelopmentArtifact.universal}); expect(artifact1.didUpdate, false); expect(artifact2.didUpdate, true); }); @@ -299,7 +314,8 @@ void main() { logger: logger, ); await expectLater( - () => cache.updateAll({DevelopmentArtifact.universal}), + () => cache + .updateAll({DevelopmentArtifact.universal}), throwsException, ); expect(artifact1.didUpdate, true); @@ -311,7 +327,9 @@ void main() { testWithoutContext('Invalid URI for FLUTTER_STORAGE_BASE_URL throws ToolExit', () async { final cache = Cache.test( platform: FakePlatform( - environment: {'FLUTTER_STORAGE_BASE_URL': ' http://foo'}, + environment: { + 'FLUTTER_STORAGE_BASE_URL': ' http://foo' + }, ), processManager: FakeProcessManager.any(), ); @@ -329,7 +347,8 @@ void main() { ); expect(cache.storageBaseUrl, baseUrl); - expect(logger.warningText, contains('Flutter assets will be downloaded from $baseUrl')); + expect(logger.warningText, + contains('Flutter assets will be downloaded from $baseUrl')); expect(logger.statusText, isEmpty); }); @@ -354,15 +373,18 @@ void main() { testWithoutContext('flattenNameSubdirs', () { expect( - flattenNameSubdirs(Uri.parse('http://flutter.dev/foo/bar'), MemoryFileSystem.test()), + flattenNameSubdirs( + Uri.parse('http://flutter.dev/foo/bar'), MemoryFileSystem.test()), 'flutter.dev/foo/bar', ); expect( - flattenNameSubdirs(Uri.parse('http://api.flutter.dev/foo/bar'), MemoryFileSystem.test()), + flattenNameSubdirs( + Uri.parse('http://api.flutter.dev/foo/bar'), MemoryFileSystem.test()), 'api.flutter.dev/foo/bar', ); expect( - flattenNameSubdirs(Uri.parse('https://www.flutter.dev'), MemoryFileSystem.test()), + flattenNameSubdirs( + Uri.parse('https://www.flutter.dev'), MemoryFileSystem.test()), 'www.flutter.dev', ); }); @@ -372,10 +394,12 @@ void main() { () async { final operatingSystemUtils = FakeOperatingSystemUtils(); final FileSystem fileSystem = MemoryFileSystem.test(); - final Directory artifactDir = fileSystem.systemTempDirectory.createTempSync( + final Directory artifactDir = + fileSystem.systemTempDirectory.createTempSync( 'flutter_cache_test_artifact.', ); - final Directory downloadDir = fileSystem.systemTempDirectory.createTempSync( + final Directory downloadDir = + fileSystem.systemTempDirectory.createTempSync( 'flutter_cache_test_download.', ); final cache = FakeSecondaryCache() @@ -400,7 +424,10 @@ void main() { expect(dir, isNotNull); expect(dir.path, artifactDir.childDirectory('bin_dir').path); expect(operatingSystemUtils.chmods, >[ - ['/.tmp_rand0/flutter_cache_test_artifact.rand0/bin_dir', 'a+r,a+x'], + [ + '/.tmp_rand0/flutter_cache_test_artifact.rand0/bin_dir', + 'a+r,a+x' + ], ]); }, ); @@ -433,7 +460,8 @@ void main() { packageUrl = url; }; - await artifact.updateInner(artifactUpdater, fileSystem, operatingSystemUtils); + await artifact.updateInner( + artifactUpdater, fileSystem, operatingSystemUtils); expect(packageUrl, isNotNull); expect( packageUrl.toString(), @@ -483,8 +511,8 @@ void main() { platform: FakePlatform(operatingSystem: 'macos'), ); iosUsbArtifacts.location.createSync(); - final File ideviceScreenshotFile = iosUsbArtifacts.location.childFile('idevicescreenshot') - ..createSync(); + final File ideviceScreenshotFile = + iosUsbArtifacts.location.childFile('idevicescreenshot')..createSync(); iosUsbArtifacts.location.childFile('idevicesyslog').createSync(); expect(iosUsbArtifacts.isUpToDateInner(fileSystem), true); @@ -504,7 +532,8 @@ void main() { platform: FakePlatform(operatingSystem: 'macos'), ); iosUsbArtifacts.location.createSync(); - final File iproxy = iosUsbArtifacts.location.childFile('iproxy')..createSync(); + final File iproxy = iosUsbArtifacts.location.childFile('iproxy') + ..createSync(); expect(iosUsbArtifacts.isUpToDateInner(fileSystem), true); @@ -549,10 +578,13 @@ void main() { platform: FakePlatform(operatingSystem: 'macos'), ); - expect(iosUsbArtifacts.archiveUri.toString(), isNot(contains('/unsigned/'))); + expect( + iosUsbArtifacts.archiveUri.toString(), isNot(contains('/unsigned/'))); }); - testWithoutContext('FlutterRunnerDebugSymbols downloads Flutter runner debug symbols', () async { + testWithoutContext( + 'FlutterRunnerDebugSymbols downloads Flutter runner debug symbols', + () async { final FileSystem fileSystem = MemoryFileSystem.test(); final Cache cache = FakeSecondaryCache() ..artifactDirectory = fileSystem.currentDirectory @@ -623,7 +655,8 @@ void main() { testWithoutContext('FontSubset artifacts on macos', () { fakeProcessManager.addCommands([ - const FakeCommand(command: ['which', 'sysctl'], stdout: '/sbin/sysctl'), + const FakeCommand( + command: ['which', 'sysctl'], stdout: '/sbin/sysctl'), const FakeCommand( command: ['sysctl', 'hw.optional.arm64'], stdout: 'hw.optional.arm64: 0', @@ -672,7 +705,8 @@ void main() { ]); }); - testWithoutContext('FontSubset artifacts for all platforms on arm64 hosts', () { + testWithoutContext('FontSubset artifacts for all platforms on arm64 hosts', + () { fakeProcessManager.addCommand(unameCommandForArm64); final Cache cache = createCache(FakePlatform(operatingSystem: 'fuchsia')); @@ -732,11 +766,15 @@ void main() { expect( artifacts.getBinaryDirs(), - containsAll([contains(contains('profile')), contains(contains('release'))]), + containsAll([ + contains(contains('profile')), + contains(contains('release')) + ]), ); }); - testWithoutContext('Linux desktop artifacts ignore filtering when requested', () { + testWithoutContext('Linux desktop artifacts ignore filtering when requested', + () { fakeProcessManager.addCommand(unameCommandForX64); final Cache cache = createCache(FakePlatform()); @@ -747,7 +785,9 @@ void main() { expect(artifacts.getBinaryDirs(), isNotEmpty); }); - testWithoutContext('Linux desktop artifacts for x64 include profile and release artifacts', () { + testWithoutContext( + 'Linux desktop artifacts for x64 include profile and release artifacts', + () { fakeProcessManager.addCommand(unameCommandForX64); final Cache cache = createCache(FakePlatform()); @@ -755,12 +795,20 @@ void main() { expect(artifacts.getBinaryDirs(), >[ ['linux-x64', 'linux-x64-debug/linux-x64-flutter-gtk.zip'], - ['linux-x64-profile', 'linux-x64-profile/linux-x64-flutter-gtk.zip'], - ['linux-x64-release', 'linux-x64-release/linux-x64-flutter-gtk.zip'], + [ + 'linux-x64-profile', + 'linux-x64-profile/linux-x64-flutter-gtk.zip' + ], + [ + 'linux-x64-release', + 'linux-x64-release/linux-x64-flutter-gtk.zip' + ], ]); }); - testWithoutContext('Linux desktop artifacts for arm64 include profile and release artifacts', () { + testWithoutContext( + 'Linux desktop artifacts for arm64 include profile and release artifacts', + () { fakeProcessManager.addCommand(unameCommandForArm64); final Cache cache = createCache(FakePlatform()); @@ -768,8 +816,14 @@ void main() { expect(artifacts.getBinaryDirs(), >[ ['linux-arm64', 'linux-arm64-debug/linux-arm64-flutter-gtk.zip'], - ['linux-arm64-profile', 'linux-arm64-profile/linux-arm64-flutter-gtk.zip'], - ['linux-arm64-release', 'linux-arm64-release/linux-arm64-flutter-gtk.zip'], + [ + 'linux-arm64-profile', + 'linux-arm64-profile/linux-arm64-flutter-gtk.zip' + ], + [ + 'linux-arm64-release', + 'linux-arm64-release/linux-arm64-flutter-gtk.zip' + ], ]); }); @@ -799,7 +853,8 @@ void main() { expect(toolStampFile, isNot(exists)); }); - testWithoutContext('Cache does not attempt to delete already missing stamp files', () { + testWithoutContext( + 'Cache does not attempt to delete already missing stamp files', () { final FileSystem fileSystem = MemoryFileSystem.test(); final artifactSet = FakeIosUsbArtifacts(); final logger = BufferLogger.test(); @@ -824,7 +879,8 @@ void main() { expect(toolStampFile, isNot(exists)); }); - testWithoutContext('Cache catches file system exception from missing tool stamp file', () { + testWithoutContext( + 'Cache catches file system exception from missing tool stamp file', () { final FileSystem fileSystem = MemoryFileSystem.test(); final artifactSet = FakeIosUsbArtifacts(); final logger = BufferLogger.test(); @@ -851,7 +907,8 @@ void main() { final Directory internalDir = fileSystem.currentDirectory .childDirectory('bin') .childDirectory('internal'); - final File canvasKitVersionFile = internalDir.childFile('canvaskit.version'); + final File canvasKitVersionFile = + internalDir.childFile('canvaskit.version'); canvasKitVersionFile.createSync(recursive: true); canvasKitVersionFile.writeAsStringSync('abcdefg'); @@ -879,7 +936,8 @@ void main() { }; webCacheDirectory.childFile('bar').createSync(recursive: true); - await webSdk.updateInner(artifactUpdater, fileSystem, FakeOperatingSystemUtils()); + await webSdk.updateInner( + artifactUpdater, fileSystem, FakeOperatingSystemUtils()); expect(messages, ['Downloading Web SDK...']); @@ -901,7 +959,8 @@ void main() { final Directory internalDir = fileSystem.currentDirectory .childDirectory('bin') .childDirectory('internal'); - final File canvasKitVersionFile = internalDir.childFile('canvaskit.version'); + final File canvasKitVersionFile = + internalDir.childFile('canvaskit.version'); canvasKitVersionFile.createSync(recursive: true); canvasKitVersionFile.writeAsStringSync('abcdefg'); @@ -935,7 +994,8 @@ void main() { }; webCacheDirectory.childFile('bar').createSync(recursive: true); - await webSdk.updateInner(artifactUpdater, fileSystem, FakeOperatingSystemUtils()); + await webSdk.updateInner( + artifactUpdater, fileSystem, FakeOperatingSystemUtils()); expect(downloads, [ 'https://flutter.storage.com/override/flutter_infra_release/flutter/hijklmnop/flutter-web-sdk.zip', @@ -970,9 +1030,11 @@ void main() { ); await expectLater( - () => webSdk.updateInner(artifactUpdater, fileSystem, FakeOperatingSystemUtils()), + () => webSdk.updateInner( + artifactUpdater, fileSystem, FakeOperatingSystemUtils()), throwsToolExit( - message: RegExp('Unable to delete file or directory at "/bin/cache/flutter_web_sdk"'), + message: RegExp( + 'Unable to delete file or directory at "/bin/cache/flutter_web_sdk"'), ), ); }); @@ -1018,7 +1080,8 @@ void main() { final fileSystem = MemoryFileSystem.test(opHandle: handler.opHandle); final cache = Cache.test(processManager: FakeProcessManager.any(), fileSystem: fileSystem); final File canvasKitWasm = fileSystem.file( - fileSystem.path.join(cache.getRoot().path, 'canvaskit', 'canvaskit.wasm'), + fileSystem.path + .join(cache.getRoot().path, 'canvaskit', 'canvaskit.wasm'), ); canvasKitWasm.createSync(recursive: true); canvasKitWasm.writeAsStringSync('hello world'); @@ -1053,7 +1116,8 @@ void main() { expect(cache.getStampFor('foo'), null); file.createSync(); - exceptionHandler.addError(file, FileSystemOp.read, const FileSystemException()); + exceptionHandler.addError( + file, FileSystemOp.read, const FileSystemException()); expect(cache.getStampFor('foo'), null); }); @@ -1074,7 +1138,8 @@ void main() { expect(cache.getStampFor('foo'), 'ABC'); }); - testWithoutContext('PubDependencies needs to be updated if the package config' + testWithoutContext( + 'PubDependencies needs to be updated if the package config' ' file or the source directories are missing', () async { final logger = BufferLogger.test(); final fileSystem = MemoryFileSystem.test(); @@ -1085,7 +1150,8 @@ void main() { projectFactory: FakeFlutterProjectFactory(), ); - expect(await pubDependencies.isUpToDate(fileSystem), false); // no package config + expect(await pubDependencies.isUpToDate(fileSystem), + false); // no package config fileSystem.file('packages/flutter_tools/.dart_tool/package_config.json') ..createSync(recursive: true) @@ -1106,7 +1172,8 @@ void main() { } '''); - expect(await pubDependencies.isUpToDate(fileSystem), false); // dependencies are missing. + expect(await pubDependencies.isUpToDate(fileSystem), + false); // dependencies are missing. fileSystem .file('.pub-cache/hosted/pub.dartlang.org/example-7.0.0/pubspec.yaml') @@ -1137,7 +1204,8 @@ void main() { expect( pub.invocations.first, predicate( - (FakePubInvocation invocation) => invocation.outputMode == PubOutputMode.failuresOnly, + (FakePubInvocation invocation) => + invocation.outputMode == PubOutputMode.failuresOnly, 'Pub invoked with PubOutputMode.none', ), ); @@ -1165,7 +1233,9 @@ void main() { setUp(() { memoryFileSystem = MemoryFileSystem.test(); - cache = Cache.test(fileSystem: memoryFileSystem, processManager: FakeProcessManager.any()); + cache = Cache.test( + fileSystem: memoryFileSystem, + processManager: FakeProcessManager.any()); fakeAndroidSdk = FakeAndroidSdk(); }); @@ -1175,7 +1245,8 @@ void main() { java: FakeJava(), platform: FakePlatform(), ); - expect(mavenArtifacts.developmentArtifact, DevelopmentArtifact.androidMaven); + expect( + mavenArtifacts.developmentArtifact, DevelopmentArtifact.androidMaven); }); testUsingContext( @@ -1191,10 +1262,13 @@ void main() { ); expect(await mavenArtifacts.isUpToDate(memoryFileSystem!), isFalse); - final Directory gradleWrapperDir = cache!.getArtifactDirectory('gradle_wrapper') + final Directory gradleWrapperDir = cache! + .getArtifactDirectory('gradle_wrapper') ..createSync(recursive: true); gradleWrapperDir.childFile('gradlew').writeAsStringSync('irrelevant'); - gradleWrapperDir.childFile('gradlew.bat').writeAsStringSync('irrelevant'); + gradleWrapperDir + .childFile('gradlew.bat') + .writeAsStringSync('irrelevant'); await mavenArtifacts.update( FakeArtifactUpdater(), @@ -1283,7 +1357,8 @@ class FakeCachedArtifact extends EngineCachedArtifact { } class FakeSimpleArtifact extends CachedArtifact { - FakeSimpleArtifact(Cache cache) : super('fake', cache, DevelopmentArtifact.universal); + FakeSimpleArtifact(Cache cache) + : super('fake', cache, DevelopmentArtifact.universal); @override Future updateInner( @@ -1432,12 +1507,14 @@ class FakeArtifactUpdater extends Fake implements ArtifactUpdater { void Function(String, Uri, Directory)? onDownloadFile; @override - Future downloadZippedTarball(String message, Uri url, Directory location) async { + Future downloadZippedTarball( + String message, Uri url, Directory location) async { onDownloadZipTarball?.call(message, url, location); } @override - Future downloadZipArchive(String message, Uri url, Directory location) async { + Future downloadZipArchive( + String message, Uri url, Directory location) async { onDownloadZipArchive?.call(message, url, location); } diff --git a/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart b/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart new file mode 100644 index 0000000000000..602de37681555 --- /dev/null +++ b/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart @@ -0,0 +1,101 @@ +// Copyright 2024 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'package:flutter_tools/src/shorebird/shorebird_yaml.dart'; +import 'package:test/test.dart'; +import 'package:yaml/yaml.dart'; + +void main() { + group('ShorebirdYaml', () { + test('yaml ignores comments', () { + const String yamlContents = ''' +# This file is used to configure the Shorebird updater used by your app. +app_id: 6160a7d8-cc18-4928-1233-05b51c0bb02c + +# auto_update controls if Shorebird should automatically update in the background on launch. +auto_update: false +'''; + final YamlDocument input = loadYamlDocument(yamlContents); + final YamlMap yamlMap = input.contents as YamlMap; + final Map compiled = + compileShorebirdYaml(yamlMap, flavor: null, environment: {}); + expect(compiled, { + 'app_id': '6160a7d8-cc18-4928-1233-05b51c0bb02c', + 'auto_update': false, + }); + }); + test('flavors', () { + // These are invalid app_ids but make for easy testing. + const String yamlContents = ''' +app_id: 1-a +auto_update: false +flavors: + foo: 2-a + bar: 3-a +'''; + final YamlDocument input = loadYamlDocument(yamlContents); + final YamlMap yamlMap = input.contents as YamlMap; + expect(appIdForFlavor(yamlMap, flavor: null), '1-a'); + expect(appIdForFlavor(yamlMap, flavor: 'foo'), '2-a'); + expect(appIdForFlavor(yamlMap, flavor: 'bar'), '3-a'); + expect(() => appIdForFlavor(yamlMap, flavor: 'unknown'), throwsException); + }); + test('all values', () { + // These are invalid app_ids but make for easy testing. + const String yamlContents = ''' +app_id: 1-a +auto_update: false +flavors: + foo: 2-a + bar: 3-a +base_url: https://example.com +'''; + final YamlDocument input = loadYamlDocument(yamlContents); + final YamlMap yamlMap = input.contents as YamlMap; + final Map compiled1 = + compileShorebirdYaml(yamlMap, flavor: null, environment: {}); + expect(compiled1, { + 'app_id': '1-a', + 'auto_update': false, + 'base_url': 'https://example.com', + }); + final Map compiled2 = + compileShorebirdYaml(yamlMap, flavor: 'foo', environment: {'SHOREBIRD_PUBLIC_KEY': '4-a'}); + expect(compiled2, { + 'app_id': '2-a', + 'auto_update': false, + 'base_url': 'https://example.com', + 'patch_public_key': '4-a', + }); + }); + test('edit in place', () { + const String yamlContents = ''' +app_id: 1-a +auto_update: false +flavors: + foo: 2-a + bar: 3-a +base_url: https://example.com +'''; + // Make a temporary file to test editing in place. + final Directory tempDir = Directory.systemTemp.createTempSync('shorebird_yaml_test.'); + final File tempFile = File('${tempDir.path}/shorebird.yaml'); + tempFile.writeAsStringSync(yamlContents); + updateShorebirdYaml( + 'foo', + tempFile.path, + environment: {'SHOREBIRD_PUBLIC_KEY': '4-a'}, + ); + final String updatedContents = tempFile.readAsStringSync(); + // Order is not guaranteed, so parse as YAML to compare. + final YamlDocument updated = loadYamlDocument(updatedContents); + final YamlMap yamlMap = updated.contents as YamlMap; + expect(yamlMap['app_id'], '2-a'); + expect(yamlMap['auto_update'], false); + expect(yamlMap['base_url'], 'https://example.com'); + }); + }); +} diff --git a/packages/shorebird_tests/.gitignore b/packages/shorebird_tests/.gitignore new file mode 100644 index 0000000000000..3a85790408401 --- /dev/null +++ b/packages/shorebird_tests/.gitignore @@ -0,0 +1,3 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ diff --git a/packages/shorebird_tests/README.md b/packages/shorebird_tests/README.md new file mode 100644 index 0000000000000..b87fb24a480da --- /dev/null +++ b/packages/shorebird_tests/README.md @@ -0,0 +1,2 @@ +A dart project that includes tests that perform asserts in the modifications +made on the Flutter framework by the Shorebird team. diff --git a/packages/shorebird_tests/analysis_options.yaml b/packages/shorebird_tests/analysis_options.yaml new file mode 100644 index 0000000000000..a767d79d7f4b1 --- /dev/null +++ b/packages/shorebird_tests/analysis_options.yaml @@ -0,0 +1,2 @@ +# This file configures the static analysis results for your project (errors, +include: package:lints/recommended.yaml diff --git a/packages/shorebird_tests/pubspec.yaml b/packages/shorebird_tests/pubspec.yaml new file mode 100644 index 0000000000000..d0fba0c1fa95d --- /dev/null +++ b/packages/shorebird_tests/pubspec.yaml @@ -0,0 +1,16 @@ +name: shorebird_tests +description: Shorebird's Flutter customizations tests +version: 1.0.0 + +environment: + sdk: ^3.3.4 + +dependencies: + archive: ^3.5.1 + path: ^1.9.0 + yaml: ^3.1.2 + +dev_dependencies: + lints: ^3.0.0 + meta: ^1.15.0 + test: ^1.24.0 diff --git a/packages/shorebird_tests/test/android_test.dart b/packages/shorebird_tests/test/android_test.dart new file mode 100644 index 0000000000000..b24cce0ee9bf6 --- /dev/null +++ b/packages/shorebird_tests/test/android_test.dart @@ -0,0 +1,92 @@ +import 'package:test/test.dart'; + +import 'shorebird_tests.dart'; + +void main() { + group('shorebird android projects', () { + testWithShorebirdProject('can build an apk', (projectDirectory) async { + await projectDirectory.runFlutterBuildApk(); + + expect(projectDirectory.apkFile().existsSync(), isTrue); + expect(projectDirectory.shorebirdFile.existsSync(), isTrue); + expect(projectDirectory.getGeneratedAndroidShorebirdYaml(), completes); + }); + + group('when passing the public key through the environment variable', () { + testWithShorebirdProject( + 'adds the public key on top of the original file', + (projectDirectory) async { + final originalYaml = projectDirectory.shorebirdYaml; + + const base64PublicKey = 'public_123'; + await projectDirectory.runFlutterBuildApk( + environment: { + 'SHOREBIRD_PUBLIC_KEY': base64PublicKey, + }, + ); + + final generatedYaml = + await projectDirectory.getGeneratedAndroidShorebirdYaml(); + + expect( + generatedYaml.keys, + containsAll(originalYaml.keys), + ); + + expect( + generatedYaml['patch_public_key'], + equals(base64PublicKey), + ); + }, + ); + }); + + group('when building with a flavor', () { + testWithShorebirdProject( + 'correctly changes the app id', + (projectDirectory) async { + await projectDirectory.addProjectFlavors(); + projectDirectory.addShorebirdFlavors(); + + await projectDirectory.runFlutterBuildApk(flavor: 'internal'); + + final generatedYaml = + await projectDirectory.getGeneratedAndroidShorebirdYaml( + flavor: 'internal', + ); + + expect(generatedYaml['app_id'], equals('internal_123')); + }, + ); + + group('when public key passed through environment variable', () { + testWithShorebirdProject( + 'correctly changes the app id and adds the public key', + (projectDirectory) async { + const base64PublicKey = 'public_123'; + await projectDirectory.addProjectFlavors(); + projectDirectory.addShorebirdFlavors(); + + await projectDirectory.runFlutterBuildApk( + flavor: 'internal', + environment: { + 'SHOREBIRD_PUBLIC_KEY': base64PublicKey, + }, + ); + + final generatedYaml = + await projectDirectory.getGeneratedAndroidShorebirdYaml( + flavor: 'internal', + ); + + expect(generatedYaml['app_id'], equals('internal_123')); + expect( + generatedYaml['patch_public_key'], + equals(base64PublicKey), + ); + }, + ); + }); + }); + }); +} diff --git a/packages/shorebird_tests/test/base_test.dart b/packages/shorebird_tests/test/base_test.dart new file mode 100644 index 0000000000000..14dba7ca6cc85 --- /dev/null +++ b/packages/shorebird_tests/test/base_test.dart @@ -0,0 +1,15 @@ +import 'package:test/test.dart'; + +import 'shorebird_tests.dart'; + +void main() { + group('shorebird helpers', () { + testWithShorebirdProject('can build a base project', + (projectDirectory) async { + expect(projectDirectory.existsSync(), isTrue); + + expect(projectDirectory.pubspecFile.existsSync(), isTrue); + expect(projectDirectory.shorebirdFile.existsSync(), isTrue); + }); + }); +} diff --git a/packages/shorebird_tests/test/ios_test.dart b/packages/shorebird_tests/test/ios_test.dart new file mode 100644 index 0000000000000..3fe6e1652c3e5 --- /dev/null +++ b/packages/shorebird_tests/test/ios_test.dart @@ -0,0 +1,91 @@ +import 'package:test/test.dart'; + +import 'shorebird_tests.dart'; + +void main() { + group( + 'shorebird ios projects', + () { + testWithShorebirdProject('can build', (projectDirectory) async { + await projectDirectory.runFlutterBuildIos(); + + expect(projectDirectory.iosArchiveFile().existsSync(), isTrue); + expect(projectDirectory.getGeneratedIosShorebirdYaml(), completes); + }); + + group('when passing the public key through the environment variable', () { + testWithShorebirdProject( + 'adds the public key on top of the original file', + (projectDirectory) async { + final originalYaml = projectDirectory.shorebirdYaml; + + const base64PublicKey = 'public_123'; + await projectDirectory.runFlutterBuildIos( + environment: { + 'SHOREBIRD_PUBLIC_KEY': base64PublicKey, + }, + ); + + final generatedYaml = + await projectDirectory.getGeneratedIosShorebirdYaml(); + + expect( + generatedYaml.keys, + containsAll(originalYaml.keys), + ); + + expect( + generatedYaml['patch_public_key'], + equals(base64PublicKey), + ); + }, + ); + }); + + group('when building with a flavor', () { + testWithShorebirdProject( + 'correctly changes the app id', + (projectDirectory) async { + await projectDirectory.addProjectFlavors(); + projectDirectory.addShorebirdFlavors(); + + await projectDirectory.runFlutterBuildIos(flavor: 'internal'); + + final generatedYaml = + await projectDirectory.getGeneratedIosShorebirdYaml(); + + expect(generatedYaml['app_id'], equals('internal_123')); + }, + ); + + group('when public key passed through environment variable', () { + testWithShorebirdProject( + 'correctly changes the app id and adds the public key', + (projectDirectory) async { + const base64PublicKey = 'public_123'; + await projectDirectory.addProjectFlavors(); + projectDirectory.addShorebirdFlavors(); + + await projectDirectory.runFlutterBuildIos( + flavor: 'internal', + environment: { + 'SHOREBIRD_PUBLIC_KEY': base64PublicKey, + }, + ); + + final generatedYaml = + await projectDirectory.getGeneratedIosShorebirdYaml(); + + expect(generatedYaml['app_id'], equals('internal_123')); + expect( + generatedYaml['patch_public_key'], + equals(base64PublicKey), + ); + }, + ); + }); + }); + }, + testOn: 'mac-os', + ); +} diff --git a/packages/shorebird_tests/test/shorebird_tests.dart b/packages/shorebird_tests/test/shorebird_tests.dart new file mode 100644 index 0000000000000..4bead2de4503e --- /dev/null +++ b/packages/shorebird_tests/test/shorebird_tests.dart @@ -0,0 +1,281 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:archive/archive_io.dart'; +import 'package:path/path.dart' as path; + +import 'package:meta/meta.dart'; +import 'package:test/test.dart'; +import 'package:yaml/yaml.dart'; + +/// This will be the path to the flutter binary housed in this flutter repository. +/// +/// Which since we are running the tests from this inner package , we need to go up two directories +/// in order to find the flutter binary in the bin folder. +File get _flutterBinaryFile => File( + path.join( + Directory.current.path, + '..', + '..', + 'bin', + 'flutter${Platform.isWindows ? '.bat' : ''}', + ), + ); + +/// Runs a flutter command using the correct binary ([_flutterBinaryFile]) with the given arguments. +Future _runFlutterCommand( + List arguments, { + required Directory workingDirectory, + Map? environment, +}) { + return Process.run( + _flutterBinaryFile.absolute.path, + arguments, + workingDirectory: workingDirectory.path, + environment: { + 'FLUTTER_STORAGE_BASE_URL': 'https://download.shorebird.dev', + if (environment != null) ...environment, + }, + ); +} + +Future _createFlutterProject(Directory projectDirectory) async { + final result = await _runFlutterCommand( + ['create', '--empty', '.'], + workingDirectory: projectDirectory, + ); + if (result.exitCode != 0) { + throw Exception('Failed to create Flutter project: ${result.stderr}'); + } +} + +@isTest +Future testWithShorebirdProject(String name, + FutureOr Function(Directory projectDirectory) testFn) async { + test( + name, + () async { + final parentDirectory = Directory.systemTemp.createTempSync(); + final projectDirectory = Directory( + path.join( + parentDirectory.path, + 'shorebird_test', + ), + )..createSync(); + + try { + await _createFlutterProject(projectDirectory); + + projectDirectory.pubspecFile.writeAsStringSync(''' +${projectDirectory.pubspecFile.readAsStringSync()} + assets: + - shorebird.yaml +'''); + + File( + path.join( + projectDirectory.path, + 'shorebird.yaml', + ), + ).writeAsStringSync(''' +app_id: "123" +'''); + + await testFn(projectDirectory); + } finally { + projectDirectory.deleteSync(recursive: true); + } + }, + timeout: Timeout( + // These tests usually run flutter create, flutter build, etc, which can take a while, + // specially in CI, so setting from the default of 30 seconds to 6 minutes. + Duration(minutes: 6), + ), + ); +} + +extension ShorebirdProjectDirectoryOnDirectory on Directory { + File get pubspecFile => File( + path.join(this.path, 'pubspec.yaml'), + ); + + File get shorebirdFile => File( + path.join(this.path, 'shorebird.yaml'), + ); + + YamlMap get shorebirdYaml => + loadYaml(shorebirdFile.readAsStringSync()) as YamlMap; + + File get appGradleFile => File( + path.join(this.path, 'android', 'app', 'build.gradle'), + ); + + Future addPubDependency(String name, {bool dev = false}) async { + final result = await _runFlutterCommand( + ['pub', 'add', if (dev) '--dev', name], + workingDirectory: this, + ); + if (result.exitCode != 0) { + throw Exception( + 'Failed to run `flutter pub add $name`: ${result.stderr}'); + } + } + + Future addProjectFlavors() async { + await addPubDependency( + // TODO(felangel): revert to using published version once 3.29.0 support is released. + // https://github.com/AngeloAvv/flutter_flavorizr/pull/291 + 'dev:flutter_flavorizr:{"git":{"url":"https://github.com/wjlee611/flutter_flavorizr.git","ref":"chore/temp-migrate-3-29","path":"."}}', + ); + + await File( + path.join( + this.path, + 'flavorizr.yaml', + ), + ).writeAsString(''' +flavors: + playStore: + app: + name: "App" + + android: + applicationId: "com.example.shorebird_test" + ios: + bundleId: "com.example.shorebird_test" + internal: + app: + name: "App (Internal)" + + android: + applicationId: "com.example.shorebird_test.internal" + ios: + bundleId: "com.example.shorebird_test.internal" + global: + app: + name: "App (Global)" + + android: + applicationId: "com.example.shorebird_test.global" + ios: + bundleId: "com.example.shorebird_test.global" +'''); + + final result = await _runFlutterCommand( + ['pub', 'run', 'flutter_flavorizr'], + workingDirectory: this, + ); + if (result.exitCode != 0) { + throw Exception( + 'Failed to run `flutter pub run flutter_flavorizr`: ${result.stderr}'); + } + } + + void addShorebirdFlavors() { + const flavors = ''' +flavors: + global: global_123 + internal: internal_123 + playStore: playStore_123 +'''; + + final currentShorebirdContent = shorebirdFile.readAsStringSync(); + shorebirdFile.writeAsStringSync( + ''' +$currentShorebirdContent +$flavors +''', + ); + } + + Future runFlutterBuildApk({ + String? flavor, + Map? environment, + }) async { + final result = await _runFlutterCommand( + [ + 'build', + 'apk', + if (flavor != null) '--flavor=$flavor', + ], + workingDirectory: this, + environment: environment, + ); + if (result.exitCode != 0) { + throw Exception('Failed to run `flutter build apk`: ${result.stderr}'); + } + } + + Future runFlutterBuildIos({ + Map? environment, + String? flavor, + }) async { + final result = await _runFlutterCommand( + // The projects used to test are generated on spot, to make it simpler we don't + // configure any apple accounts on it, so we skip code signing here. + ['build', 'ipa', '--no-codesign', if (flavor != null) '--flavor=$flavor'], + workingDirectory: this, + environment: environment, + ); + + if (result.exitCode != 0) { + throw Exception('Failed to run `flutter build ios`: ${result.stderr}'); + } + } + + File apkFile({String? flavor}) => File( + path.join( + this.path, + 'build', + 'app', + 'outputs', + 'flutter-apk', + 'app-${flavor != null ? '$flavor-' : ''}release.apk', + ), + ); + + Directory iosArchiveFile() => Directory( + path.join( + this.path, + 'build', + 'ios', + 'archive', + 'Runner.xcarchive', + ), + ); + + Future getGeneratedAndroidShorebirdYaml({String? flavor}) async { + final decodedBytes = + ZipDecoder().decodeBytes(apkFile(flavor: flavor).readAsBytesSync()); + + await extractArchiveToDisk( + decodedBytes, path.join(this.path, 'apk-extracted')); + + final yamlString = File( + path.join( + this.path, + 'apk-extracted', + 'assets', + 'flutter_assets', + 'shorebird.yaml', + ), + ).readAsStringSync(); + return loadYaml(yamlString) as YamlMap; + } + + Future getGeneratedIosShorebirdYaml() async { + final yamlString = File( + path.join( + iosArchiveFile().path, + 'Products', + 'Applications', + 'Runner.app', + 'Frameworks', + 'App.framework', + 'flutter_assets', + 'shorebird.yaml', + ), + ).readAsStringSync(); + return loadYaml(yamlString) as YamlMap; + } +} From fc41139642685f52b5d160b74f67c92443be9499 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 14 Nov 2025 15:04:34 -0800 Subject: [PATCH 02/51] chore: run et format --- engine/src/flutter/BUILD.gn | 2 +- engine/src/flutter/shell/platform/windows/BUILD.gn | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/src/flutter/BUILD.gn b/engine/src/flutter/BUILD.gn index c8cbc6803dbe9..348f732c329ed 100644 --- a/engine/src/flutter/BUILD.gn +++ b/engine/src/flutter/BUILD.gn @@ -209,8 +209,8 @@ group("unittests") { "//flutter/runtime:no_dart_plugin_registrant_unittests", "//flutter/runtime:runtime_unittests", "//flutter/shell/common:shell_unittests", - "//flutter/shell/geometry:geometry_unittests", "//flutter/shell/common/shorebird:shorebird_unittests", + "//flutter/shell/geometry:geometry_unittests", "//flutter/shell/platform/embedder:embedder_a11y_unittests", "//flutter/shell/platform/embedder:embedder_proctable_unittests", "//flutter/shell/platform/embedder:embedder_unittests", diff --git a/engine/src/flutter/shell/platform/windows/BUILD.gn b/engine/src/flutter/shell/platform/windows/BUILD.gn index 4aeda70af9bd2..52690485316d2 100644 --- a/engine/src/flutter/shell/platform/windows/BUILD.gn +++ b/engine/src/flutter/shell/platform/windows/BUILD.gn @@ -169,7 +169,7 @@ source_set("flutter_windows_source") { ":flutter_windows_headers", "//flutter/fml", "//flutter/impeller/renderer/backend/gles", - "//flutter/shell/common/shorebird:shorebird", + "//flutter/shell/common/shorebird", "//flutter/shell/geometry", "//flutter/shell/platform/common:common_cpp", "//flutter/shell/platform/common:common_cpp_input", From 1ed7fefe74ed855730439a248853877cfeaab33f Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 14 Nov 2025 16:06:55 -0800 Subject: [PATCH 03/51] fix: update name to application_library_paths --- .../flutter/shell/common/shorebird/shorebird.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc index a2b2f382820a1..ec2bc6bf3c9cd 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.cc +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -144,7 +144,7 @@ bool ConfigureShorebird(const ShorebirdConfigArgs& args, // https://stackoverflow.com/questions/26032039/convert-vectorstring-into-char-c std::vector c_paths{}; c_paths.push_back(args.release_app_library_path.c_str()); - // Do not modify application_library_path or c_strings will invalidate. + // Do not modify application_library_paths or c_strings will invalidate. app_parameters.original_libapp_paths = c_paths.data(); app_parameters.original_libapp_paths_size = c_paths.size(); @@ -238,10 +238,10 @@ void ConfigureShorebird(std::string code_cache_path, // https://stackoverflow.com/questions/26032039/convert-vectorstring-into-char-c std::vector c_paths{}; - for (const auto& string : settings.application_library_path) { + for (const auto& string : settings.application_library_paths) { c_paths.push_back(string.c_str()); } - // Do not modify application_library_path or c_strings will invalidate. + // Do not modify application_library_paths or c_strings will invalidate. app_parameters.original_libapp_paths = c_paths.data(); app_parameters.original_libapp_paths_size = c_paths.size(); @@ -273,11 +273,11 @@ void ConfigureShorebird(std::string code_cache_path, // On iOS we add the patch to the front of the list instead of clearing // the list, to allow dart_snapshot.cc to still find the base snapshot // for the vm isolate. - settings.application_library_path.insert( - settings.application_library_path.begin(), active_path); + settings.application_library_paths.insert( + settings.application_library_paths.begin(), active_path); #else - settings.application_library_path.clear(); - settings.application_library_path.emplace_back(active_path); + settings.application_library_paths.clear(); + settings.application_library_paths.emplace_back(active_path); #endif } else { FML_LOG(INFO) << "Shorebird updater: no active patch."; From ff1258ba1bf6a88372ee53df7b9783eca443eb89 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 14 Nov 2025 17:18:37 -0800 Subject: [PATCH 04/51] fix: attempt to fix builds --- engine/src/flutter/runtime/dart_snapshot.cc | 2 +- .../flutter/shell/platform/windows/flutter_windows_engine.cc | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/engine/src/flutter/runtime/dart_snapshot.cc b/engine/src/flutter/runtime/dart_snapshot.cc index 9ab19ef4b7a67..d36e0730eb8c0 100644 --- a/engine/src/flutter/runtime/dart_snapshot.cc +++ b/engine/src/flutter/runtime/dart_snapshot.cc @@ -58,7 +58,7 @@ static std::shared_ptr SearchMapping( bool is_executable) { #if SHOREBIRD_USE_INTERPRETER // Detect when we're trying to load a Shorebird patch. - auto patch_path = native_library_path.front(); + auto patch_path = native_library_paths.front(); bool is_patch = patch_path.find(".vmcode") != std::string::npos; if (is_patch) { // We use this terrible hack to load in the patch and then extract the diff --git a/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc b/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc index 32a6a675e1cf2..22b028beb03f4 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc +++ b/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc @@ -395,9 +395,6 @@ bool SetUpShorebird(std::string assets_path_string, std::string& patch_path) { } bool FlutterWindowsEngine::Run(std::string_view entrypoint) { - std::string assets_path_string = project_->assets_path().u8string(); - std::string icu_path_string = project_->icu_path().u8string(); - if (!project_->HasValidPaths()) { FML_LOG(ERROR) << "Missing or unresolvable paths to assets."; return false; From 670445859569fadfe3e9106163ac95ea5a8bc570 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 17 Nov 2025 08:26:44 -0800 Subject: [PATCH 05/51] chore: add missing header for ios --- engine/src/flutter/runtime/dart_snapshot.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/engine/src/flutter/runtime/dart_snapshot.cc b/engine/src/flutter/runtime/dart_snapshot.cc index d36e0730eb8c0..28ccb1cc46141 100644 --- a/engine/src/flutter/runtime/dart_snapshot.cc +++ b/engine/src/flutter/runtime/dart_snapshot.cc @@ -6,6 +6,7 @@ #include +#include #include "flutter/fml/native_library.h" #include "flutter/fml/paths.h" #include "flutter/fml/trace_event.h" From e32768c8a6a76998ce6c2f4d401b4a682e7850ef Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 17 Nov 2025 10:45:57 -0800 Subject: [PATCH 06/51] fix: linux android build --- engine/src/build/config/compiler/BUILD.gn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/build/config/compiler/BUILD.gn b/engine/src/build/config/compiler/BUILD.gn index 70ebcf699bbbb..7df82c918b172 100644 --- a/engine/src/build/config/compiler/BUILD.gn +++ b/engine/src/build/config/compiler/BUILD.gn @@ -667,7 +667,7 @@ config("runtime_library") { } # libunwind.a is located in the respective android cpu subdirectories. # The clang version needs to match the version in the lib_dirs line above. - lib_dirs += [ "${android_toolchain_root}/lib/clang/18/lib/linux/${current_android_cpu}/" ] + lib_dirs += [ "${android_toolchain_root}/lib/clang/19/lib/linux/${current_android_cpu}/" ] libs += [ "clang_rt.builtins-${current_android_cpu}-android" ] } From a868599e093f971fff6b21545688f3659df849ed Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 17 Nov 2025 15:23:56 -0800 Subject: [PATCH 07/51] fix: attempt to fix shorebird flutter_tools changes --- .../lib/src/build_system/targets/assets.dart | 6 ++---- packages/flutter_tools/lib/src/version.dart | 16 +++++++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/packages/flutter_tools/lib/src/build_system/targets/assets.dart b/packages/flutter_tools/lib/src/build_system/targets/assets.dart index 409f37f994ca9..0b7be7e672975 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/assets.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/assets.dart @@ -168,12 +168,10 @@ Future copyAssets( updateShorebirdYaml( environment.defines[kFlavor], file.path, - environment: globals.platform.environment, + environment: environment.platform.environment, ); } on Exception catch (error) { - throw Exception( - 'Failed to generate shorebird configuration. Error: $error', - ); + throw Exception('Failed to generate shorebird configuration. Error: $error'); } } } diff --git a/packages/flutter_tools/lib/src/version.dart b/packages/flutter_tools/lib/src/version.dart index d7a4a765b38d9..e7c86a308cf32 100644 --- a/packages/flutter_tools/lib/src/version.dart +++ b/packages/flutter_tools/lib/src/version.dart @@ -1050,11 +1050,17 @@ class GitTagVersion { } // Check if running on a Shorebird release branch. - final String shorebirdFlutterReleases = _runGit( - 'git for-each-ref --contains $gitRef --format %(refname:short) refs/remotes/origin/flutter_release/*', - processUtils, - workingDirectory, - ).trim(); + final String shorebirdFlutterReleases = git + .runSync([ + 'for-each-ref', + '--contains', + gitRef, + '--format', + '%(refname:short)', + 'refs/remotes/origin/flutter_release/*', + ], workingDirectory: workingDirectory) + .stdout + .trim(); final String? shorebirdFlutterVersion = LineSplitter.split( shorebirdFlutterReleases, ).map((e) => e.replaceFirst('origin/flutter_release/', '')).toList().firstOrNull; From 3f3c8ff1eb6ae656ad75b6da96dfc006cd5c1fe6 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 12 Dec 2025 09:04:44 -0800 Subject: [PATCH 08/51] feat: make it possible to load two patches into the runtime at once (#96) * fix: stop using globals for patch data * chore: run et format * chore: add missing files * test: add unittest * chore: run et format * chore: move elf_cache down into runtime * chore: rename elf* to patch* * chore: clean up logs * chore: clean up comments * chore: use Shorebird dart * chore: small cleanup --- bin/internal/update_dart_sdk.sh | 2 +- engine/src/flutter/runtime/BUILD.gn | 4 + engine/src/flutter/runtime/dart_snapshot.cc | 127 +++++--------- engine/src/flutter/runtime/shorebird/BUILD.gn | 15 ++ .../flutter/runtime/shorebird/patch_cache.cc | 160 ++++++++++++++++++ .../flutter/runtime/shorebird/patch_cache.h | 104 ++++++++++++ .../runtime/shorebird/patch_mapping.cc | 51 ++++++ .../flutter/runtime/shorebird/patch_mapping.h | 55 ++++++ .../flutter/shell/common/shorebird/BUILD.gn | 2 + .../common/shorebird/patch_cache_unittests.cc | 70 ++++++++ 10 files changed, 505 insertions(+), 85 deletions(-) create mode 100644 engine/src/flutter/runtime/shorebird/BUILD.gn create mode 100644 engine/src/flutter/runtime/shorebird/patch_cache.cc create mode 100644 engine/src/flutter/runtime/shorebird/patch_cache.h create mode 100644 engine/src/flutter/runtime/shorebird/patch_mapping.cc create mode 100644 engine/src/flutter/runtime/shorebird/patch_mapping.h create mode 100644 engine/src/flutter/shell/common/shorebird/patch_cache_unittests.cc diff --git a/bin/internal/update_dart_sdk.sh b/bin/internal/update_dart_sdk.sh index 9ab6151021a8b..6e3617a328342 100755 --- a/bin/internal/update_dart_sdk.sh +++ b/bin/internal/update_dart_sdk.sh @@ -123,7 +123,7 @@ if [ ! -f "$ENGINE_STAMP" ] || [ "$ENGINE_VERSION" != `cat "$ENGINE_STAMP"` ]; t FIND=find fi - DART_SDK_BASE_URL="${FLUTTER_STORAGE_BASE_URL:-https://storage.googleapis.com}${ENGINE_REALM:+/$ENGINE_REALM}" + DART_SDK_BASE_URL="${FLUTTER_STORAGE_BASE_URL:-https://download.shorebird.dev}${ENGINE_REALM:+/$ENGINE_REALM}" DART_SDK_URL="$DART_SDK_BASE_URL/flutter_infra_release/flutter/$ENGINE_VERSION/$DART_ZIP_NAME" # if the sdk path exists, copy it to a temporary location diff --git a/engine/src/flutter/runtime/BUILD.gn b/engine/src/flutter/runtime/BUILD.gn index 2f7aea3d0d107..1d35652477aca 100644 --- a/engine/src/flutter/runtime/BUILD.gn +++ b/engine/src/flutter/runtime/BUILD.gn @@ -118,6 +118,10 @@ source_set("runtime") { "//flutter/third_party/tonic", "//flutter/txt", ] + + if (is_ios) { + deps += [ "//flutter/runtime/shorebird:patch_cache" ] + } } if (enable_unittests) { diff --git a/engine/src/flutter/runtime/dart_snapshot.cc b/engine/src/flutter/runtime/dart_snapshot.cc index 28ccb1cc46141..b822d98604712 100644 --- a/engine/src/flutter/runtime/dart_snapshot.cc +++ b/engine/src/flutter/runtime/dart_snapshot.cc @@ -14,6 +14,10 @@ #include "flutter/runtime/dart_vm.h" #include "third_party/dart/runtime/include/dart_api.h" +#if SHOREBIRD_USE_INTERPRETER +#include "flutter/runtime/shorebird/patch_cache.h" // nogncheck +#endif + namespace flutter { const char* DartSnapshot::kVMDataSymbol = "kDartVmSnapshotData"; @@ -57,93 +61,33 @@ static std::shared_ptr SearchMapping( const std::vector& native_library_paths, const char* native_library_symbol_name, bool is_executable) { -#if SHOREBIRD_USE_INTERPRETER - // Detect when we're trying to load a Shorebird patch. - auto patch_path = native_library_paths.front(); - bool is_patch = patch_path.find(".vmcode") != std::string::npos; - if (is_patch) { - // We use this terrible hack to load in the patch and then extract the - // symbols from it when the path is not App.framework/App but rather - // foo.vmcode, etc. We read the symbols into static variables, but then I - // believe we need to hold onto the ELF itself, otherwise the symbols - // become invalid. - // "leaked_elf" is meant to indicate that we're not freeing the ELF. - static Dart_LoadedElf* leaked_elf = nullptr; - // The VM Snapshot is identical for all binaries produced by a given version - // of Dart. Our linker checks this and will fail to link if ever the VM - // snapshot changes. - const uint8_t* ignored_vm_data = nullptr; - const uint8_t* ignored_vm_instrs = nullptr; - static const uint8_t* isolate_data = nullptr; - static const uint8_t* isolate_instrs = nullptr; - if (leaked_elf == nullptr) { - const char* error = nullptr; - // vmcode files are elf files prefixed with a shorebird linker header. - auto elf_mapping = GetFileMapping(patch_path, false /* executable */); - int elf_file_offset = Shorebird_ReadLinkHeader(elf_mapping->GetMapping(), - elf_mapping->GetSize()); - - leaked_elf = Dart_LoadELF(patch_path.c_str(), elf_file_offset, &error, - &ignored_vm_data, &ignored_vm_instrs, - &isolate_data, &isolate_instrs, - /* load as read-only, not rx */ false); - if (leaked_elf != nullptr) { - FML_LOG(INFO) << "Loaded ELF"; - } else { - FML_LOG(FATAL) << "Failed to load ELF at " << patch_path - << " error: " << error; - abort(); - } - } - - FML_LOG(INFO) << "Loading symbol from ELF " << native_library_symbol_name; - - if (native_library_symbol_name == DartSnapshot::kIsolateDataSymbol) { - return std::make_unique(isolate_data, 0, - nullptr, true); - } else if (native_library_symbol_name == - DartSnapshot::kIsolateInstructionsSymbol) { - return std::make_unique(isolate_instrs, 0, - nullptr, true); - } - // Fall through to normal lookups for VM data and instructions. - // This fallthrough depends on the fact that NativeLibrary below can't - // read the ELF out of our .vmcode files. - } else { - // Only try to open the file if we're not loading a patch. -#endif - - // Ask the embedder. There is no fallback as we expect the embedders (via - // their embedding APIs) to just specify the mappings directly. - if (embedder_mapping_callback) { - // Note that mapping will be nullptr if the mapping callback returns an - // invalid mapping. If all the other methods for resolving the data also - // fail, the engine will stop with accompanying error logs. - if (auto mapping = embedder_mapping_callback()) { - return mapping; - } + // Ask the embedder. There is no fallback as we expect the embedders (via + // their embedding APIs) to just specify the mappings directly. + if (embedder_mapping_callback) { + // Note that mapping will be nullptr if the mapping callback returns an + // invalid mapping. If all the other methods for resolving the data also + // fail, the engine will stop with accompanying error logs. + if (auto mapping = embedder_mapping_callback()) { + return mapping; } + } - // Attempt to open file at path specified. - if (!file_path.empty()) { - if (auto file_mapping = GetFileMapping(file_path, is_executable)) { - return file_mapping; - } + // Attempt to open file at path specified. + if (!file_path.empty()) { + if (auto file_mapping = GetFileMapping(file_path, is_executable)) { + return file_mapping; } + } - // Look in application specified native library if specified. - for (const std::string& path : native_library_paths) { - auto native_library = fml::NativeLibrary::Create(path.c_str()); - auto symbol_mapping = std::make_unique( - native_library, native_library_symbol_name); - if (symbol_mapping->GetMapping() != nullptr) { - return symbol_mapping; - } + // Look in application specified native library if specified. + for (const std::string& path : native_library_paths) { + auto native_library = fml::NativeLibrary::Create(path.c_str()); + auto symbol_mapping = std::make_unique( + native_library, native_library_symbol_name); + if (symbol_mapping->GetMapping() != nullptr) { + return symbol_mapping; } - -#if SHOREBIRD_USE_INTERPRETER - } // !is_patch -#endif + } // Look inside the currently loaded process. { @@ -206,7 +150,14 @@ static std::shared_ptr ResolveIsolateData( nullptr, // release_func true // dontneed_safe ); -#else // DART_SNAPSHOT_STATIC_LINK +#else // DART_SNAPSHOT_STATIC_LINK +#if SHOREBIRD_USE_INTERPRETER + // Try loading from a Shorebird patch first. + if (auto mapping = TryLoadFromPatch(settings.application_library_paths, + DartSnapshot::kIsolateDataSymbol)) { + return mapping; + } +#endif // SHOREBIRD_USE_INTERPRETER return SearchMapping( settings.isolate_snapshot_data, // embedder_mapping_callback settings.isolate_snapshot_data_path, // file_path @@ -226,7 +177,15 @@ static std::shared_ptr ResolveIsolateInstructions( nullptr, // release_func true // dontneed_safe ); -#else // DART_SNAPSHOT_STATIC_LINK +#else // DART_SNAPSHOT_STATIC_LINK +#if SHOREBIRD_USE_INTERPRETER + // Try loading from a Shorebird patch first. + if (auto mapping = + TryLoadFromPatch(settings.application_library_paths, + DartSnapshot::kIsolateInstructionsSymbol)) { + return mapping; + } +#endif // SHOREBIRD_USE_INTERPRETER return SearchMapping( settings.isolate_snapshot_instr, // embedder_mapping_callback settings.isolate_snapshot_instr_path, // file_path diff --git a/engine/src/flutter/runtime/shorebird/BUILD.gn b/engine/src/flutter/runtime/shorebird/BUILD.gn new file mode 100644 index 0000000000000..facee07ba5cd1 --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/BUILD.gn @@ -0,0 +1,15 @@ +import("//flutter/common/config.gni") + +source_set("patch_cache") { + sources = [ + "patch_cache.cc", + "patch_cache.h", + "patch_mapping.cc", + "patch_mapping.h", + ] + + deps = [ + "//flutter/fml", + "//flutter/runtime:libdart", + ] +} diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.cc b/engine/src/flutter/runtime/shorebird/patch_cache.cc new file mode 100644 index 0000000000000..30c99aea3a011 --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/patch_cache.cc @@ -0,0 +1,160 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/runtime/shorebird/patch_cache.h" + +#include "flutter/fml/logging.h" +#include "flutter/fml/mapping.h" +#include "flutter/runtime/shorebird/patch_mapping.h" + +namespace flutter { + +namespace { + +// These symbol names match the constants in dart_snapshot.cc. +// We duplicate them here rather than extracting them into a header. +// They are actually defined down in Dart and will never change. +constexpr const char* kIsolateDataSymbol = "kDartIsolateSnapshotData"; +constexpr const char* kIsolateInstructionsSymbol = + "kDartIsolateSnapshotInstructions"; + +} // namespace + +// PatchCacheEntry implementation + +std::shared_ptr PatchCacheEntry::Create( + const std::string& path) { + // vmcode files currently use ELF internally after a prefix of a Shorebird + // linker header. + auto elf_mapping = fml::FileMapping::CreateReadOnly(path); + if (!elf_mapping) { + FML_LOG(ERROR) << "Failed to map file: " << path; + return nullptr; + } + + int elf_file_offset = Shorebird_ReadLinkHeader(elf_mapping->GetMapping(), + elf_mapping->GetSize()); + + const char* error = nullptr; + // The VM Snapshot is identical for all binaries produced by a given version + // of Dart. Our linker checks this and will fail to link if ever the VM + // snapshot changes. We ignore the VM data/instrs here. + const uint8_t* ignored_vm_data = nullptr; + const uint8_t* ignored_vm_instrs = nullptr; + const uint8_t* isolate_data = nullptr; + const uint8_t* isolate_instrs = nullptr; + + Dart_LoadedElf* elf = + Dart_LoadELF(path.c_str(), elf_file_offset, &error, &ignored_vm_data, + &ignored_vm_instrs, &isolate_data, &isolate_instrs, + /* load as read-only, not rx */ false); + + if (elf == nullptr) { + FML_LOG(ERROR) << "Failed to load patch at " << path << " error: " << error; + return nullptr; + } + + FML_LOG(INFO) << "Loaded patch from " << path; + + return std::shared_ptr( + new PatchCacheEntry(path, elf, isolate_data, isolate_instrs)); +} + +PatchCacheEntry::PatchCacheEntry(const std::string& path, + Dart_LoadedElf* elf, + const uint8_t* isolate_data, + const uint8_t* isolate_instrs) + : path_(path), + elf_(elf), + isolate_data_(isolate_data), + isolate_instrs_(isolate_instrs) {} + +PatchCacheEntry::~PatchCacheEntry() { + if (elf_ != nullptr) { + FML_LOG(INFO) << "Unloading patch from " << path_; + Dart_UnloadELF(elf_); + elf_ = nullptr; + } +} + +PatchCache& PatchCache::Instance() { + static PatchCache instance; + return instance; +} + +std::shared_ptr PatchCache::GetOrLoad( + const std::string& path) { + std::lock_guard lock(mutex_); + + // Check if we have a cached entry that's still alive + auto it = cache_.find(path); + if (it != cache_.end()) { + if (auto entry = it->second.lock()) { + FML_LOG(INFO) << "PatchCache hit for " << path; + return entry; + } + // Entry expired, remove it + cache_.erase(it); + } + + // Load a new entry + auto entry = PatchCacheEntry::Create(path); + if (entry) { + cache_[path] = entry; // Store weak_ptr + } + + return entry; +} + +void PatchCache::PruneExpired() { + std::lock_guard lock(mutex_); + + for (auto it = cache_.begin(); it != cache_.end();) { + if (it->second.expired()) { + it = cache_.erase(it); + } else { + ++it; + } + } +} + +std::shared_ptr TryLoadFromPatch( + const std::vector& native_library_paths, + const char* symbol_name) { + if (native_library_paths.empty()) { + return nullptr; + } + + // Check if the first path is a Shorebird patch (.vmcode file) + const auto& patch_path = native_library_paths.front(); + bool is_patch = patch_path.find(".vmcode") != std::string::npos; + if (!is_patch) { + return nullptr; + } + + // Patches only contain isolate data/instructions, not VM data/instructions. + // Return nullptr for VM symbols to allow fallback to the base app. + std::string symbol(symbol_name); + if (symbol != kIsolateDataSymbol && symbol != kIsolateInstructionsSymbol) { + return nullptr; + } + + // Load the patch using the cache. + auto cache_entry = PatchCache::Instance().GetOrLoad(patch_path); + if (!cache_entry) { + FML_LOG(FATAL) << "Failed to load symbol from patch at " << patch_path; + return nullptr; + } + + FML_LOG(INFO) << "Loading symbol from patch: " << symbol_name; + + if (symbol == kIsolateDataSymbol) { + return PatchMapping::CreateIsolateData(cache_entry); + } else { + FML_CHECK(symbol == kIsolateInstructionsSymbol); + return PatchMapping::CreateIsolateInstructions(cache_entry); + } +} + +} // namespace flutter diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.h b/engine/src/flutter/runtime/shorebird/patch_cache.h new file mode 100644 index 0000000000000..1f2e14fab711d --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/patch_cache.h @@ -0,0 +1,104 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_RUNTIME_SHOREBIRD_PATCH_CACHE_H_ +#define FLUTTER_RUNTIME_SHOREBIRD_PATCH_CACHE_H_ + +#include +#include +#include +#include +#include + +#include + +#include "flutter/fml/macros.h" +#include "flutter/fml/mapping.h" + +namespace flutter { + +/// A cache entry that holds a loaded patch file and its extracted snapshot +/// pointers. The patch is automatically unloaded when the last reference to +/// this entry is released. +class PatchCacheEntry { + public: + /// Creates a new cache entry by loading the patch file at the given path. + /// Returns nullptr if loading fails. + static std::shared_ptr Create(const std::string& path); + + ~PatchCacheEntry(); + + /// Returns the isolate snapshot data pointer. + const uint8_t* isolate_data() const { return isolate_data_; } + + /// Returns the isolate snapshot instructions pointer. + const uint8_t* isolate_instructions() const { return isolate_instrs_; } + + /// Returns the path this entry was loaded from. + const std::string& path() const { return path_; } + + private: + PatchCacheEntry(const std::string& path, + Dart_LoadedElf* elf, + const uint8_t* isolate_data, + const uint8_t* isolate_instrs); + + std::string path_; + Dart_LoadedElf* elf_; + const uint8_t* isolate_data_; + const uint8_t* isolate_instrs_; + + FML_DISALLOW_COPY_AND_ASSIGN(PatchCacheEntry); +}; + +/// A thread-safe cache for loaded patch files. Cache entries are automatically +/// removed when all references to them are released. +class PatchCache { + public: + /// Returns the singleton instance of the cache. + static PatchCache& Instance(); + + /// Gets or loads a patch file at the given path. If the file is already + /// cached and the entry is still alive, returns the existing entry. + /// Otherwise, loads the file and creates a new cache entry. + /// Returns nullptr if loading fails. + std::shared_ptr GetOrLoad(const std::string& path); + + /// Removes expired entries from the cache. This is called automatically + /// by GetOrLoad, but can also be called explicitly. + void PruneExpired(); + + private: + PatchCache() = default; + ~PatchCache() = default; + + std::mutex mutex_; + // We store weak references so entries are automatically cleaned up when + // all ElfMapping instances release their references. + std::map> cache_; + + FML_DISALLOW_COPY_AND_ASSIGN(PatchCache); +}; + +/// Checks if the first path in native_library_paths is a Shorebird patch +/// (.vmcode file) and if so, attempts to load the requested symbol from +/// the patch. +/// +/// @param native_library_paths The list of library paths to check. The first +/// path is checked for the .vmcode extension. +/// @param symbol_name The symbol to load (kIsolateDataSymbol or +/// kIsolateInstructionsSymbol). +/// @return A mapping for the requested symbol if this is a patch and the +/// symbol is available in the patch, nullptr otherwise. +/// +/// Note: Patches only contain isolate data/instructions, not VM +/// data/instructions. For VM symbols, this will always return nullptr, +/// allowing the caller to fall back to loading from the base app. +std::shared_ptr TryLoadFromPatch( + const std::vector& native_library_paths, + const char* symbol_name); + +} // namespace flutter + +#endif // FLUTTER_RUNTIME_SHOREBIRD_PATCH_CACHE_H_ diff --git a/engine/src/flutter/runtime/shorebird/patch_mapping.cc b/engine/src/flutter/runtime/shorebird/patch_mapping.cc new file mode 100644 index 0000000000000..d444cda817c34 --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/patch_mapping.cc @@ -0,0 +1,51 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/runtime/shorebird/patch_mapping.h" + +#include "third_party/dart/runtime/include/dart_native_api.h" + +namespace flutter { + +std::shared_ptr PatchMapping::CreateIsolateData( + std::shared_ptr entry) { + if (!entry) { + return nullptr; + } + const uint8_t* data = entry->isolate_data(); + size_t size = Dart_SnapshotDataSize(data); + return std::shared_ptr(new PatchMapping(entry, data, size)); +} + +std::shared_ptr PatchMapping::CreateIsolateInstructions( + std::shared_ptr entry) { + if (!entry) { + return nullptr; + } + const uint8_t* data = entry->isolate_instructions(); + size_t size = Dart_SnapshotInstrSize(data); + return std::shared_ptr(new PatchMapping(entry, data, size)); +} + +PatchMapping::PatchMapping(std::shared_ptr entry, + const uint8_t* data, + size_t size) + : cache_entry_(std::move(entry)), data_(data), size_(size) {} + +PatchMapping::~PatchMapping() = default; + +size_t PatchMapping::GetSize() const { + return size_; +} + +const uint8_t* PatchMapping::GetMapping() const { + return data_; +} + +bool PatchMapping::IsDontNeedSafe() const { + // Patch mappings are file-backed and safe for madvise(DONTNEED). + return true; +} + +} // namespace flutter diff --git a/engine/src/flutter/runtime/shorebird/patch_mapping.h b/engine/src/flutter/runtime/shorebird/patch_mapping.h new file mode 100644 index 0000000000000..8c2e494a9004c --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/patch_mapping.h @@ -0,0 +1,55 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_RUNTIME_SHOREBIRD_PATCH_MAPPING_H_ +#define FLUTTER_RUNTIME_SHOREBIRD_PATCH_MAPPING_H_ + +#include + +#include "flutter/fml/macros.h" +#include "flutter/fml/mapping.h" +#include "flutter/runtime/shorebird/patch_cache.h" + +namespace flutter { + +/// A Mapping implementation that references data from a cached patch file. +/// Holding a reference to this mapping keeps the underlying patch loaded. +class PatchMapping final : public fml::Mapping { + public: + /// Creates a mapping for the isolate snapshot data from the given cache + /// entry. + static std::shared_ptr CreateIsolateData( + std::shared_ptr entry); + + /// Creates a mapping for the isolate snapshot instructions from the given + /// cache entry. + static std::shared_ptr CreateIsolateInstructions( + std::shared_ptr entry); + + ~PatchMapping() override; + + // |fml::Mapping| + size_t GetSize() const override; + + // |fml::Mapping| + const uint8_t* GetMapping() const override; + + // |fml::Mapping| + bool IsDontNeedSafe() const override; + + private: + PatchMapping(std::shared_ptr entry, + const uint8_t* data, + size_t size); + + std::shared_ptr cache_entry_; + const uint8_t* data_; + size_t size_; + + FML_DISALLOW_COPY_AND_ASSIGN(PatchMapping); +}; + +} // namespace flutter + +#endif // FLUTTER_RUNTIME_SHOREBIRD_PATCH_MAPPING_H_ diff --git a/engine/src/flutter/shell/common/shorebird/BUILD.gn b/engine/src/flutter/shell/common/shorebird/BUILD.gn index 2c7def6991542..dc2fc5515bd7c 100644 --- a/engine/src/flutter/shell/common/shorebird/BUILD.gn +++ b/engine/src/flutter/shell/common/shorebird/BUILD.gn @@ -42,6 +42,7 @@ if (enable_unittests) { testonly = true sources = [ + "patch_cache_unittests.cc", "shorebird_unittests.cc", "snapshots_data_handle_unittests.cc", ] @@ -53,6 +54,7 @@ if (enable_unittests) { ":shorebird_fixtures", ":snapshots_data_handle", "//flutter/runtime", + "//flutter/runtime/shorebird:patch_cache", "//flutter/testing", "//flutter/testing:fixture_test", ] diff --git a/engine/src/flutter/shell/common/shorebird/patch_cache_unittests.cc b/engine/src/flutter/shell/common/shorebird/patch_cache_unittests.cc new file mode 100644 index 0000000000000..51fc20f9ca562 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/patch_cache_unittests.cc @@ -0,0 +1,70 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/runtime/shorebird/patch_cache.h" + +#include +#include + +#include "gtest/gtest.h" + +namespace flutter { +namespace testing { + +TEST(PatchCache, InstanceReturnsSameInstance) { + PatchCache& instance1 = PatchCache::Instance(); + PatchCache& instance2 = PatchCache::Instance(); + EXPECT_EQ(&instance1, &instance2); +} + +TEST(TryLoadFromPatch, ReturnsNullptrForEmptyPaths) { + std::vector empty_paths; + auto result = TryLoadFromPatch(empty_paths, "kDartIsolateSnapshotData"); + EXPECT_EQ(result, nullptr); +} + +TEST(TryLoadFromPatch, ReturnsNullptrForNonVmcodePath) { + std::vector paths = {"/path/to/some/file.so"}; + auto result = TryLoadFromPatch(paths, "kDartIsolateSnapshotData"); + EXPECT_EQ(result, nullptr); +} + +TEST(TryLoadFromPatch, ReturnsNullptrForVmSymbol) { + // Even with a .vmcode path, VM symbols should return nullptr + // (we can't actually load the file, but we can verify the symbol check) + std::vector paths = {"/path/to/patch.vmcode"}; + + // VM data symbol should return nullptr (patches don't contain VM snapshots) + auto result_vm_data = TryLoadFromPatch(paths, "kDartVmSnapshotData"); + EXPECT_EQ(result_vm_data, nullptr); + + // VM instructions symbol should return nullptr + auto result_vm_instrs = + TryLoadFromPatch(paths, "kDartVmSnapshotInstructions"); + EXPECT_EQ(result_vm_instrs, nullptr); +} + +TEST(TryLoadFromPatch, ReturnsNullptrForUnknownSymbol) { + std::vector paths = {"/path/to/patch.vmcode"}; + auto result = TryLoadFromPatch(paths, "kSomeUnknownSymbol"); + EXPECT_EQ(result, nullptr); +} + +TEST(TryLoadFromPatch, ChecksOnlyFirstPath) { + // Only the first path should be checked for .vmcode extension + std::vector paths = {"/path/to/regular.so", + "/path/to/patch.vmcode"}; + auto result = TryLoadFromPatch(paths, "kDartIsolateSnapshotData"); + // Should return nullptr because first path is not .vmcode + EXPECT_EQ(result, nullptr); +} + +TEST(PatchCache, GetOrLoadReturnsNullptrForNonexistentFile) { + auto result = + PatchCache::Instance().GetOrLoad("/nonexistent/path/to/file.vmcode"); + EXPECT_EQ(result, nullptr); +} + +} // namespace testing +} // namespace flutter From 15182ab88f68922076f6f5756fc57b14dde14006 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 12 Dec 2025 14:36:18 -0800 Subject: [PATCH 09/51] feat: shorebird flutter should work without setting FLUTTER_STORAGE_BASE_URL (#97) * fix: make dart/flutter work without FLUTTER_STORAGE_BASE_URL * feat: shorebird flutter should work without setting FLUTTER_STORAGE_BASE_URL * fix: flutter_tools test fixes * fix: enable running flutter_tools tests * chore: remove unnecessary workflow --- .github/workflows/shorebird_ci.yml | 5 +- bin/internal/update_dart_sdk.ps1 | 2 +- dev/bots/post_process_docs.dart | 2 +- dev/bots/unpublish_package.dart | 2 +- .../settings.gradle | 2 +- .../settings.gradle.kts | 2 +- dev/tools/create_api_docs.dart | 4 +- engine/src/flutter/build/zip_bundle.gni | 2 +- .../impeller/toolkit/interop/README.md | 2 +- .../web_ui/dev/steps/copy_artifacts_step.dart | 4 +- .../gradle/aar_init_script.gradle | 2 +- .../src/main/kotlin/FlutterPluginConstants.kt | 2 +- packages/flutter_tools/lib/src/cache.dart | 4 +- .../lib/src/http_host_validator.dart | 2 +- .../test/general.shard/base/build_test.dart | 14 +++++ .../build_system/targets/common_test.dart | 12 ++++ .../build_system/targets/macos_test.dart | 59 +++++++++++++++---- .../test/general.shard/cache_test.dart | 4 +- .../test/general.shard/version_test.dart | 14 +++++ 19 files changed, 106 insertions(+), 34 deletions(-) diff --git a/.github/workflows/shorebird_ci.yml b/.github/workflows/shorebird_ci.yml index ec147b4489255..7b712b1583b87 100644 --- a/.github/workflows/shorebird_ci.yml +++ b/.github/workflows/shorebird_ci.yml @@ -46,10 +46,7 @@ jobs: - name: ๐Ÿฆ Run Flutter Tools Tests # TODO(eseidel): Find a nice way to run this on windows. if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} - # TODO(eseidel): We can't run all flutter_tools tests until we make - # our changes not throw exceptions on missing shorebird.yaml. - # https://github.com/shorebirdtech/shorebird/issues/2392 - run: ../../bin/flutter test test/general.shard/shorebird + run: ../../bin/flutter test test/general.shard working-directory: packages/flutter_tools - name: ๐Ÿฆ Run Shorebird Tests diff --git a/bin/internal/update_dart_sdk.ps1 b/bin/internal/update_dart_sdk.ps1 index 318ef62add853..a588137834eb6 100644 --- a/bin/internal/update_dart_sdk.ps1 +++ b/bin/internal/update_dart_sdk.ps1 @@ -41,7 +41,7 @@ if ((Test-Path $engineStamp) -and ($engineVersion -eq (Get-Content $engineStamp) $dartSdkBaseUrl = $Env:FLUTTER_STORAGE_BASE_URL if (-not $dartSdkBaseUrl) { - $dartSdkBaseUrl = "https://storage.googleapis.com" + $dartSdkBaseUrl = "https://download.shorebird.dev" } if ($engineRealm) { $dartSdkBaseUrl = "$dartSdkBaseUrl/$engineRealm" diff --git a/dev/bots/post_process_docs.dart b/dev/bots/post_process_docs.dart index aaf08e82e4913..5c40a60cd4f3b 100644 --- a/dev/bots/post_process_docs.dart +++ b/dev/bots/post_process_docs.dart @@ -38,7 +38,7 @@ Future postProcess() async { await runProcessWithValidations([ 'curl', '-L', - 'https://storage.googleapis.com/flutter_infra_release/flutter/$revision/api_docs.zip', + 'https://download.shorebird.dev/flutter_infra_release/flutter/$revision/api_docs.zip', '--output', zipDestination, '--fail', diff --git a/dev/bots/unpublish_package.dart b/dev/bots/unpublish_package.dart index 022111bd16ed9..03cc3a24d2fab 100644 --- a/dev/bots/unpublish_package.dart +++ b/dev/bots/unpublish_package.dart @@ -23,7 +23,7 @@ import 'package:process/process.dart'; const String gsBase = 'gs://flutter_infra_release'; const String releaseFolder = '/releases'; const String gsReleaseFolder = '$gsBase$releaseFolder'; -const String baseUrl = 'https://storage.googleapis.com/flutter_infra_release'; +const String baseUrl = 'https://download.shorebird.dev/flutter_infra_release'; /// Exception class for when a process fails to run, so we can catch /// it and provide something more readable than a stack trace. diff --git a/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle b/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle index 5368feb7ecde7..66364cdf4a190 100644 --- a/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle +++ b/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle @@ -7,7 +7,7 @@ dependencyResolutionManagement { repositories { google() mavenCentral() - def flutterStorageUrl = System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "https://storage.googleapis.com" + def flutterStorageUrl = System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "https://download.shorebird.dev" maven { url = uri("$flutterStorageUrl/download.flutter.io") } diff --git a/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts b/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts index f6d75bce11757..da25b49a46f7f 100644 --- a/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts +++ b/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts @@ -16,7 +16,7 @@ dependencyResolutionManagement { repositories { google() mavenCentral() - val flutterStorageUrl = System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "https://storage.googleapis.com" + val flutterStorageUrl = System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "https://download.shorebird.dev" maven("$flutterStorageUrl/download.flutter.io") } } diff --git a/dev/tools/create_api_docs.dart b/dev/tools/create_api_docs.dart index a98f16398b21f..4ece883e09df7 100644 --- a/dev/tools/create_api_docs.dart +++ b/dev/tools/create_api_docs.dart @@ -923,8 +923,8 @@ class PlatformDocGenerator { for (final String platform in kPlatformDocs.keys) { final String zipFile = kPlatformDocs[platform]!.zipName; - final url = - 'https://storage.googleapis.com/${realm}flutter_infra_release/flutter/$engineRevision/$zipFile'; + final String url = + 'https://download.shorebird.dev/${realm}flutter_infra_release/flutter/$engineRevision/$zipFile'; await _extractDocs(url, platform, kPlatformDocs[platform]!, outputDir); } } diff --git a/engine/src/flutter/build/zip_bundle.gni b/engine/src/flutter/build/zip_bundle.gni index 51e72df0ad854..707ab046d3637 100644 --- a/engine/src/flutter/build/zip_bundle.gni +++ b/engine/src/flutter/build/zip_bundle.gni @@ -55,7 +55,7 @@ template("zip_bundle") { license_path = rebase_path("//flutter/sky/packages/sky_engine/LICENSE", "//flutter") git_url = "https://github.com/flutter/engine/tree/$engine_version" - sky_engine_url = "https://storage.googleapis.com/flutter_infra_release/flutter/$engine_version/sky_engine.zip" + sky_engine_url = "https://download.shorebird.dev/flutter_infra_release/flutter/$engine_version/sky_engine.zip" outputs = [ license_readme ] contents = [ "# $target_name", diff --git a/engine/src/flutter/impeller/toolkit/interop/README.md b/engine/src/flutter/impeller/toolkit/interop/README.md index 6e75a53a11775..279379217e143 100644 --- a/engine/src/flutter/impeller/toolkit/interop/README.md +++ b/engine/src/flutter/impeller/toolkit/interop/README.md @@ -27,7 +27,7 @@ A single-header C API for 2D graphics and text rendering. [Impeller](../../READM Users may plug in a custom toolchain into the Flutter Engine build system to build the `libimpeller.so` dynamic library. However, for the common platforms, the CI bots upload a tarball containing the library and headers. This URL for the SDK tarball for a particular platform can be constructed as follows: ```sh -https://storage.googleapis.com/flutter_infra_release/flutter/$FLUTTER_SHA/$PLATFORM_ARCH/impeller_sdk.zip +https://download.shorebird.dev/flutter_infra_release/flutter/$FLUTTER_SHA/$PLATFORM_ARCH/impeller_sdk.zip ``` The `$FLUTTER_SHA` is the Git hash in the [Flutter repository](https://github.com/flutter/flutter). The `$PLATFORM_ARCH` can be determined from the table below. diff --git a/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart b/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart index e95e0b3f0a4b3..5d94008deed48 100644 --- a/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart +++ b/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart @@ -55,8 +55,8 @@ class CopyArtifactsStep implements PipelineStep { 'Could not generate artifact bucket url for unknown realm.', ), }; - final url = Uri.https( - 'storage.googleapis.com', + final Uri url = Uri.https( + 'download.shorebird.dev', '${realmComponent}flutter_infra_release/flutter/${realm == LuciRealm.Try ? gitRevision : contentHash}/flutter-web-sdk.zip', ); final http.Response response = await http.Client().get(url); diff --git a/packages/flutter_tools/gradle/aar_init_script.gradle b/packages/flutter_tools/gradle/aar_init_script.gradle index 0d43a9b103fd2..0abc801a2fb62 100644 --- a/packages/flutter_tools/gradle/aar_init_script.gradle +++ b/packages/flutter_tools/gradle/aar_init_script.gradle @@ -42,7 +42,7 @@ void configureProject(Project project, String outputDir) { return } - String storageUrl = System.getenv('FLUTTER_STORAGE_BASE_URL') ?: "https://storage.googleapis.com" + String storageUrl = System.getenv('FLUTTER_STORAGE_BASE_URL') ?: "https://download.shorebird.dev" String engineRealm = Paths.get(getFlutterRoot(project), "bin", "cache", "engine.realm") .toFile().text.trim() diff --git a/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt b/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt index f9525ef479e31..4cc13207211cb 100644 --- a/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt +++ b/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt @@ -19,7 +19,7 @@ object FlutterPluginConstants { const val INTERMEDIATES_DIR = "intermediates" const val FLUTTER_STORAGE_BASE_URL = "FLUTTER_STORAGE_BASE_URL" - const val DEFAULT_MAVEN_HOST = "https://storage.googleapis.com" + const val DEFAULT_MAVEN_HOST = "https://download.shorebird.dev" /** Maps platforms to ABI architectures. */ @JvmStatic val PLATFORM_ARCH_MAP = diff --git a/packages/flutter_tools/lib/src/cache.dart b/packages/flutter_tools/lib/src/cache.dart index 59d16d8983ca9..2cac940fccfb1 100644 --- a/packages/flutter_tools/lib/src/cache.dart +++ b/packages/flutter_tools/lib/src/cache.dart @@ -518,7 +518,7 @@ class Cache { /// The base for URLs that store Flutter engine artifacts that are fetched /// during the installation of the Flutter SDK. /// - /// By default the base URL is https://storage.googleapis.com. However, if + /// By default the base URL is https://download.shorebird.dev. However, if /// `FLUTTER_STORAGE_BASE_URL` environment variable ([kFlutterStorageBaseUrl]) /// is provided, the environment variable value is returned instead. /// @@ -530,7 +530,7 @@ class Cache { String? overrideUrl = _platform.environment[kFlutterStorageBaseUrl]; if (overrideUrl == null) { return storageRealm.isEmpty - ? 'https://storage.googleapis.com' + ? 'https://download.shorebird.dev' : 'https://storage.googleapis.com/$storageRealm'; } // Shorebird's artifact proxy is a trusted source. diff --git a/packages/flutter_tools/lib/src/http_host_validator.dart b/packages/flutter_tools/lib/src/http_host_validator.dart index 9702f83bffd08..8363221166398 100644 --- a/packages/flutter_tools/lib/src/http_host_validator.dart +++ b/packages/flutter_tools/lib/src/http_host_validator.dart @@ -11,7 +11,7 @@ import 'doctor_validator.dart'; import 'features.dart'; /// Common Flutter HTTP hosts. -const kCloudHost = 'https://storage.googleapis.com/'; +const kCloudHost = 'https://download.shorebird.dev/'; const kCocoaPods = 'https://cocoapods.org/'; const kGitHub = 'https://github.com/'; const kMaven = 'https://maven.google.com/'; diff --git a/packages/flutter_tools/test/general.shard/base/build_test.dart b/packages/flutter_tools/test/general.shard/base/build_test.dart index de41b2e569322..90cc0c26ac1db 100644 --- a/packages/flutter_tools/test/general.shard/base/build_test.dart +++ b/packages/flutter_tools/test/general.shard/base/build_test.dart @@ -37,6 +37,17 @@ const kDefaultClang = [ 'build/foo/snapshot_assembly.o', ]; +// Shorebird link info arguments added for iOS/macOS builds. +// These correspond to the dumpLinkInfoArgs in AOTSnapshotter.build(). +const kLinkInfoArgs = [ + '--print_class_table_link_debug_info_to=build/App.class_table.json', + '--print_class_table_link_info_to=build/App.ct.link', + '--print_field_table_link_debug_info_to=build/App.field_table.json', + '--print_field_table_link_info_to=build/App.ft.link', + '--print_dispatch_table_link_debug_info_to=build/App.dispatch_table.json', + '--print_dispatch_table_link_info_to=build/App.dt.link', +]; + void main() { group('GenSnapshot', () { late GenSnapshot genSnapshot; @@ -203,6 +214,7 @@ void main() { command: [ genSnapshotPath, '--deterministic', + ...kLinkInfoArgs, '--snapshot_kind=app-aot-assembly', '--assembly=$assembly', '--dwarf-stack-traces', @@ -278,6 +290,7 @@ void main() { command: [ genSnapshotPath, '--deterministic', + ...kLinkInfoArgs, '--snapshot_kind=app-aot-assembly', '--assembly=$assembly', '--obfuscate', @@ -349,6 +362,7 @@ void main() { command: [ genSnapshotPath, '--deterministic', + ...kLinkInfoArgs, '--snapshot_kind=app-aot-assembly', '--assembly=${fileSystem.path.join(outputPath, 'snapshot_assembly.S')}', 'main.dill', diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart index 2e0f99300c3be..c0dda6b07591e 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart @@ -25,6 +25,17 @@ const kBoundaryKey = '4d2d9609-c662-4571-afde-31410f96caa6'; const kElfAot = '--snapshot_kind=app-aot-elf'; const kAssemblyAot = '--snapshot_kind=app-aot-assembly'; +/// Generate Shorebird link info arguments for iOS/macOS AOT builds. +/// The [buildPath] should be the build directory path (outputDir.parent.path). +List linkInfoArgsFor(String buildPath) => [ + '--print_class_table_link_debug_info_to=$buildPath/App.class_table.json', + '--print_class_table_link_info_to=$buildPath/App.ct.link', + '--print_field_table_link_debug_info_to=$buildPath/App.field_table.json', + '--print_field_table_link_info_to=$buildPath/App.ft.link', + '--print_dispatch_table_link_debug_info_to=$buildPath/App.dispatch_table.json', + '--print_dispatch_table_link_info_to=$buildPath/App.dt.link', +]; + final Platform macPlatform = FakePlatform( operatingSystem: 'macos', environment: {}, @@ -803,6 +814,7 @@ void main() { // This path is not known by the cache due to the iOS gen_snapshot split. 'Artifact.genSnapshotArm64.TargetPlatform.ios.profile', '--deterministic', + ...linkInfoArgsFor(build), '--write-v8-snapshot-profile-to=code_size_1/snapshot.arm64.json', '--trace-precompiler-to=code_size_1/trace.arm64.json', kAssemblyAot, diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart index 4ada1d72a0f8e..d57c76c444c7f 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart @@ -20,6 +20,17 @@ import '../../../src/fake_process_manager.dart'; import '../../../src/fakes.dart'; import '../../../src/package_config.dart'; +/// Generate Shorebird link info arguments for iOS/macOS AOT builds. +/// The [buildPath] should be the build directory path (outputDir.parent.path). +List linkInfoArgsFor(String buildPath) => [ + '--print_class_table_link_debug_info_to=$buildPath/App.class_table.json', + '--print_class_table_link_info_to=$buildPath/App.ct.link', + '--print_field_table_link_debug_info_to=$buildPath/App.field_table.json', + '--print_field_table_link_info_to=$buildPath/App.ft.link', + '--print_dispatch_table_link_debug_info_to=$buildPath/App.dispatch_table.json', + '--print_dispatch_table_link_info_to=$buildPath/App.dt.link', +]; + void main() { late Environment environment; late MemoryFileSystem fileSystem; @@ -811,28 +822,36 @@ void main() { environment.defines[kXcodeAction] = 'install'; environment.defines[kFlavor] = 'internal'; + // Set up engine artifacts fileSystem .file('bin/cache/artifacts/engine/darwin-x64/vm_isolate_snapshot.bin') .createSync(recursive: true); fileSystem .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') .createSync(recursive: true); + + // Set up App.framework binary fileSystem .file(fileSystem.path .join(environment.buildDir.path, 'App.framework', 'App')) .createSync(recursive: true); - final String shorebirdYamlPath = fileSystem.path.join( - environment.buildDir.path, - 'App.framework', - 'Versions', - 'A', - 'Resources', - 'flutter_assets', - 'shorebird.yaml', - ); - fileSystem.file(fileSystem.path - .join(environment.buildDir.path, 'App.framework', 'App')) - ..createSync(recursive: true) + + // Set up native_assets.json (required by MacOSBundleFlutterAssets) + environment.buildDir.childFile('native_assets.json').createSync(); + + // Set up pubspec.yaml with shorebird.yaml as an asset + fileSystem.file('pubspec.yaml') + ..createSync() + ..writeAsStringSync(''' +name: example +flutter: + assets: + - shorebird.yaml +'''); + + // Create the shorebird.yaml asset file + fileSystem.file('shorebird.yaml') + ..createSync() ..writeAsStringSync(''' # Some other text that should be removed app_id: base-app-id @@ -841,8 +860,21 @@ flavors: stable: stable-app-id '''); + // Set up package config + writePackageConfigFiles(directory: fileSystem.currentDirectory, mainLibName: 'example'); + await const ReleaseMacOSBundleFlutterAssets().build(environment); + // The output is in environment.outputDir, not buildDir + final String shorebirdYamlPath = fileSystem.path.join( + environment.outputDir.path, + 'App.framework', + 'Versions', + 'A', + 'Resources', + 'flutter_assets', + 'shorebird.yaml', + ); expect(fileSystem.file(shorebirdYamlPath).readAsStringSync(), 'app_id: internal-app-id'); }, @@ -912,11 +944,13 @@ flavors: .childFile('x86_64/App.framework.dSYM/Contents/Resources/DWARF/App') .createSync(recursive: true); + final build = environment.buildDir.path; processManager.addCommands([ FakeCommand( command: [ 'Artifact.genSnapshotArm64.TargetPlatform.darwin.release', '--deterministic', + ...linkInfoArgsFor(build), '--snapshot_kind=app-aot-assembly', '--assembly=${environment.buildDir.childFile('arm64/snapshot_assembly.S').path}', environment.buildDir.childFile('app.dill').path, @@ -926,6 +960,7 @@ flavors: command: [ 'Artifact.genSnapshotX64.TargetPlatform.darwin.release', '--deterministic', + ...linkInfoArgsFor(build), '--snapshot_kind=app-aot-assembly', '--assembly=${environment.buildDir.childFile('x86_64/snapshot_assembly.S').path}', environment.buildDir.childFile('app.dill').path, diff --git a/packages/flutter_tools/test/general.shard/cache_test.dart b/packages/flutter_tools/test/general.shard/cache_test.dart index 08bd0a02c62d9..1e71e52385441 100644 --- a/packages/flutter_tools/test/general.shard/cache_test.dart +++ b/packages/flutter_tools/test/general.shard/cache_test.dart @@ -942,7 +942,7 @@ void main() { expect(messages, ['Downloading Web SDK...']); expect(downloads, [ - 'https://storage.googleapis.com/flutter_infra_release/flutter/hijklmnop/flutter-web-sdk.zip', + 'https://download.shorebird.dev/flutter_infra_release/flutter/hijklmnop/flutter-web-sdk.zip', ]); expect(locations, ['/bin/cache/flutter_web_sdk']); @@ -1067,7 +1067,7 @@ void main() { expect(messages, ['Downloading engine information...']); expect(downloads, [ - 'https://storage.googleapis.com/flutter_infra_release/flutter/hijklmnop/engine_stamp.json', + 'https://download.shorebird.dev/flutter_infra_release/flutter/hijklmnop/engine_stamp.json', ]); expect(locations, ['/bin/cache']); // file copy is done by the real uploader; not the fake. diff --git a/packages/flutter_tools/test/general.shard/version_test.dart b/packages/flutter_tools/test/general.shard/version_test.dart index 1d8360c14866c..5d66fee5f3529 100644 --- a/packages/flutter_tools/test/general.shard/version_test.dart +++ b/packages/flutter_tools/test/general.shard/version_test.dart @@ -67,6 +67,20 @@ void main() { required int commitsBetweenRefs, }) { return [ + // Shorebird release branch check (returns empty to fall through to + // regular version lookup). + FakeCommand( + command: [ + 'git', + 'for-each-ref', + '--contains', + headRef, + '--format', + '%(refname:short)', + 'refs/remotes/origin/flutter_release/*', + ], + stdout: '', + ), FakeCommand( command: const [ 'git', From 9e14cbb056c743bc0043e099d98f10db52a950f5 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 12 Dec 2025 14:50:17 -0800 Subject: [PATCH 10/51] chore: move build_engine scripts into this repo (#98) * chore: move build_engine scripts into this repo * chore: fix path of content_aware_hash.sh --- shorebird/ci/internal/generate_manifest.sh | 93 ++++++++++ shorebird/ci/internal/linux_build.sh | 105 +++++++++++ shorebird/ci/internal/linux_setup.sh | 30 +++ shorebird/ci/internal/linux_test_build.sh | 29 +++ shorebird/ci/internal/linux_upload.sh | 138 ++++++++++++++ shorebird/ci/internal/mac_build.sh | 204 +++++++++++++++++++++ shorebird/ci/internal/mac_setup.sh | 11 ++ shorebird/ci/internal/mac_upload.sh | 180 ++++++++++++++++++ shorebird/ci/internal/win_build.sh | 63 +++++++ shorebird/ci/internal/win_setup.sh | 8 + shorebird/ci/internal/win_upload.sh | 90 +++++++++ shorebird/ci/linux_build_and_upload.sh | 32 ++++ shorebird/ci/mac_build_and_upload.sh | 32 ++++ shorebird/ci/win_build_and_upload.sh | 32 ++++ 14 files changed, 1047 insertions(+) create mode 100755 shorebird/ci/internal/generate_manifest.sh create mode 100755 shorebird/ci/internal/linux_build.sh create mode 100755 shorebird/ci/internal/linux_setup.sh create mode 100755 shorebird/ci/internal/linux_test_build.sh create mode 100755 shorebird/ci/internal/linux_upload.sh create mode 100755 shorebird/ci/internal/mac_build.sh create mode 100755 shorebird/ci/internal/mac_setup.sh create mode 100755 shorebird/ci/internal/mac_upload.sh create mode 100755 shorebird/ci/internal/win_build.sh create mode 100755 shorebird/ci/internal/win_setup.sh create mode 100755 shorebird/ci/internal/win_upload.sh create mode 100755 shorebird/ci/linux_build_and_upload.sh create mode 100755 shorebird/ci/mac_build_and_upload.sh create mode 100755 shorebird/ci/win_build_and_upload.sh diff --git a/shorebird/ci/internal/generate_manifest.sh b/shorebird/ci/internal/generate_manifest.sh new file mode 100755 index 0000000000000..4c95de6baf098 --- /dev/null +++ b/shorebird/ci/internal/generate_manifest.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +# This script outputs an artifact_manifest.yaml mapping +# a shorebird engine revision to a flutter engine revision. +# Usage: +# ./generate_manifest.sh > artifact_manifest.yaml + +set -e + +# NOTE: If you edit this file you also may need to edit the global list +# of all known artifacts in the artifact_proxy's config.dart + +if [ "$#" -ne 1 ]; then + echo "Usage: ./generate_manifest.sh " + exit 1 +fi + +FLUTTER_ENGINE_REVISION=$1 + +cat <, but that no longer seems needed. +# We always use the hermetic NDK from the engine repo. +ANDROID_NDK_HOME="$ENGINE_SRC/flutter/third_party/android_tools/ndk" \ +cargo ndk \ + --target armv7-linux-androideabi \ + --target aarch64-linux-android \ + --target i686-linux-android \ + --target x86_64-linux-android \ + build --release + +cargo build --release --target x86_64-unknown-linux-gnu + +# Build the patch tool. +cd $UPDATER_SRC/patch +cargo build --release + +# Compile the engine using the steps here: +# https://github.com/flutter/flutter/wiki/Compiling-the-engine#compiling-for-android-from-macos-or-linux +cd $ENGINE_SRC + +NINJA="ninja" +GN=./flutter/tools/gn +# We could probably use our own prebuilt dart SDK, by modifying the gn files. +GN_ARGS="--no-rbe --no-enable-unittests" + +# We could use Linux to generate all of our Android binaries, but we don't yet. +# https://github.com/flutter/engine/blob/e590b24f3962fda3ec9144dcee3f7565b195839a/ci/builders/linux_android_aot_engine.json#L40 + +# Build the default and gen_snapshot targets. +# +# Linux doesn't seem to use "archive_gen_snapshot" as a target name yet. +# https://github.com/flutter/flutter/issues/105351#issuecomment-1650686247 +ANDROID_TARGETS="default gen_snapshot" + +# Android arm64 release +$GN $GN_ARGS --android --android-cpu=arm64 --runtime-mode=release +$NINJA -C ./out/android_release_arm64 $ANDROID_TARGETS + +# Android arm32 release +$GN $GN_ARGS --runtime-mode=release --android +$NINJA -C out/android_release $ANDROID_TARGETS + +# Android x64 release +$GN $GN_ARGS --android --android-cpu=x64 --runtime-mode=release +$NINJA -C ./out/android_release_x64 $ANDROID_TARGETS + +# Build Dart and Flutter +$GN $GN_ARGS --runtime-mode=release --no-prebuilt-dart-sdk +# build Dart and the linux shell and flutter_patched_sdk_product.zip +$NINJA -C out/host_release dart_sdk flutter/shell/platform/linux:flutter_gtk flutter/build/archives:flutter_patched_sdk +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 + +# Build debug Linux artifacts +# These are output to the `linux-x64` directory in host_debug, and are used +# by `flutter build linux --release`. +$GN $GN_ARGS --no-prebuilt-dart-sdk +$NINJA -C ./out/host_debug flutter/build/archives:artifacts + +# Shorebird AOT Tools (Linker) +mkdir -p $ENGINE_OUT/host_release/aot_tools + +# Dart kernel (.dill) files are not stable and can change with the version of Dart, so we +# can't use this machine's `dart`. Here we're using the version of Dart that this +# version of the engine depends on, which should be the same version that +# `flutter` ends up depending on. +DART=$ENGINE_OUT/host_release/dart-sdk/bin/dart +AOT_TOOLS_PKG=$ENGINE_SRC/flutter/third_party/dart/pkg/aot_tools +# This should be part of `gclient sync` https://github.com/shorebirdtech/_build_engine/issues/113 +(cd $AOT_TOOLS_PKG; $DART pub get) +# This should be built as part of Dart and then pulled down as part of the engine build. +# https://github.com/shorebirdtech/_build_engine/issues/88 +$DART compile kernel $AOT_TOOLS_PKG/bin/aot_tools.dart -o $ENGINE_OUT/host_release/aot_tools/aot-tools.dill + +mkdir -p $ENGINE_OUT/host_release/updater_tools +UPDATER_TOOLS_PKG=$ENGINE_SRC/flutter/third_party/updater/updater_tools +# This should be part of `gclient sync` https://github.com/shorebirdtech/_build_engine/issues/113 + +# We could also build the `patch` tool for Linux here. diff --git a/shorebird/ci/internal/linux_setup.sh b/shorebird/ci/internal/linux_setup.sh new file mode 100755 index 0000000000000..626ad211536eb --- /dev/null +++ b/shorebird/ci/internal/linux_setup.sh @@ -0,0 +1,30 @@ +#!/bin/bash -e + +# Usage: +# ./linux_setup.sh + +# Per https://github.com/flutter/flutter/wiki/Setting-up-the-Engine-development-environment +# Subset of ./flutter/build/install-build-deps-linux-desktop.sh +sudo apt -y install libfreetype6-dev pkg-config + +# This assumes rust is installed, but could also install rust/cargo. + +# Need NDK from https://developer.android.com/ndk/downloads +# The NDK version should match that of DEPS, e.g. +# https://github.com/flutter/flutter/blame/b45fa18946ecc2d9b4009952c636ba7e2ffbb787/DEPS#L615 +# Example: +# curl -O https://dl.google.com/android/repository/android-ndk-r27d-linux.zip +# unzip android-ndk-r27d-linux.zip +# On the GHA runners we set this in .github/workflows/build_engine.yaml +# env: +# NDK_HOME: /home/gha/bin/android-ndk-r27d + +# We require an old version of cargo-ndk to support the old NDK Flutter +# engine currently uses. +cargo install cargo-ndk +rustup target add \ + aarch64-linux-android \ + armv7-linux-androideabi \ + x86_64-linux-android \ + i686-linux-android \ + x86_64-unknown-linux-gnu diff --git a/shorebird/ci/internal/linux_test_build.sh b/shorebird/ci/internal/linux_test_build.sh new file mode 100755 index 0000000000000..be3c54da547ec --- /dev/null +++ b/shorebird/ci/internal/linux_test_build.sh @@ -0,0 +1,29 @@ +#!/bin/bash -e + +# The path to the Flutter engine. +# Convert to an absolute path so we don't need to worry about cd'ing back to +# the root directory between commands. +ENGINE_ROOT=$(realpath $1) +ENGINE_SRC=$ENGINE_ROOT/src + +cd $ENGINE_SRC + +UPDATER_SRC=$ENGINE_SRC/flutter/third_party/updater +(cd $UPDATER_SRC && + ANDROID_NDK_HOME="$ENGINE_SRC/flutter/third_party/android_tools/ndk" \ + cargo ndk \ + --target armv7-linux-androideabi \ + --target aarch64-linux-android \ + --target i686-linux-android \ + --target x86_64-linux-android \ + build --release && + cargo build --release --target x86_64-unknown-linux-gnu +) + +# Generate an unoptimized debug build of the engine (expected by the test script). +./flutter/tools/gn --unoptimized --no-rbe +ninja -C out/host_debug_unopt + +# Generate an unoptimized android debug build for java engine tests +./flutter/tools/gn --android --unoptimized --no-rbe +ninja -C out/android_debug_unopt diff --git a/shorebird/ci/internal/linux_upload.sh b/shorebird/ci/internal/linux_upload.sh new file mode 100755 index 0000000000000..873f0f3edb969 --- /dev/null +++ b/shorebird/ci/internal/linux_upload.sh @@ -0,0 +1,138 @@ +#!/bin/bash -e + +# Usage: +# ./linux_upload.sh engine_path git_hash + +# Convert to an absolute path so we don't need to worry about cd'ing back to +# the root directory between commands. +ENGINE_ROOT=$(realpath $1) +ENGINE_HASH=$2 + +STORAGE_BUCKET="download.shorebird.dev" +SHOREBIRD_ROOT=gs://$STORAGE_BUCKET/shorebird/$ENGINE_HASH + +ENGINE_SRC=$ENGINE_ROOT/src +ENGINE_OUT=$ENGINE_SRC/out +ENGINE_FLUTTER=$ENGINE_SRC/flutter +# FLUTTER_ROOT is the Flutter monorepo root (parent of engine/) +FLUTTER_ROOT=$(dirname $ENGINE_ROOT) + +cd $FLUTTER_ROOT + +# Compute the content-aware hash for the Dart SDK. +# This allows Flutter checkouts that haven't changed engine content to share +# the same pre-built Dart SDK, even if they have different git commit SHAs. +CONTENT_HASH=$($FLUTTER_ROOT/bin/internal/content_aware_hash.sh) + +# We do not generate a manifest file, we assume another builder did that. +# TODO(bryanoltman): should we generate a manifest file as part of an upload +# script, or should it be done once all build and uploads have completed? +# See https://github.com/shorebirdtech/build_engine/issues/25 + +# TODO(eseidel): This should not be in shell, it's too complicated/repetitive. + +HOST_ARCH='linux-x64' + +INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$ENGINE_HASH" +MAVEN_VER="1.0.0-$ENGINE_HASH" +MAVEN_ROOT="gs://$STORAGE_BUCKET/download.flutter.io/io/flutter" + +# Dart SDK +# This gets uploaded to flutter_infra_release/flutter/\$engine/dart-sdk-$HOST_ARCH.zip +# We also upload to the content-aware hash path to support local development branches. +CONTENT_INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$CONTENT_HASH" + +HOST_RELEASE=$ENGINE_OUT/host_release +DART_ZIP_FILE=dart-sdk-$HOST_ARCH.zip +( + cd $HOST_RELEASE; + zip -r $DART_ZIP_FILE dart-sdk +) +ZIPS_DEST=$INFRA_ROOT/$DART_ZIP_FILE +gsutil cp $HOST_RELEASE/$DART_ZIP_FILE $ZIPS_DEST +# Also upload to content-aware hash path +gsutil cp $HOST_RELEASE/$DART_ZIP_FILE $CONTENT_INFRA_ROOT/$DART_ZIP_FILE + +# Android Arm64 release Flutter artifacts +ARCH_OUT=$ENGINE_OUT/android_release_arm64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm64-release +ZIPS_DEST=$INFRA_ROOT/android-arm64-release +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip +gsutil cp $ZIPS_OUT/symbols.zip $ZIPS_DEST/symbols.zip +# Android Arm64 release Maven artifacts +ARCH_PATH=$ARCH_OUT/arm64_v8a_release +MAVEN_PATH=$MAVEN_ROOT/arm64_v8a_release/$MAVEN_VER/arm64_v8a_release-$MAVEN_VER +gsutil cp $ARCH_PATH.pom $MAVEN_PATH.pom +gsutil cp $ARCH_PATH.jar $MAVEN_PATH.jar +gsutil cp $ARCH_PATH.maven-metadata.xml $MAVEN_PATH.maven-metadata.xml + +# Android Arm32 release Flutter artifacts +ARCH_OUT=$ENGINE_OUT/android_release +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm-release +ZIPS_DEST=$INFRA_ROOT/android-arm-release +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip +gsutil cp $ZIPS_OUT/symbols.zip $ZIPS_DEST/symbols.zip +# Android Arm32 release Maven artifacts +ARCH_PATH=$ARCH_OUT/armeabi_v7a_release +MAVEN_PATH=$MAVEN_ROOT/armeabi_v7a_release/$MAVEN_VER/armeabi_v7a_release-$MAVEN_VER +gsutil cp $ARCH_PATH.pom $MAVEN_PATH.pom +gsutil cp $ARCH_PATH.jar $MAVEN_PATH.jar +gsutil cp $ARCH_PATH.maven-metadata.xml $MAVEN_PATH.maven-metadata.xml + +# Not sure which flutter_embedding_release files to use? 32 or 64 bit? +# It does not seem to contain the libflutter.so file, but does seem to +# differ between the two build dirs. +ARCH_OUT=$ENGINE_OUT/android_release +ARCH_PATH=$ARCH_OUT/flutter_embedding_release +MAVEN_PATH=$MAVEN_ROOT/flutter_embedding_release/$MAVEN_VER/flutter_embedding_release-$MAVEN_VER +gsutil cp $ARCH_PATH.pom $MAVEN_PATH.pom +gsutil cp $ARCH_PATH.jar $MAVEN_PATH.jar +gsutil cp $ARCH_PATH.maven-metadata.xml $MAVEN_PATH.maven-metadata.xml + +# Android x64 release Flutter artifacts +ARCH_OUT=$ENGINE_OUT/android_release_x64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-x64-release +ZIPS_DEST=$INFRA_ROOT/android-x64-release +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip +gsutil cp $ZIPS_OUT/symbols.zip $ZIPS_DEST/symbols.zip +# Android x64 release Maven artifacts +ARCH_PATH=$ARCH_OUT/x86_64_release +MAVEN_PATH=$MAVEN_ROOT/x86_64_release/$MAVEN_VER/x86_64_release-$MAVEN_VER +gsutil cp $ARCH_PATH.pom $MAVEN_PATH.pom +gsutil cp $ARCH_PATH.jar $MAVEN_PATH.jar +gsutil cp $ARCH_PATH.maven-metadata.xml $MAVEN_PATH.maven-metadata.xml + +# Shorebird AOT Tools (Linker) +gsutil cp $ENGINE_OUT/host_release/aot_tools/aot-tools.dill $SHOREBIRD_ROOT/aot-tools.dill + +# Common Product-mode artifacts +ARCH_OUT=$ENGINE_OUT/host_release +ZIPS_OUT=$ARCH_OUT/zip_archives +ZIPS_DEST=$INFRA_ROOT +gsutil cp $ZIPS_OUT/flutter_patched_sdk_product.zip $ZIPS_DEST/flutter_patched_sdk_product.zip + +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 +# Linux x64 host_release font_subset (ConstFinder) +# ARCH_OUT=$ENGINE_OUT/host_release +# ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +# ZIPS_DEST=$INFRA_ROOT/linux-x64-release +# gsutil cp $ZIPS_OUT/font-subset.zip $ZIPS_DEST/font-subset.zip + +# Linux Desktop Support +ARCH_OUT=$ENGINE_OUT/host_release +ZIPS_OUT=$ARCH_OUT/zip_archives/linux-x64-release +ZIPS_DEST=$INFRA_ROOT/linux-x64-release +gsutil cp $ZIPS_OUT/linux-x64-flutter-gtk.zip $ZIPS_DEST/linux-x64-flutter-gtk.zip + +ARCH_OUT=$ENGINE_OUT/host_debug +ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +ZIPS_DEST=$INFRA_ROOT/$HOST_ARCH +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip + +# We could upload patch if we built it here. +# gsutil cp $ENGINE_OUT/host_release/patch.zip $SHOREBIRD_ROOT/patch-win-x64.zip diff --git a/shorebird/ci/internal/mac_build.sh b/shorebird/ci/internal/mac_build.sh new file mode 100755 index 0000000000000..a860da431f269 --- /dev/null +++ b/shorebird/ci/internal/mac_build.sh @@ -0,0 +1,204 @@ +#!/bin/bash -e + +# FIXME: This script should be deleted and instead these steps be part +# of the GN build process. +# I haven't investigated how to build rust from GN with the Android NDK yet. + +# Usage: +# ./mac_build.sh engine_path + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 engine_path" + exit 1 +fi + +# Convert to an absolute path so we don't need to worry about cd'ing back to +# the root directory between commands. +ENGINE_ROOT=$(realpath $1) + +ENGINE_SRC=$ENGINE_ROOT/src +ENGINE_OUT=$ENGINE_SRC/out +UPDATER_SRC=$ENGINE_SRC/flutter/third_party/updater +HOST_ARCH='darwin-x64' + +# Build the Rust library. +cd $UPDATER_SRC/library + +# Build iOS and MacOS +cargo build \ + --target aarch64-apple-ios \ + --target x86_64-apple-ios \ + --target aarch64-apple-darwin \ + --target x86_64-apple-darwin \ + --release + +# Build the patch tool. +# Again, this belongs as part of the gn build. +cd $UPDATER_SRC/patch +cargo build --release + +# Compile the engine using the steps here: +# https://github.com/flutter/flutter/wiki/Compiling-the-engine#compiling-for-android-from-macos-or-linux +cd $ENGINE_SRC + +NINJA="ninja" +GN=./flutter/tools/gn +ET=./flutter/bin/et +# We could probably use our own prebuilt dart SDK, by modifying the gn files. +# `--no-enable-unittests` is needed on Flutter 3.10.1 and 3.10.2 to avoid +# https://github.com/flutter/flutter/issues/128135 +GN_ARGS="--no-rbe --no-enable-unittests" + +# FIXME: These build commands likely could build fewer targets. + +# Mac doesn't seem to use "archive_gen_snapshot" as a target name yet. +# https://github.com/flutter/flutter/issues/105351#issuecomment-1650686247 +ANDROID_TARGETS="flutter/shell/platform/android:gen_snapshot" + +# Because Flutter does not yet build universal binaries for macOS, we need to +# ensure we're building for x64 for the time being so we can support both Intel +# and Apple Silicon Macs. We do this by telling gn to use host_cpu="x64". + +# Android arm64 release +$GN $GN_ARGS --android --android-cpu=arm64 --runtime-mode=release --gn-args='host_cpu="x64"' +$NINJA -C ./out/android_release_arm64 $ANDROID_TARGETS + +# Android arm32 release +$GN $GN_ARGS --runtime-mode=release --android --gn-args='host_cpu="x64"' +$NINJA -C out/android_release $ANDROID_TARGETS + +# Android x64 release +$GN $GN_ARGS --android --android-cpu=x64 --runtime-mode=release --gn-args='host_cpu="x64"' +$NINJA -C ./out/android_release_x64 $ANDROID_TARGETS + +# We only need two targets (per the mac builders): +# "flutter/shell/platform/darwin/ios:flutter_framework", +# "flutter/lib/snapshot:generate_snapshot_bins", which builds both gen_snapshot and analyze_snapshot binaries. +# https://github.com/flutter/engine/blob/main/ci/builders/mac_ios_engine.json#L139 +# https://github.com/flutter/engine/blob/main/ci/builders/README.md +# The files created by these targets are packaged into a framework and an artifacts.zip file +# by the create_full_ios_framework.py and create_macos_framework.py scripts. + +IOS_TARGETS="flutter/shell/platform/darwin/ios:flutter_framework flutter/lib/snapshot:generate_snapshot_bins" +# You will also need to build vm_platform_strong.dill if you're using a local engine build. + +# From ci/builders/mac_host_engine.json in the engine repo +MACOS_TARGETS="flutter/shell/platform/darwin/macos:zip_macos_flutter_framework flutter/lib/snapshot:generate_snapshot_bins flutter/build/archives:artifacts" + +# Build x64 Dart SDK +$GN $GN_ARGS --runtime-mode=release --mac-cpu=x64 --no-prebuilt-dart-sdk +$NINJA -C out/host_release dart_sdk +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 + +# Build arm64 Dart SDK +$GN $GN_ARGS --runtime-mode=release --mac-cpu=arm64 --no-prebuilt-dart-sdk +$NINJA -C out/host_release_arm64 dart_sdk +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 + +# iOS arm64 release +$GN $GN_ARGS --runtime-mode=release --ios --gn-arg='shorebird_runtime=true' +$NINJA -C out/ios_release $IOS_TARGETS + +$GN $GN_ARGS --ios --runtime-mode=release --darwin-extension-safe --xcode-symlinks --gn-arg='shorebird_runtime=true' +$NINJA -C out/ios_release_extension_safe $IOS_TARGETS + +# iOS simulator-x64 release +$GN $GN_ARGS --runtime-mode=debug --ios --simulator +$NINJA -C out/ios_debug_sim $IOS_TARGETS + +$GN $GN_ARGS --runtime-mode=debug --darwin-extension-safe --ios --simulator +$NINJA -C out/ios_debug_sim_extension_safe $IOS_TARGETS + +# iOS simulator-arm64 release +$GN $GN_ARGS --runtime-mode=debug --ios --simulator --simulator-cpu=arm64 +$NINJA -C out/ios_debug_sim_arm64 $IOS_TARGETS + +$GN $GN_ARGS --runtime-mode=debug --darwin-extension-safe --ios --simulator --simulator-cpu=arm64 +$NINJA -C out/ios_debug_sim_arm64_extension_safe $IOS_TARGETS + +# macOS arm64 release +$GN $GN_ARGS --runtime-mode=release --mac --mac-cpu=arm64 +$NINJA -C out/mac_release_arm64 $MACOS_TARGETS + +# macOS x64 release +# Note: we don't enable the simulator here because the simulator is an arm64 simulator, +# which won't work for x64 apps. +$GN $GN_ARGS --runtime-mode=release --mac --mac-cpu=x64 +$NINJA -C out/mac_release $MACOS_TARGETS + +# The python scripts below fail if the out/release directory already exists. +rm -rf out/release + +# We have to create a composite Flutter.framework for iOS and macOS, matching +# what the Flutter builders do: +IOS_FRAMEWORK_OUT=out/release +echo "Building Flutter.framework for iOS" +python3 flutter/sky/tools/create_ios_framework.py \ + --dst $IOS_FRAMEWORK_OUT \ + --arm64-out-dir out/ios_release \ + --simulator-x64-out-dir out/ios_debug_sim \ + --simulator-arm64-out-dir out/ios_debug_sim_arm64 \ + --dsym \ + --strip +echo "Built Flutter.framework for iOS" + +MAC_FRAMEWORK_OUT=out/release/framework +echo "Building Flutter.framework for macOS" +python3 flutter/sky/tools/create_macos_framework.py \ + --dst $MAC_FRAMEWORK_OUT \ + --arm64-out-dir out/mac_release_arm64 \ + --x64-out-dir out/mac_release \ + --dsym \ + --strip \ + --zip +echo "Built Flutter.framework for macOS" + +echo "Creating macOS gen_snapshot" +python3 flutter/sky/tools/create_macos_gen_snapshots.py \ + --dst out/release/snapshot \ + --arm64-path out/mac_release_arm64/universal/gen_snapshot_arm64 \ + --x64-path out/mac_release/universal/gen_snapshot_x64 \ + --zip +echo "Created macOS gen_snapshot" + +# Zip the dSYMs +zip -r $IOS_FRAMEWORK_OUT/Flutter.framework.dSYM.zip $IOS_FRAMEWORK_OUT/Flutter.framework.dSYM +zip -r $MAC_FRAMEWORK_OUT/FlutterMacOS.framework.dSYM.zip $MAC_FRAMEWORK_OUT/FlutterMacOS.framework.dSYM + +sign_flutter_xcframework() { + pushd $ENGINE_OUT/release + + # Unzip the artifacts zip file, which contains the Flutter.xcframework. + rm -rf artifacts + unzip artifacts.zip -d artifacts + + # Keep a copy of the old artifacts.zip for now, we may decide to remove this later + mv artifacts.zip artifacts.old.zip + + # Sign the Flutter.xcframework + cd artifacts + + # In case the artifacts are already signed, remove the signature + codesign -v --remove-signature Flutter.xcframework + codesign -v --sign "Apple Distribution: Code Town Inc (6V53YACS2W)" Flutter.xcframework + + # Zip the artifacts back up + zip -r "../artifacts.zip" * + + # Cleanup + cd .. + rm -rf artifacts + + popd +} + +sign_flutter_xcframework + +# Create out/engine_stamp.json +# We can remove this explicit step once we're using et in any of the lines +# above. +$ET stamp diff --git a/shorebird/ci/internal/mac_setup.sh b/shorebird/ci/internal/mac_setup.sh new file mode 100755 index 0000000000000..8453f8b9b1a9e --- /dev/null +++ b/shorebird/ci/internal/mac_setup.sh @@ -0,0 +1,11 @@ +#!/bin/bash -e + +# Usage: +# ./mac_setup.sh + +# This assumes rust is installed, but could also install rust/cargo. +rustup target add \ + x86_64-apple-ios \ + aarch64-apple-ios \ + aarch64-apple-darwin \ + x86_64-apple-darwin diff --git a/shorebird/ci/internal/mac_upload.sh b/shorebird/ci/internal/mac_upload.sh new file mode 100755 index 0000000000000..8ec266246a8e1 --- /dev/null +++ b/shorebird/ci/internal/mac_upload.sh @@ -0,0 +1,180 @@ +#!/bin/bash -e + +# Usage: +# ./mac_upload.sh engine_path git_hash + +# Convert to an absolute path so we don't need to worry about cd'ing back to +# the root directory between commands. +ENGINE_ROOT=$(realpath $1) +ENGINE_HASH=$2 + +# Get the absolute path to the directory of this script. +SCRIPT_DIR=$(cd $(dirname $0) && pwd) + +STORAGE_BUCKET="download.shorebird.dev" +SHOREBIRD_ROOT=gs://$STORAGE_BUCKET/shorebird/$ENGINE_HASH + +ENGINE_SRC=$ENGINE_ROOT/src +ENGINE_OUT=$ENGINE_SRC/out +ENGINE_FLUTTER=$ENGINE_SRC/flutter +# FLUTTER_ROOT is the Flutter monorepo root (parent of engine/) +FLUTTER_ROOT=$(dirname $ENGINE_ROOT) + +cd $FLUTTER_ROOT + +# Compute the content-aware hash for the Dart SDK. +# This allows Flutter checkouts that haven't changed engine content to share +# the same pre-built Dart SDK, even if they have different git commit SHAs. +CONTENT_HASH=$($FLUTTER_ROOT/bin/internal/content_aware_hash.sh) +# Can't just `git merge-base` because the engine branches for each +# major version (e.g. 3.7, 3.8) (e.g. upstream/flutter-3.7-candidate.1) +# but it's not clear which branch we're forked from, only that we took +# some tag and added our commits (but we don't know what tag). +BASE_FLUTTER_TAG=`git describe --tags --abbrev=0` +# Read the first line from bin/internal/engine.version file and trim whitespace. +BASE_ENGINE_HASH=`git show $BASE_FLUTTER_TAG:bin/internal/engine.version | head -n 1 | tr -d '[:space:]'` + +# Build the artifacts manifest: +MANIFEST_FILE=`mktemp` +# Note that any uploads which are *not* listed in the manifest will be +# ignored by the artifact proxy. +# if you add uploads here, they also need to be reflected in the manifest. +$SCRIPT_DIR/generate_manifest.sh $BASE_ENGINE_HASH > $MANIFEST_FILE + +# FIXME: This should not be in shell, it's too complicated/repetitive. +# Only need the libflutter.so (and flutter.jar) artifacts +# Artifact list: https://github.com/shorebirdtech/shorebird/blob/main/packages/artifact_proxy/lib/config.dart + +HOST_ARCH='darwin-x64' +ARM64_HOST_ARCH='darwin-arm64' + +INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$ENGINE_HASH" + +# engine_stamp.json +ENGINE_STAMP_FILE=$ENGINE_OUT/engine_stamp.json +gsutil cp $ENGINE_STAMP_FILE $INFRA_ROOT/engine_stamp.json + +# Dart SDK +# This gets uploaded to flutter_infra_release/flutter/\$engine/dart-sdk-$HOST_ARCH.zip +# We also upload to the content-aware hash path to support local development branches. +CONTENT_INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$CONTENT_HASH" + +# x64 Dart SDK +HOST_RELEASE=$ENGINE_OUT/host_release +DART_ZIP_FILE=dart-sdk-$HOST_ARCH.zip +( + cd $HOST_RELEASE; + zip -r $DART_ZIP_FILE dart-sdk +) +ZIPS_DEST=$INFRA_ROOT/$DART_ZIP_FILE +gsutil cp $HOST_RELEASE/$DART_ZIP_FILE $ZIPS_DEST +# Also upload to content-aware hash path +gsutil cp $HOST_RELEASE/$DART_ZIP_FILE $CONTENT_INFRA_ROOT/$DART_ZIP_FILE + +# arm64 Dart SDK +HOST_RELEASE_ARM64=$ENGINE_OUT/host_release_arm64 +DART_ZIP_FILE=dart-sdk-$ARM64_HOST_ARCH.zip +( + cd $HOST_RELEASE_ARM64; + zip -r $DART_ZIP_FILE dart-sdk +) +ZIPS_DEST=$INFRA_ROOT/$DART_ZIP_FILE +gsutil cp $HOST_RELEASE_ARM64/$DART_ZIP_FILE $ZIPS_DEST +# Also upload to content-aware hash path +gsutil cp $HOST_RELEASE_ARM64/$DART_ZIP_FILE $CONTENT_INFRA_ROOT/$DART_ZIP_FILE + +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 +# # mac x64 host_release font_subset (ConstFinder) +# ARCH_OUT=$ENGINE_OUT/host_release +# ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +# ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +# gsutil cp $ZIPS_OUT/font-subset.zip $ZIPS_DEST/font-subset.zip + +# # mac arm64 host_release font_subset (ConstFinder) +# ARCH_OUT=$ENGINE_OUT/host_release_arm64 +# ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +# ZIPS_DEST=$INFRA_ROOT/darwin-arm64-release +# gsutil cp $ZIPS_OUT/font-subset.zip $ZIPS_DEST/font-subset.zip + +# Android Arm64 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release_arm64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm64-release +ZIPS_DEST=$INFRA_ROOT/android-arm64-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Android Arm32 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm-release +ZIPS_DEST=$INFRA_ROOT/android-arm-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Android x64 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release_x64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-x64-release +ZIPS_DEST=$INFRA_ROOT/android-x64-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Match the upload pattern from iOS: +# https://github.com/flutter/engine/commit/1d7f0c66c316a37105601b13136f890f6595aebc + +# iOS release Flutter artifacts +ARCH_OUT=$ENGINE_OUT/release +ZIPS_OUT=$ARCH_OUT +ZIPS_DEST=$INFRA_ROOT/ios-release +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip + +# iOS dSYM +gsutil cp $ZIPS_OUT/Flutter.framework.dSYM.zip $ZIPS_DEST/Flutter.framework.dSYM.zip + +# macOS framework +ARCH_OUT=$ENGINE_OUT/release +ZIPS_OUT=$ARCH_OUT/framework +ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +gsutil cp $ZIPS_OUT/framework.zip $ZIPS_DEST/framework.zip + +# macOS gen_snapshot +ARCH_OUT=$ENGINE_OUT/release +ZIPS_OUT=$ARCH_OUT/snapshot +ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +gsutil cp $ZIPS_OUT/gen_snapshot.zip $ZIPS_DEST/gen_snapshot.zip + +# FIXME: these should go where we're putting the arm64 macOS artifacts +# (darwin-x64-release), however, arm macs use darwin-x64-release and we +# currently only support those. We need to find a way to support both. +# macOS x64 release artifacts +# ARCH_OUT=$ENGINE_OUT/mac_release +# ZIPS_OUT=$ARCH_OUT/zip_archives/darwin-x64-release +# ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +# gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip + +# macOS arm64 release artifacts +ARCH_OUT=$ENGINE_OUT/mac_release_arm64 +ZIPS_OUT=$ARCH_OUT/zip_archives/darwin-arm64-release +# This looks wrong - why are we putting arm64 artifacts in darwin-x64-release +# instead of darwin-arm64-release? This is because arm macs use darwin-x64-release +# and we need to use the artifacts we've built for arm64 macs. +ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip + +# macOS dSYM (used for symbolication, not by Flutter) +ARCH_OUT=$ENGINE_OUT/release +ZIPS_OUT=$ARCH_OUT/framework +ZIPS_DEST=$INFRA_ROOT/darwin-x64 +gsutil cp $ZIPS_OUT/FlutterMacOS.framework.dSYM.zip $ZIPS_DEST/FlutterMacOS.framework.dSYM.zip + +TMP_DIR=$(mktemp -d) + +PATCH_VERSION=0.2.1 +GH_RELEASE=https://github.com/shorebirdtech/updater/releases/download/patch-v$PATCH_VERSION/ +cd $TMP_DIR +curl -L $GH_RELEASE/patch-x86_64-apple-darwin.zip -o patch-x86_64-apple-darwin.zip +curl -L $GH_RELEASE/patch-x86_64-pc-windows-msvc.zip -o patch-x86_64-pc-windows-msvc.zip +curl -L $GH_RELEASE/patch-x86_64-unknown-linux-musl.zip -o patch-x86_64-unknown-linux-musl.zip + +gsutil cp patch-x86_64-apple-darwin.zip $SHOREBIRD_ROOT/patch-darwin-x64.zip +gsutil cp patch-x86_64-pc-windows-msvc.zip $SHOREBIRD_ROOT/patch-windows-x64.zip +gsutil cp patch-x86_64-unknown-linux-musl.zip $SHOREBIRD_ROOT/patch-linux-x64.zip + +gsutil cp $MANIFEST_FILE $SHOREBIRD_ROOT/artifacts_manifest.yaml diff --git a/shorebird/ci/internal/win_build.sh b/shorebird/ci/internal/win_build.sh new file mode 100755 index 0000000000000..2df02670aa78f --- /dev/null +++ b/shorebird/ci/internal/win_build.sh @@ -0,0 +1,63 @@ +#!/bin/bash -e + +# Usage: +# ./win_build.sh engine_path + +ENGINE_ROOT=$1 + +ENGINE_SRC=$ENGINE_ROOT/src +ENGINE_OUT=$ENGINE_SRC/out +UPDATER_SRC=$ENGINE_SRC/flutter/third_party/updater +HOST_ARCH='windows-x64' + +# Build the Rust library. +cd $UPDATER_SRC/library + +cargo build --release \ + --target x86_64-pc-windows-msvc + +# Compile the engine using the steps here: +# https://github.com/flutter/flutter/wiki/Compiling-the-engine#compiling-for-android-from-macos-or-linux +cd $ENGINE_SRC + +NINJA="ninja" +GN=./flutter/tools/gn +# We could probably use our own prebuilt dart SDK, by modifying the gn files. +GN_ARGS="--no-rbe --no-enable-unittests" + +# Windows only needs gen_snapshot for each Android CPU type. +# See https://github.com/flutter/engine/blob/e590b24f3962fda3ec9144dcee3f7565b195839a/ci/builders/windows_android_aot_engine.json + +TARGETS="archive_win_gen_snapshot" + +# Build host_release +$GN $GN_ARGS --runtime-mode=release --no-prebuilt-dart-sdk +$NINJA -C out/host_release dart_sdk +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 + +# Build windows desktop targets +$GN $GN_ARGS --runtime-mode=release --no-prebuilt-dart-sdk +$NINJA -C ./out/host_release flutter/build/archives:windows_flutter gen_snapshot windows flutter/build/archives:artifacts + +# Build debug Windows artifacts +# These are output to the `windows-x64` directory in host_debug, and are used +# by `flutter build windows --release`. +$GN $GN_ARGS --no-prebuilt-dart-sdk +$NINJA -C ./out/host_debug flutter/build/archives:artifacts + +# If this gives you trouble, try using VS2019 instead. I had trouble with 2022. +# Android arm64 release +$GN $GN_ARGS --android --android-cpu=arm64 --runtime-mode=release +$NINJA -C ./out/android_release_arm64 $TARGETS + +# Android arm32 release +$GN $GN_ARGS --runtime-mode=release --android +$NINJA -C out/android_release $TARGETS + +# Android x64 release +$GN $GN_ARGS --android --android-cpu=x64 --runtime-mode=release +$NINJA -C ./out/android_release_x64 $TARGETS + +# We could also build the `patch` tool for Windows here. diff --git a/shorebird/ci/internal/win_setup.sh b/shorebird/ci/internal/win_setup.sh new file mode 100755 index 0000000000000..971135644f66b --- /dev/null +++ b/shorebird/ci/internal/win_setup.sh @@ -0,0 +1,8 @@ +#!/bin/bash -e + +# Usage: +# ./windows_setup.sh + +# Add the MSVC toolchain to Rust. +rustup target add \ + x86_64-pc-windows-msvc diff --git a/shorebird/ci/internal/win_upload.sh b/shorebird/ci/internal/win_upload.sh new file mode 100755 index 0000000000000..952457109e40c --- /dev/null +++ b/shorebird/ci/internal/win_upload.sh @@ -0,0 +1,90 @@ +#!/bin/bash -e + +# Usage: +# ./win_upload.sh engine_path git_hash +ENGINE_ROOT=$1 +ENGINE_HASH=$2 + +STORAGE_BUCKET="download.shorebird.dev" +SHOREBIRD_ROOT=gs://$STORAGE_BUCKET/shorebird/$ENGINE_HASH + +ENGINE_SRC=$ENGINE_ROOT/src +ENGINE_OUT=$ENGINE_SRC/out +ENGINE_FLUTTER=$ENGINE_SRC/flutter +# FLUTTER_ROOT is the Flutter monorepo root (parent of engine/) +FLUTTER_ROOT=$(dirname $ENGINE_ROOT) + +cd $FLUTTER_ROOT + +# Compute the content-aware hash for the Dart SDK. +# This allows Flutter checkouts that haven't changed engine content to share +# the same pre-built Dart SDK, even if they have different git commit SHAs. +CONTENT_HASH=$($FLUTTER_ROOT/bin/internal/content_aware_hash.sh) + +# We do not generate a manifest file, we assume another builder did that. + +# TODO(eseidel): This should not be in shell, it's too complicated/repetitive. + +HOST_ARCH='windows-x64' + +INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$ENGINE_HASH" + +# Dart SDK +# This gets uploaded to flutter_infra_release/flutter/\$engine/dart-sdk-$HOST_ARCH.zip +# We also upload to the content-aware hash path to support local development branches. +CONTENT_INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$CONTENT_HASH" + +DART_SDK_DIR=$ENGINE_OUT/host_release/dart-sdk +DART_ZIP_FILE=dart-sdk-$HOST_ARCH.zip + +# Use 7zip to compress the Dart SDK, as zip isn't available on Windows and +# Powershell, which we would normally use in the form of +# `powershell Compress-Archive dart-sdk dart-sdk.zip`, doesn't play nicely +# with git bash paths (e.g. /c/Users/... instead of C:/Users/...) +/c/Program\ Files/7-Zip/7z a $DART_ZIP_FILE $DART_SDK_DIR +ZIPS_DEST=$INFRA_ROOT/$DART_ZIP_FILE +gsutil cp $DART_ZIP_FILE $ZIPS_DEST +# Also upload to content-aware hash path +gsutil cp $DART_ZIP_FILE $CONTENT_INFRA_ROOT/$DART_ZIP_FILE + +# Android Arm64 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release_arm64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm64-release +ZIPS_DEST=$INFRA_ROOT/android-arm64-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Android Arm32 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm-release +ZIPS_DEST=$INFRA_ROOT/android-arm-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Android x64 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release_x64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-x64-release +ZIPS_DEST=$INFRA_ROOT/android-x64-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# We could upload patch if we built it here. +# gsutil cp $ENGINE_OUT/host_release/patch.zip $SHOREBIRD_ROOT/patch-win-x64.zip + +# Engine release artifacts +ARCH_OUT=$ENGINE_OUT/host_release +ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH-release +ZIPS_DEST=$INFRA_ROOT/$HOST_ARCH-release +gsutil cp $ZIPS_OUT/$HOST_ARCH-flutter.zip $ZIPS_DEST/$HOST_ARCH-flutter.zip + +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 +# # Windows x64 host_release font_subset (ConstFinder) +# ARCH_OUT=$ENGINE_OUT/host_release +# ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +# ZIPS_DEST=$INFRA_ROOT/windows-x64-release +# gsutil cp $ZIPS_OUT/font-subset.zip $ZIPS_DEST/font-subset.zip + +# Engine debug artifacts (not sure why this is needed?) +ARCH_OUT=$ENGINE_OUT/host_debug +ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +ZIPS_DEST=$INFRA_ROOT/$HOST_ARCH +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip diff --git a/shorebird/ci/linux_build_and_upload.sh b/shorebird/ci/linux_build_and_upload.sh new file mode 100755 index 0000000000000..9657e14178695 --- /dev/null +++ b/shorebird/ci/linux_build_and_upload.sh @@ -0,0 +1,32 @@ +#!/bin/bash -e + +# Usage: +# ./linux_build_and_upload.sh flutter_root engine_hash +# +# This is the main entrypoint for building and uploading Linux engine artifacts. +# It is called from the _build_engine repository's CI scripts. + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 flutter_root engine_hash" + exit 1 +fi + +FLUTTER_ROOT=$1 +ENGINE_HASH=$2 +ENGINE_ROOT=$FLUTTER_ROOT/engine + +# Get the absolute path to the directory of this script. +SCRIPT_DIR=$(cd $(dirname $0) && pwd) + +echo "Building engine at $ENGINE_ROOT and uploading to gs://download.shorebird.dev" + +cd $SCRIPT_DIR + +# Run the setup script. +./internal/linux_setup.sh + +# Then run the build. +./internal/linux_build.sh $ENGINE_ROOT + +# Copy Shorebird engine artifacts to Google Cloud Storage. +./internal/linux_upload.sh $ENGINE_ROOT $ENGINE_HASH diff --git a/shorebird/ci/mac_build_and_upload.sh b/shorebird/ci/mac_build_and_upload.sh new file mode 100755 index 0000000000000..9ee302620d207 --- /dev/null +++ b/shorebird/ci/mac_build_and_upload.sh @@ -0,0 +1,32 @@ +#!/bin/bash -e + +# Usage: +# ./mac_build_and_upload.sh flutter_root engine_hash +# +# This is the main entrypoint for building and uploading macOS engine artifacts. +# It is called from the _build_engine repository's CI scripts. + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 flutter_root engine_hash" + exit 1 +fi + +FLUTTER_ROOT=$1 +ENGINE_HASH=$2 +ENGINE_ROOT=$FLUTTER_ROOT/engine + +# Get the absolute path to the directory of this script. +SCRIPT_DIR=$(cd $(dirname $0) && pwd) + +echo "Building engine at $ENGINE_ROOT and uploading to gs://download.shorebird.dev" + +cd $SCRIPT_DIR + +# Run the setup script. +./internal/mac_setup.sh + +# Then run the build. +./internal/mac_build.sh $ENGINE_ROOT + +# Copy Shorebird engine artifacts to Google Cloud Storage. +./internal/mac_upload.sh $ENGINE_ROOT $ENGINE_HASH diff --git a/shorebird/ci/win_build_and_upload.sh b/shorebird/ci/win_build_and_upload.sh new file mode 100755 index 0000000000000..bbda403ff2c67 --- /dev/null +++ b/shorebird/ci/win_build_and_upload.sh @@ -0,0 +1,32 @@ +#!/bin/bash -e + +# Usage: +# ./win_build_and_upload.sh flutter_root engine_hash +# +# This is the main entrypoint for building and uploading Windows engine artifacts. +# It is called from the _build_engine repository's CI scripts. + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 flutter_root engine_hash" + exit 1 +fi + +FLUTTER_ROOT=$1 +ENGINE_HASH=$2 +ENGINE_ROOT=$FLUTTER_ROOT/engine + +# Get the absolute path to the directory of this script. +SCRIPT_DIR=$(cd $(dirname $0) && pwd) + +echo "Building engine at $ENGINE_ROOT and uploading to gs://download.shorebird.dev" + +cd $SCRIPT_DIR + +# Run the setup script. +./internal/win_setup.sh + +# Then run the build. +./internal/win_build.sh $ENGINE_ROOT + +# Copy Shorebird engine artifacts to Google Cloud Storage. +./internal/win_upload.sh $ENGINE_ROOT $ENGINE_HASH From 3207ad178d8a2a8730af4c6ee802e7a826039685 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 15 Dec 2025 16:55:22 -0800 Subject: [PATCH 11/51] chore: update to use new CreateGroupIsolate API (#99) * chore: roll dart to 6a78a2deaee05bc74775fcfa2ff27aa53c96efca * wip * chore: run et format * chore: attempt to clean up shorebird.cc * chore: fix build * chore: remove FLUTTER_STORAGE_BASE_URL override --- .github/workflows/shorebird_ci.yml | 5 -- engine/src/flutter/runtime/dart_isolate.cc | 31 +++++++++- .../runtime/dart_isolate_group_data.cc | 8 ++- .../flutter/runtime/dart_isolate_group_data.h | 8 ++- .../shell/common/shorebird/shorebird.cc | 61 +++++++++++++------ .../shell/common/shorebird/shorebird.h | 19 ++++++ 6 files changed, 105 insertions(+), 27 deletions(-) diff --git a/.github/workflows/shorebird_ci.yml b/.github/workflows/shorebird_ci.yml index 7b712b1583b87..9597d6a599344 100644 --- a/.github/workflows/shorebird_ci.yml +++ b/.github/workflows/shorebird_ci.yml @@ -21,11 +21,6 @@ jobs: name: ๐Ÿฆ Shorebird Test - # TODO(eseidel): This is also set inside shorebird_tests, unclear if - # if it's needed here as well. - env: - FLUTTER_STORAGE_BASE_URL: https://download.shorebird.dev - steps: - name: ๐Ÿ“š Git Checkout uses: actions/checkout@v4 diff --git a/engine/src/flutter/runtime/dart_isolate.cc b/engine/src/flutter/runtime/dart_isolate.cc index 521c284f1c905..3735cd03fcd0e 100644 --- a/engine/src/flutter/runtime/dart_isolate.cc +++ b/engine/src/flutter/runtime/dart_isolate.cc @@ -21,6 +21,10 @@ #include "flutter/runtime/isolate_configuration.h" #include "flutter/runtime/platform_isolate_manager.h" #include "fml/message_loop_task_queues.h" + +#if SHOREBIRD_USE_INTERPRETER +#include "flutter/shell/common/shorebird/shorebird.h" // nogncheck +#endif #include "fml/task_source.h" #include "fml/time/time_point.h" #include "third_party/dart/runtime/include/bin/native_assets_api.h" @@ -245,6 +249,12 @@ std::weak_ptr DartIsolate::CreateRootIsolate( } else { // The child isolate preparer is null but will be set when the isolate is // being prepared to run. +#if SHOREBIRD_USE_INTERPRETER + // Get the base snapshot for Shorebird linking support (may be null). + fml::RefPtr base_snapshot = GetBaseIsolateSnapshot(); +#else + fml::RefPtr base_snapshot = nullptr; +#endif isolate_group_data = std::make_unique>( std::shared_ptr(new DartIsolateGroupData( @@ -254,13 +264,30 @@ std::weak_ptr DartIsolate::CreateRootIsolate( context.advisory_script_entrypoint, // advisory entrypoint nullptr, // child isolate preparer isolate_create_callback, // isolate create callback - isolate_shutdown_callback, // isolate shutdown callback - std::move(native_assets_manager) // + isolate_shutdown_callback, // isolate shutdown callback + std::move(native_assets_manager), // native assets manager + std::move(base_snapshot) // base snapshot (Shorebird) ))); isolate_maker = [](std::shared_ptr* isolate_group_data, std::shared_ptr* isolate_data, Dart_IsolateFlags* flags, char** error) { +#if SHOREBIRD_USE_INTERPRETER + auto base_snapshot = (*isolate_group_data)->GetBaseSnapshot(); + if (base_snapshot) { + // Use the Shorebird API that accepts base snapshot for linking. + return Dart_CreateIsolateGroupWithBaseSnapshot( + (*isolate_group_data)->GetAdvisoryScriptURI().c_str(), + (*isolate_group_data)->GetAdvisoryScriptEntrypoint().c_str(), + (*isolate_group_data)->GetIsolateSnapshot()->GetDataMapping(), + (*isolate_group_data) + ->GetIsolateSnapshot() + ->GetInstructionsMapping(), + base_snapshot->GetDataMapping(), + base_snapshot->GetInstructionsMapping(), flags, isolate_group_data, + isolate_data, error); + } +#endif return Dart_CreateIsolateGroup( (*isolate_group_data)->GetAdvisoryScriptURI().c_str(), (*isolate_group_data)->GetAdvisoryScriptEntrypoint().c_str(), diff --git a/engine/src/flutter/runtime/dart_isolate_group_data.cc b/engine/src/flutter/runtime/dart_isolate_group_data.cc index 0e844d3e3c417..a8ecffee5b1ea 100644 --- a/engine/src/flutter/runtime/dart_isolate_group_data.cc +++ b/engine/src/flutter/runtime/dart_isolate_group_data.cc @@ -18,9 +18,11 @@ DartIsolateGroupData::DartIsolateGroupData( const ChildIsolatePreparer& child_isolate_preparer, const fml::closure& isolate_create_callback, const fml::closure& isolate_shutdown_callback, - std::shared_ptr native_assets_manager) + std::shared_ptr native_assets_manager, + fml::RefPtr base_snapshot) : settings_(settings), isolate_snapshot_(std::move(isolate_snapshot)), + base_snapshot_(std::move(base_snapshot)), advisory_script_uri_(std::move(advisory_script_uri)), advisory_script_entrypoint_(std::move(advisory_script_entrypoint)), child_isolate_preparer_(child_isolate_preparer), @@ -41,6 +43,10 @@ fml::RefPtr DartIsolateGroupData::GetIsolateSnapshot() return isolate_snapshot_; } +fml::RefPtr DartIsolateGroupData::GetBaseSnapshot() const { + return base_snapshot_; +} + const std::string& DartIsolateGroupData::GetAdvisoryScriptURI() const { return advisory_script_uri_; } diff --git a/engine/src/flutter/runtime/dart_isolate_group_data.h b/engine/src/flutter/runtime/dart_isolate_group_data.h index 8ff29272150ab..1502b2fc8e290 100644 --- a/engine/src/flutter/runtime/dart_isolate_group_data.h +++ b/engine/src/flutter/runtime/dart_isolate_group_data.h @@ -39,7 +39,8 @@ class DartIsolateGroupData : public PlatformMessageHandlerStorage { const ChildIsolatePreparer& child_isolate_preparer, const fml::closure& isolate_create_callback, const fml::closure& isolate_shutdown_callback, - std::shared_ptr native_assets_manager = nullptr); + std::shared_ptr native_assets_manager = nullptr, + fml::RefPtr base_snapshot = nullptr); ~DartIsolateGroupData(); @@ -47,6 +48,10 @@ class DartIsolateGroupData : public PlatformMessageHandlerStorage { fml::RefPtr GetIsolateSnapshot() const; + /// Returns the base snapshot for Shorebird linking support. + /// May be null if not using Shorebird or if no patch is active. + fml::RefPtr GetBaseSnapshot() const; + const std::string& GetAdvisoryScriptURI() const; const std::string& GetAdvisoryScriptEntrypoint() const; @@ -81,6 +86,7 @@ class DartIsolateGroupData : public PlatformMessageHandlerStorage { std::vector> kernel_buffers_; const Settings settings_; const fml::RefPtr isolate_snapshot_; + const fml::RefPtr base_snapshot_; const std::string advisory_script_uri_; const std::string advisory_script_entrypoint_; mutable std::mutex child_isolate_preparer_mutex_; diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc index ec2bc6bf3c9cd..8cb32cfc9916b 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.cc +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -45,24 +45,38 @@ extern "C" __attribute__((weak)) unsigned long getauxval(unsigned long type) { } #endif -// TODO(eseidel): I believe we need to leak these or we'll sometimes crash -// when using the base snapshot in mixed mode. This likely will not play -// nicely with multi-engine support and will need to be refactored. +#if SHOREBIRD_USE_INTERPRETER + +// Global references to the base (unpatched) snapshots from the App.framework. +// These are process-global because: +// 1. The Shorebird updater library is a process-global singleton with its own +// internal state. FileCallbacksImpl provides it access to the base snapshot +// data for patch generation/validation. +// 2. The base snapshots are immutable (baked into the IPA) so sharing them +// across isolate groups is safe. +// 3. GetBaseIsolateSnapshot() returns the isolate snapshot for use when +// creating isolate groups with Dart_CreateIsolateGroupWithBaseSnapshot(), +// which needs the base snapshot for linking patched code. +// +// Note: This design doesn't support multiple engines with different base +// snapshots, but I'm not aware of any use cases for that on iOS. static fml::RefPtr vm_snapshot; static fml::RefPtr isolate_snapshot; -void SetBaseSnapshot(Settings& settings) { - // These mappings happen to be to static data in the App.framework, but - // we still need to seem to hold onto the DartSnapshot objects to keep - // the mappings alive. +void StoreBaseSnapshots(Settings& settings) { + // Create DartSnapshot objects that hold references to the symbol mappings + // in the App.framework. The snapshots are static data in the framework, + // but we need DartSnapshot objects to keep the NativeLibrary refs alive. vm_snapshot = DartSnapshot::VMSnapshotFromSettings(settings); isolate_snapshot = DartSnapshot::IsolateSnapshotFromSettings(settings); - Shorebird_SetBaseSnapshots(isolate_snapshot->GetDataMapping(), - isolate_snapshot->GetInstructionsMapping(), - vm_snapshot->GetDataMapping(), - vm_snapshot->GetInstructionsMapping()); } +fml::RefPtr GetBaseIsolateSnapshot() { + return isolate_snapshot; +} + +#endif // SHOREBIRD_USE_INTERPRETER + class FileCallbacksImpl { public: static void* Open(); @@ -104,7 +118,9 @@ std::string GetValueFromYaml(const std::string& yaml, const std::string& key) { return ""; } -// FIXME: consolidate this with the other ConfigureShorebird +/// Newer api, used by Desktop implementations. +/// Does not directly manipulate Settings. +// FIXME: Consolidate this with the other ConfigureShorebird() API. bool ConfigureShorebird(const ShorebirdConfigArgs& args, std::string& patch_path) { patch_path = fml::PathToUtf8(args.release_app_library_path); @@ -201,6 +217,8 @@ bool ConfigureShorebird(const ShorebirdConfigArgs& args, return true; } +/// Older api used by iOS and Android, directly manipulates Settings. +// FIXME: Consolidate this with the other ConfigureShorebird() API. void ConfigureShorebird(std::string code_cache_path, std::string app_storage_path, Settings& settings, @@ -251,15 +269,14 @@ void ConfigureShorebird(std::string code_cache_path, shorebird_yaml.c_str()); } - // We've decided not to support synchronous updates on launch for now. - // It's a terrible user experience (having the app hang on launch) and - // instead we will provide examples of how to build a custom update UI - // within Dart, including updating as part of login, etc. + // We do not support synchronous updates on launch, it's a terrible UX. + // Users can implement custom check-for-updates using + // package:shorebird_code_push. // https://github.com/shorebirdtech/shorebird/issues/950 - // We only set the base snapshot on iOS for now. + // We store the base snapshot on iOS for use when creating the isolate group. #if SHOREBIRD_USE_INTERPRETER - SetBaseSnapshot(settings); + StoreBaseSnapshots(settings); #endif shorebird_validate_next_boot_patch(); @@ -312,9 +329,17 @@ void ConfigureShorebird(std::string code_cache_path, } void* FileCallbacksImpl::Open() { +#if SHOREBIRD_USE_INTERPRETER return SnapshotsDataHandle::createForSnapshots(*vm_snapshot, *isolate_snapshot) .release(); +#else + // SnapshotsDataHandle exists on all platforms (for testing)but is only used + // on iOS. iOS patches are generated from just the Dart parts of the snapshot, + // excluding the Mach-O specific headers which contain dates and paths that + // make them change on every build. + return nullptr; +#endif // SHOREBIRD_USE_INTERPRETER } uintptr_t FileCallbacksImpl::Read(void* file, diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.h b/engine/src/flutter/shell/common/shorebird/shorebird.h index c6873c5014a00..50c6f59e1fa60 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.h +++ b/engine/src/flutter/shell/common/shorebird/shorebird.h @@ -2,15 +2,22 @@ #define FLUTTER_SHELL_COMMON_SHOREBIRD_SHOREBIRD_H_ #include "flutter/common/settings.h" +#include "flutter/fml/memory/ref_ptr.h" #include "shell/platform/embedder/embedder.h" namespace flutter { +class DartSnapshot; + +/// Version and build number of the release. +/// Used by ShorebirdConfigArgs. struct ReleaseVersion { std::string version; std::string build_number; }; +/// Arguments for ConfigureShorebird. +/// Used by Desktop implementations. struct ShorebirdConfigArgs { std::string code_cache_path; std::string app_storage_path; @@ -30,9 +37,12 @@ struct ShorebirdConfigArgs { release_version(release_version) {} }; +/// Newer api, used by Desktop implementations. +/// Does not directly manipulate Settings. bool ConfigureShorebird(const ShorebirdConfigArgs& args, std::string& patch_path); +/// Older api used by iOS and Android, directly manipulates Settings. void ConfigureShorebird(std::string code_cache_path, std::string app_storage_path, Settings& settings, @@ -40,8 +50,17 @@ void ConfigureShorebird(std::string code_cache_path, const std::string& version, const std::string& version_code); +/// Used for reading app_id from shorebird.yaml. +/// Exposed for testing. std::string GetValueFromYaml(const std::string& yaml, const std::string& key); +#if SHOREBIRD_USE_INTERPRETER +/// Returns the base isolate snapshot for Shorebird linking support. +/// Must be called after ConfigureShorebird() has stored the base snapshots. +/// May return null if not using Shorebird interpreter mode. +fml::RefPtr GetBaseIsolateSnapshot(); +#endif // SHOREBIRD_USE_INTERPRETER + } // namespace flutter #endif // FLUTTER_SHELL_COMMON_SHOREBIRD_SHOREBIRD_H_ From 9c591c460632083584a29486b2e7e6b42db43155 Mon Sep 17 00:00:00 2001 From: Bryan Oltman Date: Fri, 19 Dec 2025 13:18:34 -0500 Subject: [PATCH 12/51] bump updater rev --- DEPS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPS b/DEPS index 8f74948904f13..d772d67e5c623 100644 --- a/DEPS +++ b/DEPS @@ -19,7 +19,7 @@ vars = { "dart_sdk_revision": "b65ce89c8057d6880e00693a7b0ecd7b9e5f61ca", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", - "updater_rev": "76f005940db57c38b479cee858abc0cfbd12ac28", + "updater_rev": "9db198a6344f7b9ec2843e06a459938a520e314f", # WARNING: DO NOT EDIT canvaskit_cipd_instance MANUALLY # See `lib/web_ui/README.md` for how to roll CanvasKit to a new version. From 9f0182b32e07ba86df4f93d8e5a95c1a06e5d87b Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Tue, 30 Dec 2025 16:10:40 -0800 Subject: [PATCH 13/51] fix: crash after patching on iOS --- engine/src/flutter/runtime/dart_isolate.cc | 20 +++++++++- shorebird/docs/BUILDING.md | 44 ++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 shorebird/docs/BUILDING.md diff --git a/engine/src/flutter/runtime/dart_isolate.cc b/engine/src/flutter/runtime/dart_isolate.cc index 3735cd03fcd0e..c1e4fb5911f48 100644 --- a/engine/src/flutter/runtime/dart_isolate.cc +++ b/engine/src/flutter/runtime/dart_isolate.cc @@ -1135,7 +1135,9 @@ Dart_Isolate DartIsolate::DartIsolateGroupCreateCallback( advisory_script_entrypoint, parent_group_data.GetChildIsolatePreparer(), parent_group_data.GetIsolateCreateCallback(), - parent_group_data.GetIsolateShutdownCallback()))); + parent_group_data.GetIsolateShutdownCallback(), + nullptr, // native_assets_manager + parent_group_data.GetBaseSnapshot()))); TaskRunners null_task_runners(advisory_script_uri, /* platform= */ nullptr, @@ -1157,6 +1159,22 @@ Dart_Isolate DartIsolate::DartIsolateGroupCreateCallback( [](std::shared_ptr* isolate_group_data, std::shared_ptr* isolate_data, Dart_IsolateFlags* flags, char** error) { +#if SHOREBIRD_USE_INTERPRETER + auto base_snapshot = (*isolate_group_data)->GetBaseSnapshot(); + if (base_snapshot) { + // Use the Shorebird API that accepts base snapshot for linking. + return Dart_CreateIsolateGroupWithBaseSnapshot( + (*isolate_group_data)->GetAdvisoryScriptURI().c_str(), + (*isolate_group_data)->GetAdvisoryScriptEntrypoint().c_str(), + (*isolate_group_data)->GetIsolateSnapshot()->GetDataMapping(), + (*isolate_group_data) + ->GetIsolateSnapshot() + ->GetInstructionsMapping(), + base_snapshot->GetDataMapping(), + base_snapshot->GetInstructionsMapping(), flags, + isolate_group_data, isolate_data, error); + } +#endif return Dart_CreateIsolateGroup( (*isolate_group_data)->GetAdvisoryScriptURI().c_str(), (*isolate_group_data)->GetAdvisoryScriptEntrypoint().c_str(), diff --git a/shorebird/docs/BUILDING.md b/shorebird/docs/BUILDING.md new file mode 100644 index 0000000000000..dfd3c655a1324 --- /dev/null +++ b/shorebird/docs/BUILDING.md @@ -0,0 +1,44 @@ +# Building the Shorebird Engine Locally + +This document explains how to build the Shorebird engine locally for different platforms. + +All commands assume you are running from the `engine/src` directory. + +## Prerequisites + +- Follow the standard Flutter engine setup instructions +- Ensure you have the necessary toolchains installed for your target platform + +## macOS + +### iOS (arm64) + +```bash +./flutter/tools/gn --no-rbe --no-enable-unittests --runtime-mode=release --ios --gn-arg='shorebird_runtime=true' +ninja -C out/ios_release flutter/shell/platform/darwin/ios:flutter_framework flutter/lib/snapshot:generate_snapshot_bins +``` + +### Android arm64 + +```bash +./flutter/tools/gn --no-rbe --no-enable-unittests --android --android-cpu=arm64 --runtime-mode=release --gn-args='host_cpu="x64"' +ninja -C out/android_release_arm64 flutter/shell/platform/android:gen_snapshot +``` + +## Windows + +### Android arm64 + +```bash +./flutter/tools/gn --no-rbe --no-enable-unittests --android --android-cpu=arm64 --runtime-mode=release +ninja -C out/android_release_arm64 archive_win_gen_snapshot +``` + +## Linux + +### Android arm64 + +```bash +./flutter/tools/gn --no-rbe --no-enable-unittests --android --android-cpu=arm64 --runtime-mode=release +ninja -C out/android_release_arm64 default gen_snapshot +``` From 47cc69796cea4f509037ede0c72d7eb236fbc6a9 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 5 Jan 2026 16:53:37 -0800 Subject: [PATCH 14/51] fix: revert to using Shorebird_SetBaseSnapshots to stop iOS crashers --- engine/src/flutter/runtime/dart_isolate.cc | 51 ++----------------- .../runtime/dart_isolate_group_data.cc | 8 +-- .../flutter/runtime/dart_isolate_group_data.h | 8 +-- .../flutter/runtime/shorebird/patch_cache.cc | 7 ++- .../shell/common/shorebird/shorebird.cc | 38 ++++++-------- .../shell/common/shorebird/shorebird.h | 7 --- 6 files changed, 25 insertions(+), 94 deletions(-) diff --git a/engine/src/flutter/runtime/dart_isolate.cc b/engine/src/flutter/runtime/dart_isolate.cc index c1e4fb5911f48..c81ab7c7bd44d 100644 --- a/engine/src/flutter/runtime/dart_isolate.cc +++ b/engine/src/flutter/runtime/dart_isolate.cc @@ -21,10 +21,6 @@ #include "flutter/runtime/isolate_configuration.h" #include "flutter/runtime/platform_isolate_manager.h" #include "fml/message_loop_task_queues.h" - -#if SHOREBIRD_USE_INTERPRETER -#include "flutter/shell/common/shorebird/shorebird.h" // nogncheck -#endif #include "fml/task_source.h" #include "fml/time/time_point.h" #include "third_party/dart/runtime/include/bin/native_assets_api.h" @@ -249,12 +245,6 @@ std::weak_ptr DartIsolate::CreateRootIsolate( } else { // The child isolate preparer is null but will be set when the isolate is // being prepared to run. -#if SHOREBIRD_USE_INTERPRETER - // Get the base snapshot for Shorebird linking support (may be null). - fml::RefPtr base_snapshot = GetBaseIsolateSnapshot(); -#else - fml::RefPtr base_snapshot = nullptr; -#endif isolate_group_data = std::make_unique>( std::shared_ptr(new DartIsolateGroupData( @@ -264,30 +254,13 @@ std::weak_ptr DartIsolate::CreateRootIsolate( context.advisory_script_entrypoint, // advisory entrypoint nullptr, // child isolate preparer isolate_create_callback, // isolate create callback - isolate_shutdown_callback, // isolate shutdown callback - std::move(native_assets_manager), // native assets manager - std::move(base_snapshot) // base snapshot (Shorebird) + isolate_shutdown_callback, // isolate shutdown callback + std::move(native_assets_manager) // ))); isolate_maker = [](std::shared_ptr* isolate_group_data, std::shared_ptr* isolate_data, Dart_IsolateFlags* flags, char** error) { -#if SHOREBIRD_USE_INTERPRETER - auto base_snapshot = (*isolate_group_data)->GetBaseSnapshot(); - if (base_snapshot) { - // Use the Shorebird API that accepts base snapshot for linking. - return Dart_CreateIsolateGroupWithBaseSnapshot( - (*isolate_group_data)->GetAdvisoryScriptURI().c_str(), - (*isolate_group_data)->GetAdvisoryScriptEntrypoint().c_str(), - (*isolate_group_data)->GetIsolateSnapshot()->GetDataMapping(), - (*isolate_group_data) - ->GetIsolateSnapshot() - ->GetInstructionsMapping(), - base_snapshot->GetDataMapping(), - base_snapshot->GetInstructionsMapping(), flags, isolate_group_data, - isolate_data, error); - } -#endif return Dart_CreateIsolateGroup( (*isolate_group_data)->GetAdvisoryScriptURI().c_str(), (*isolate_group_data)->GetAdvisoryScriptEntrypoint().c_str(), @@ -1136,8 +1109,8 @@ Dart_Isolate DartIsolate::DartIsolateGroupCreateCallback( parent_group_data.GetChildIsolatePreparer(), parent_group_data.GetIsolateCreateCallback(), parent_group_data.GetIsolateShutdownCallback(), - nullptr, // native_assets_manager - parent_group_data.GetBaseSnapshot()))); + nullptr // native_assets_manager + ))); TaskRunners null_task_runners(advisory_script_uri, /* platform= */ nullptr, @@ -1159,22 +1132,6 @@ Dart_Isolate DartIsolate::DartIsolateGroupCreateCallback( [](std::shared_ptr* isolate_group_data, std::shared_ptr* isolate_data, Dart_IsolateFlags* flags, char** error) { -#if SHOREBIRD_USE_INTERPRETER - auto base_snapshot = (*isolate_group_data)->GetBaseSnapshot(); - if (base_snapshot) { - // Use the Shorebird API that accepts base snapshot for linking. - return Dart_CreateIsolateGroupWithBaseSnapshot( - (*isolate_group_data)->GetAdvisoryScriptURI().c_str(), - (*isolate_group_data)->GetAdvisoryScriptEntrypoint().c_str(), - (*isolate_group_data)->GetIsolateSnapshot()->GetDataMapping(), - (*isolate_group_data) - ->GetIsolateSnapshot() - ->GetInstructionsMapping(), - base_snapshot->GetDataMapping(), - base_snapshot->GetInstructionsMapping(), flags, - isolate_group_data, isolate_data, error); - } -#endif return Dart_CreateIsolateGroup( (*isolate_group_data)->GetAdvisoryScriptURI().c_str(), (*isolate_group_data)->GetAdvisoryScriptEntrypoint().c_str(), diff --git a/engine/src/flutter/runtime/dart_isolate_group_data.cc b/engine/src/flutter/runtime/dart_isolate_group_data.cc index a8ecffee5b1ea..0e844d3e3c417 100644 --- a/engine/src/flutter/runtime/dart_isolate_group_data.cc +++ b/engine/src/flutter/runtime/dart_isolate_group_data.cc @@ -18,11 +18,9 @@ DartIsolateGroupData::DartIsolateGroupData( const ChildIsolatePreparer& child_isolate_preparer, const fml::closure& isolate_create_callback, const fml::closure& isolate_shutdown_callback, - std::shared_ptr native_assets_manager, - fml::RefPtr base_snapshot) + std::shared_ptr native_assets_manager) : settings_(settings), isolate_snapshot_(std::move(isolate_snapshot)), - base_snapshot_(std::move(base_snapshot)), advisory_script_uri_(std::move(advisory_script_uri)), advisory_script_entrypoint_(std::move(advisory_script_entrypoint)), child_isolate_preparer_(child_isolate_preparer), @@ -43,10 +41,6 @@ fml::RefPtr DartIsolateGroupData::GetIsolateSnapshot() return isolate_snapshot_; } -fml::RefPtr DartIsolateGroupData::GetBaseSnapshot() const { - return base_snapshot_; -} - const std::string& DartIsolateGroupData::GetAdvisoryScriptURI() const { return advisory_script_uri_; } diff --git a/engine/src/flutter/runtime/dart_isolate_group_data.h b/engine/src/flutter/runtime/dart_isolate_group_data.h index 1502b2fc8e290..8ff29272150ab 100644 --- a/engine/src/flutter/runtime/dart_isolate_group_data.h +++ b/engine/src/flutter/runtime/dart_isolate_group_data.h @@ -39,8 +39,7 @@ class DartIsolateGroupData : public PlatformMessageHandlerStorage { const ChildIsolatePreparer& child_isolate_preparer, const fml::closure& isolate_create_callback, const fml::closure& isolate_shutdown_callback, - std::shared_ptr native_assets_manager = nullptr, - fml::RefPtr base_snapshot = nullptr); + std::shared_ptr native_assets_manager = nullptr); ~DartIsolateGroupData(); @@ -48,10 +47,6 @@ class DartIsolateGroupData : public PlatformMessageHandlerStorage { fml::RefPtr GetIsolateSnapshot() const; - /// Returns the base snapshot for Shorebird linking support. - /// May be null if not using Shorebird or if no patch is active. - fml::RefPtr GetBaseSnapshot() const; - const std::string& GetAdvisoryScriptURI() const; const std::string& GetAdvisoryScriptEntrypoint() const; @@ -86,7 +81,6 @@ class DartIsolateGroupData : public PlatformMessageHandlerStorage { std::vector> kernel_buffers_; const Settings settings_; const fml::RefPtr isolate_snapshot_; - const fml::RefPtr base_snapshot_; const std::string advisory_script_uri_; const std::string advisory_script_entrypoint_; mutable std::mutex child_isolate_preparer_mutex_; diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.cc b/engine/src/flutter/runtime/shorebird/patch_cache.cc index 30c99aea3a011..d3dbe617ebfea 100644 --- a/engine/src/flutter/runtime/shorebird/patch_cache.cc +++ b/engine/src/flutter/runtime/shorebird/patch_cache.cc @@ -45,10 +45,9 @@ std::shared_ptr PatchCacheEntry::Create( const uint8_t* isolate_data = nullptr; const uint8_t* isolate_instrs = nullptr; - Dart_LoadedElf* elf = - Dart_LoadELF(path.c_str(), elf_file_offset, &error, &ignored_vm_data, - &ignored_vm_instrs, &isolate_data, &isolate_instrs, - /* load as read-only, not rx */ false); + Dart_LoadedElf* elf = Dart_LoadELF( + path.c_str(), elf_file_offset, &error, &ignored_vm_data, + &ignored_vm_instrs, &isolate_data, &isolate_instrs, dart::bin::kReadOnly); if (elf == nullptr) { FML_LOG(ERROR) << "Failed to load patch at " << path << " error: " << error; diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc index 8cb32cfc9916b..9a4136b48bbe6 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.cc +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -46,7 +46,6 @@ extern "C" __attribute__((weak)) unsigned long getauxval(unsigned long type) { #endif #if SHOREBIRD_USE_INTERPRETER - // Global references to the base (unpatched) snapshots from the App.framework. // These are process-global because: // 1. The Shorebird updater library is a process-global singleton with its own @@ -54,27 +53,23 @@ extern "C" __attribute__((weak)) unsigned long getauxval(unsigned long type) { // data for patch generation/validation. // 2. The base snapshots are immutable (baked into the IPA) so sharing them // across isolate groups is safe. -// 3. GetBaseIsolateSnapshot() returns the isolate snapshot for use when -// creating isolate groups with Dart_CreateIsolateGroupWithBaseSnapshot(), -// which needs the base snapshot for linking patched code. // // Note: This design doesn't support multiple engines with different base // snapshots, but I'm not aware of any use cases for that on iOS. static fml::RefPtr vm_snapshot; static fml::RefPtr isolate_snapshot; -void StoreBaseSnapshots(Settings& settings) { - // Create DartSnapshot objects that hold references to the symbol mappings - // in the App.framework. The snapshots are static data in the framework, - // but we need DartSnapshot objects to keep the NativeLibrary refs alive. +void SetBaseSnapshot(Settings& settings) { + // These mappings happen to be to static data in the App.framework, but + // we still need to seem to hold onto the DartSnapshot objects to keep + // the mappings alive. vm_snapshot = DartSnapshot::VMSnapshotFromSettings(settings); isolate_snapshot = DartSnapshot::IsolateSnapshotFromSettings(settings); + Shorebird_SetBaseSnapshots(isolate_snapshot->GetDataMapping(), + isolate_snapshot->GetInstructionsMapping(), + vm_snapshot->GetDataMapping(), + vm_snapshot->GetInstructionsMapping()); } - -fml::RefPtr GetBaseIsolateSnapshot() { - return isolate_snapshot; -} - #endif // SHOREBIRD_USE_INTERPRETER class FileCallbacksImpl { @@ -120,7 +115,7 @@ std::string GetValueFromYaml(const std::string& yaml, const std::string& key) { /// Newer api, used by Desktop implementations. /// Does not directly manipulate Settings. -// FIXME: Consolidate this with the other ConfigureShorebird() API. +// TODO(eseidel): Consolidate this with the other ConfigureShorebird() API. bool ConfigureShorebird(const ShorebirdConfigArgs& args, std::string& patch_path) { patch_path = fml::PathToUtf8(args.release_app_library_path); @@ -170,10 +165,9 @@ bool ConfigureShorebird(const ShorebirdConfigArgs& args, args.shorebird_yaml.c_str()); } - // We've decided not to support synchronous updates on launch for now. - // It's a terrible user experience (having the app hang on launch) and - // instead we will provide examples of how to build a custom update UI - // within Dart, including updating as part of login, etc. + // We do not support synchronous updates on launch, it's a terrible UX. + // Users can implement custom check-for-updates using + // package:shorebird_code_push. // https://github.com/shorebirdtech/shorebird/issues/950 FML_LOG(INFO) << "Checking for active patch"; @@ -218,7 +212,7 @@ bool ConfigureShorebird(const ShorebirdConfigArgs& args, } /// Older api used by iOS and Android, directly manipulates Settings. -// FIXME: Consolidate this with the other ConfigureShorebird() API. +// TODO(eseidel): Consolidate this with the other ConfigureShorebird() API. void ConfigureShorebird(std::string code_cache_path, std::string app_storage_path, Settings& settings, @@ -274,9 +268,9 @@ void ConfigureShorebird(std::string code_cache_path, // package:shorebird_code_push. // https://github.com/shorebirdtech/shorebird/issues/950 - // We store the base snapshot on iOS for use when creating the isolate group. + // We only set the base snapshot on iOS for now. #if SHOREBIRD_USE_INTERPRETER - StoreBaseSnapshots(settings); + SetBaseSnapshot(settings); #endif shorebird_validate_next_boot_patch(); @@ -334,7 +328,7 @@ void* FileCallbacksImpl::Open() { *isolate_snapshot) .release(); #else - // SnapshotsDataHandle exists on all platforms (for testing)but is only used + // SnapshotsDataHandle exists on all platforms (for testing) but is only used // on iOS. iOS patches are generated from just the Dart parts of the snapshot, // excluding the Mach-O specific headers which contain dates and paths that // make them change on every build. diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.h b/engine/src/flutter/shell/common/shorebird/shorebird.h index 50c6f59e1fa60..ab5c9162ea0f2 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.h +++ b/engine/src/flutter/shell/common/shorebird/shorebird.h @@ -54,13 +54,6 @@ void ConfigureShorebird(std::string code_cache_path, /// Exposed for testing. std::string GetValueFromYaml(const std::string& yaml, const std::string& key); -#if SHOREBIRD_USE_INTERPRETER -/// Returns the base isolate snapshot for Shorebird linking support. -/// Must be called after ConfigureShorebird() has stored the base snapshots. -/// May return null if not using Shorebird interpreter mode. -fml::RefPtr GetBaseIsolateSnapshot(); -#endif // SHOREBIRD_USE_INTERPRETER - } // namespace flutter #endif // FLUTTER_SHELL_COMMON_SHOREBIRD_SHOREBIRD_H_ From 8e810558bbb05bde39bfe4f58df10fe2b4068566 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Thu, 8 Jan 2026 16:15:45 -0800 Subject: [PATCH 15/51] chore: fix linux --- engine/src/flutter/shell/common/shorebird/shorebird.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc index 9a4136b48bbe6..81469a480f735 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.cc +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -118,7 +118,7 @@ std::string GetValueFromYaml(const std::string& yaml, const std::string& key) { // TODO(eseidel): Consolidate this with the other ConfigureShorebird() API. bool ConfigureShorebird(const ShorebirdConfigArgs& args, std::string& patch_path) { - patch_path = fml::PathToUtf8(args.release_app_library_path); + patch_path = args.release_app_library_path; auto shorebird_updater_dir_name = "shorebird_updater"; // Parse app id from shorebird.yaml From 95c8cae49758bc2a1b98e9f860348fce75d47c8b Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Sun, 11 Jan 2026 15:06:13 -0800 Subject: [PATCH 16/51] feat: add support for patch_verification_mode (#100) * feat: allow patch_verification_mode * test: update tests * chore: rename to patch_verification --- packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart | 1 + .../test/general.shard/shorebird/shorebird_yaml_test.dart | 3 +++ 2 files changed, 4 insertions(+) diff --git a/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart b/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart index 9e40f392b7eb8..a741b2683d765 100644 --- a/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart +++ b/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart @@ -56,6 +56,7 @@ Map compileShorebirdYaml(YamlMap yamlMap, {required String? fla } copyIfSet('base_url'); copyIfSet('auto_update'); + copyIfSet('patch_verification'); final String? shorebirdPublicKeyEnvVar = environment['SHOREBIRD_PUBLIC_KEY']; if (shorebirdPublicKeyEnvVar != null) { compiled['patch_public_key'] = shorebirdPublicKeyEnvVar; diff --git a/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart b/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart index 602de37681555..7e05fec171409 100644 --- a/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart +++ b/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart @@ -52,6 +52,7 @@ flavors: foo: 2-a bar: 3-a base_url: https://example.com +patch_verification: strict '''; final YamlDocument input = loadYamlDocument(yamlContents); final YamlMap yamlMap = input.contents as YamlMap; @@ -61,6 +62,7 @@ base_url: https://example.com 'app_id': '1-a', 'auto_update': false, 'base_url': 'https://example.com', + 'patch_verification': 'strict', }); final Map compiled2 = compileShorebirdYaml(yamlMap, flavor: 'foo', environment: {'SHOREBIRD_PUBLIC_KEY': '4-a'}); @@ -68,6 +70,7 @@ base_url: https://example.com 'app_id': '2-a', 'auto_update': false, 'base_url': 'https://example.com', + 'patch_verification': 'strict', 'patch_public_key': '4-a', }); }); From d0f19f58441c90977fc83d892cee8b2de175cac9 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Sun, 11 Jan 2026 15:07:00 -0800 Subject: [PATCH 17/51] chore: roll updater to 8691c8f60e69f8eb1f35361f4a22e8c9a7fdf93c to include patch_verification option --- DEPS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPS b/DEPS index d772d67e5c623..752fa14dfb943 100644 --- a/DEPS +++ b/DEPS @@ -19,7 +19,7 @@ vars = { "dart_sdk_revision": "b65ce89c8057d6880e00693a7b0ecd7b9e5f61ca", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", - "updater_rev": "9db198a6344f7b9ec2843e06a459938a520e314f", + "updater_rev": "8691c8f60e69f8eb1f35361f4a22e8c9a7fdf93c", # WARNING: DO NOT EDIT canvaskit_cipd_instance MANUALLY # See `lib/web_ui/README.md` for how to roll CanvasKit to a new version. From 63c703eceb83cd10f7972e666f457f7c360c7ea3 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Thu, 22 Jan 2026 22:20:15 -0800 Subject: [PATCH 18/51] fix: use of FlutterEngineGroup breaks patching (#101) * es/report_start_fix * fix: second callsite --- engine/src/flutter/runtime/shorebird/BUILD.gn | 5 +++ .../flutter/runtime/shorebird/patch_cache.cc | 14 +++++++ .../shell/common/shorebird/shorebird.cc | 39 +++++-------------- 3 files changed, 28 insertions(+), 30 deletions(-) diff --git a/engine/src/flutter/runtime/shorebird/BUILD.gn b/engine/src/flutter/runtime/shorebird/BUILD.gn index facee07ba5cd1..81bf3f588b3f8 100644 --- a/engine/src/flutter/runtime/shorebird/BUILD.gn +++ b/engine/src/flutter/runtime/shorebird/BUILD.gn @@ -12,4 +12,9 @@ source_set("patch_cache") { "//flutter/fml", "//flutter/runtime:libdart", ] + + # For shorebird_report_launch_start() in updater.h + # The include path "third_party/updater/library/include/updater.h" is relative + # to //flutter/, which is already in the default include path. + include_dirs = [ "//flutter" ] } diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.cc b/engine/src/flutter/runtime/shorebird/patch_cache.cc index d3dbe617ebfea..5610eb3393d60 100644 --- a/engine/src/flutter/runtime/shorebird/patch_cache.cc +++ b/engine/src/flutter/runtime/shorebird/patch_cache.cc @@ -4,9 +4,12 @@ #include "flutter/runtime/shorebird/patch_cache.h" +#include + #include "flutter/fml/logging.h" #include "flutter/fml/mapping.h" #include "flutter/runtime/shorebird/patch_mapping.h" +#include "third_party/updater/library/include/updater.h" namespace flutter { @@ -148,7 +151,18 @@ std::shared_ptr TryLoadFromPatch( FML_LOG(INFO) << "Loading symbol from patch: " << symbol_name; + // Report launch_start when we're actually about to use a patch. + // This is called at exactly the right time - right before the patched + // snapshot is loaded. We use std::once_flag to ensure it's only called + // once per process, and only for the first symbol (isolate data). + // This fixes the FlutterEngineGroup issue where report_launch_start was + // called too early (in ConfigureShorebird) before any patch was used. + static std::once_flag launch_start_flag; if (symbol == kIsolateDataSymbol) { + std::call_once(launch_start_flag, []() { + FML_LOG(INFO) << "Reporting launch start for patch"; + shorebird_report_launch_start(); + }); return PatchMapping::CreateIsolateData(cache_entry); } else { FML_CHECK(symbol == kIsolateInstructionsSymbol); diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc index 81469a480f735..d040e07e0285d 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.cc +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -181,25 +181,14 @@ bool ConfigureShorebird(const ShorebirdConfigArgs& args, FML_LOG(INFO) << "Shorebird updater: no active patch."; } - // We are careful only to report a launch start in the case where it's the - // first time we've configured shorebird this process. Otherwise we could end - // up in a case where we report a launch start, but never a completion (e.g. - // from package:flutter_work_manager which sometimes creates a FlutterEngine - // (and thus configures shorebird) but never runs it. The proper fix for this - // is probably to move the launch_start() call to be later in the lifecycle - // (when the snapshot is loaded and run, rather than when FlutterEngine is - // initialized). This "hack" will still have a problem where FlutterEngine is - // initialized but never run before the app is quit, could still cause us to - // suddenly mark-bad a patch that was never actually attempted to launch. + // Note: shorebird_report_launch_start() is now called from TryLoadFromPatch() + // in runtime/shorebird/patch_cache.cc, right before the patched snapshot is + // actually loaded. This fixes issues with FlutterEngineGroup and other cases + // where ConfigureShorebird() is called but no Shell is created. if (!init_result) { return false; } - // Once start_update_thread is called, the next_boot_patch* functions may - // change their return values if the shorebird_report_launch_failed - // function is called. - shorebird_report_launch_start(); - if (shorebird_should_auto_update()) { FML_LOG(INFO) << "Starting Shorebird update"; shorebird_start_update_thread(); @@ -294,25 +283,15 @@ void ConfigureShorebird(std::string code_cache_path, FML_LOG(INFO) << "Shorebird updater: no active patch."; } - // We are careful only to report a launch start in the case where it's the - // first time we've configured shorebird this process. Otherwise we could end - // up in a case where we report a launch start, but never a completion (e.g. - // from package:flutter_work_manager which sometimes creates a FlutterEngine - // (and thus configures shorebird) but never runs it. The proper fix for this - // is probably to move the launch_start() call to be later in the lifecycle - // (when the snapshot is loaded and run, rather than when FlutterEngine is - // initialized). This "hack" will still have a problem where FlutterEngine is - // initialized but never run before the app is quit, could still cause us to - // suddenly mark-bad a patch that was never actually attempted to launch. + // Note: shorebird_report_launch_start() is now called from TryLoadFromPatch() + // in runtime/shorebird/patch_cache.cc, right before the patched snapshot is + // actually loaded. This fixes issues with FlutterEngineGroup and other cases + // where ConfigureShorebird() is called but no Shell is created. + if (!init_result) { return; } - // Once start_update_thread is called, the next_boot_patch* functions may - // change their return values if the shorebird_report_launch_failed - // function is called. - shorebird_report_launch_start(); - if (shorebird_should_auto_update()) { FML_LOG(INFO) << "Starting Shorebird update"; shorebird_start_update_thread(); From a84cba5c5f0c7ff8a90860c2a0386720c0a93aa1 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 23 Jan 2026 15:30:24 -0800 Subject: [PATCH 19/51] chore: add a C++ interface onto the updater (#102) * chore: add a C++ interface onto the updater * chore: centralize SHOREBIRD_PLATFORM_SUPPORTED * test: fix tests --- engine/src/flutter/runtime/shorebird/BUILD.gn | 6 +- .../flutter/runtime/shorebird/patch_cache.cc | 5 +- engine/src/flutter/shell/common/BUILD.gn | 29 +-- engine/src/flutter/shell/common/shell.cc | 11 +- .../flutter/shell/common/shell_unittests.cc | 62 ++++++ .../flutter/shell/common/shorebird/BUILD.gn | 57 +++++- .../shell/common/shorebird/shorebird.cc | 103 ++++------ .../flutter/shell/common/shorebird/updater.cc | 166 +++++++++++++++ .../flutter/shell/common/shorebird/updater.h | 191 ++++++++++++++++++ .../common/shorebird/updater_unittests.cc | 127 ++++++++++++ .../flutter/shell/platform/android/BUILD.gn | 13 -- .../shell/platform/android/flutter_main.cc | 2 - .../shell/platform/darwin/ios/BUILD.gn | 8 - 13 files changed, 647 insertions(+), 133 deletions(-) create mode 100644 engine/src/flutter/shell/common/shorebird/updater.cc create mode 100644 engine/src/flutter/shell/common/shorebird/updater.h create mode 100644 engine/src/flutter/shell/common/shorebird/updater_unittests.cc diff --git a/engine/src/flutter/runtime/shorebird/BUILD.gn b/engine/src/flutter/runtime/shorebird/BUILD.gn index 81bf3f588b3f8..1f856e36d3f48 100644 --- a/engine/src/flutter/runtime/shorebird/BUILD.gn +++ b/engine/src/flutter/runtime/shorebird/BUILD.gn @@ -11,10 +11,6 @@ source_set("patch_cache") { deps = [ "//flutter/fml", "//flutter/runtime:libdart", + "//flutter/shell/common/shorebird:updater", ] - - # For shorebird_report_launch_start() in updater.h - # The include path "third_party/updater/library/include/updater.h" is relative - # to //flutter/, which is already in the default include path. - include_dirs = [ "//flutter" ] } diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.cc b/engine/src/flutter/runtime/shorebird/patch_cache.cc index 5610eb3393d60..b835d01b98639 100644 --- a/engine/src/flutter/runtime/shorebird/patch_cache.cc +++ b/engine/src/flutter/runtime/shorebird/patch_cache.cc @@ -9,7 +9,8 @@ #include "flutter/fml/logging.h" #include "flutter/fml/mapping.h" #include "flutter/runtime/shorebird/patch_mapping.h" -#include "third_party/updater/library/include/updater.h" +#include "flutter/shell/common/shorebird/updater.h" +#include "third_party/dart/runtime/include/dart_api.h" namespace flutter { @@ -161,7 +162,7 @@ std::shared_ptr TryLoadFromPatch( if (symbol == kIsolateDataSymbol) { std::call_once(launch_start_flag, []() { FML_LOG(INFO) << "Reporting launch start for patch"; - shorebird_report_launch_start(); + shorebird::Updater::Instance().ReportLaunchStart(); }); return PatchMapping::CreateIsolateData(cache_entry); } else { diff --git a/engine/src/flutter/shell/common/BUILD.gn b/engine/src/flutter/shell/common/BUILD.gn index bbe42f5a987f8..5eb7b979fe2f0 100644 --- a/engine/src/flutter/shell/common/BUILD.gn +++ b/engine/src/flutter/shell/common/BUILD.gn @@ -153,13 +153,12 @@ source_set("common") { "//flutter/lib/ui", "//flutter/runtime", "//flutter/shell/common:base64", + "//flutter/shell/common/shorebird:updater", "//flutter/shell/geometry", "//flutter/shell/profiling", "//flutter/skia", ] - include_dirs = [ "//flutter/updater" ] - if (impeller_supports_rendering) { sources += [ "snapshot_controller_impeller.cc", @@ -168,31 +167,6 @@ source_set("common") { deps += [ "//flutter/impeller" ] } - - # Needed to compile flutter_tester for macOS. - if (host_os == "mac" && target_os == "mac") { - if (target_cpu == "arm64") { - libs = [ "//flutter/third_party/updater/target/aarch64-apple-darwin/release/libupdater.a" ] - } else if (target_cpu == "x64") { - libs = [ "//flutter/third_party/updater/target/x86_64-apple-darwin/release/libupdater.a" ] - } - } - - # Needed to compile flutter_tester for Windows. - if (host_os == "win" && target_os == "win") { - if (target_cpu == "x64") { - libs = [ - "userenv.lib", - "//flutter/third_party/updater/target/x86_64-pc-windows-msvc/release/updater.lib", - ] - } - } - - if (host_os == "linux" && target_os == "linux") { - if (target_cpu == "x64") { - libs = [ "//flutter/third_party/updater/target/x86_64-unknown-linux-gnu/release/libupdater.a" ] - } - } } # These are in their own source_set to avoid a dependency cycle with //common/graphics @@ -369,6 +343,7 @@ if (enable_unittests) { "//flutter/common/graphics", "//flutter/display_list/testing:display_list_testing", "//flutter/shell/common:base64", + "//flutter/shell/common/shorebird:updater", "//flutter/shell/profiling:profiling_unittests", "//flutter/shell/version", "//flutter/testing:fixture_test", diff --git a/engine/src/flutter/shell/common/shell.cc b/engine/src/flutter/shell/common/shell.cc index fac4ec6336fbe..c424f82151e36 100644 --- a/engine/src/flutter/shell/common/shell.cc +++ b/engine/src/flutter/shell/common/shell.cc @@ -47,7 +47,7 @@ #include "third_party/skia/include/core/SkGraphics.h" #include "third_party/tonic/common/log.h" -#include "third_party/updater/library/include/updater.h" +#include "flutter/shell/common/shorebird/updater.h" namespace flutter { @@ -524,14 +524,13 @@ Shell::Shell(DartVMRef vm, is_gpu_disabled_sync_switch_(new fml::SyncSwitch(is_gpu_disabled)), weak_factory_gpu_(nullptr), weak_factory_(this) { - // FIXME: This is probably the wrong place to hook into. -#if SHOREBIRD_PLATFORM_SUPPORTED + // Report launch status to Shorebird updater for crash recovery tracking. + // On unsupported platforms, NoOpUpdater handles these calls gracefully. if (!vm_) { - shorebird_report_launch_failure(); + shorebird::Updater::Instance().ReportLaunchFailure(); } else { - shorebird_report_launch_success(); + shorebird::Updater::Instance().ReportLaunchSuccess(); } -#endif FML_CHECK(!settings.enable_software_rendering || !settings.enable_impeller) << "Software rendering is incompatible with Impeller."; if (!settings.enable_impeller && settings.warn_on_impeller_opt_out) { diff --git a/engine/src/flutter/shell/common/shell_unittests.cc b/engine/src/flutter/shell/common/shell_unittests.cc index 2ce1a9040d5a4..8a57f10a63341 100644 --- a/engine/src/flutter/shell/common/shell_unittests.cc +++ b/engine/src/flutter/shell/common/shell_unittests.cc @@ -52,6 +52,8 @@ #include "third_party/skia/include/codec/SkCodecAnimation.h" #include "third_party/tonic/converter/dart_converter.h" +#include "flutter/shell/common/shorebird/updater.h" + #ifdef SHELL_ENABLE_VULKAN #include "flutter/vulkan/vulkan_application.h" // nogncheck #endif @@ -5108,6 +5110,66 @@ TEST_F(ShellTest, ShoulDiscardLayerTreeIfFrameIsSizedIncorrectly) { DestroyShell(std::move(shell), task_runners); } +// Test that Shell creation triggers the Shorebird Updater's ReportLaunchSuccess +// call. This is important for the crash recovery mechanism to work correctly. +TEST_F(ShellTest, ShorebirdUpdaterReportLaunchSuccessOnShellCreation) { + // Install a mock updater before creating the shell + auto mock = std::make_unique(); + auto* mock_ptr = mock.get(); + shorebird::Updater::SetInstanceForTesting(std::move(mock)); + + EXPECT_EQ(mock_ptr->launch_success_count(), 0); + EXPECT_EQ(mock_ptr->launch_failure_count(), 0); + + auto settings = CreateSettingsForFixture(); + auto task_runners = GetTaskRunnersForFixture(); + auto shell = CreateShell(settings, task_runners); + ASSERT_TRUE(shell); + + // Shell constructor should have called ReportLaunchSuccess + EXPECT_EQ(mock_ptr->launch_success_count(), 1); + EXPECT_EQ(mock_ptr->launch_failure_count(), 0); + + // Verify the call was logged + const auto& log = mock_ptr->call_log(); + EXPECT_TRUE(std::find(log.begin(), log.end(), "ReportLaunchSuccess") != + log.end()); + + DestroyShell(std::move(shell), task_runners); + + // Clean up - reset the updater instance + shorebird::Updater::ResetInstanceForTesting(); +} + +// Test that creating multiple shells only calls ReportLaunchSuccess for each +// shell. This verifies that each Shell reports its own launch status. +TEST_F(ShellTest, ShorebirdUpdaterReportLaunchSuccessForMultipleShells) { + auto mock = std::make_unique(); + auto* mock_ptr = mock.get(); + shorebird::Updater::SetInstanceForTesting(std::move(mock)); + + EXPECT_EQ(mock_ptr->launch_success_count(), 0); + + auto settings = CreateSettingsForFixture(); + + // Create first shell + auto task_runners1 = GetTaskRunnersForFixture(); + auto shell1 = CreateShell(settings, task_runners1); + ASSERT_TRUE(shell1); + EXPECT_EQ(mock_ptr->launch_success_count(), 1); + + // Create second shell + auto task_runners2 = GetTaskRunnersForFixture(); + auto shell2 = CreateShell(settings, task_runners2); + ASSERT_TRUE(shell2); + EXPECT_EQ(mock_ptr->launch_success_count(), 2); + + DestroyShell(std::move(shell1), task_runners1); + DestroyShell(std::move(shell2), task_runners2); + + shorebird::Updater::ResetInstanceForTesting(); +} + } // namespace testing } // namespace flutter diff --git a/engine/src/flutter/shell/common/shorebird/BUILD.gn b/engine/src/flutter/shell/common/shorebird/BUILD.gn index dc2fc5515bd7c..2b1e29c03512d 100644 --- a/engine/src/flutter/shell/common/shorebird/BUILD.gn +++ b/engine/src/flutter/shell/common/shorebird/BUILD.gn @@ -15,6 +15,56 @@ source_set("snapshots_data_handle") { ] } +# C++ wrapper around the Rust updater C API. +# This provides a testable abstraction layer that can be mocked for testing. +source_set("updater") { + sources = [ + "updater.cc", + "updater.h", + ] + + deps = [ "//flutter/fml" ] + + # For the Rust updater C API (shorebird_report_launch_start, etc.) + include_dirs = [ "//flutter" ] + + # Link the Rust updater static library based on target platform. + if (is_android) { + if (target_cpu == "arm") { + libs = [ "//flutter/third_party/updater/target/armv7-linux-androideabi/release/libupdater.a" ] + } else if (target_cpu == "arm64") { + libs = [ "//flutter/third_party/updater/target/aarch64-linux-android/release/libupdater.a" ] + } else if (target_cpu == "x64") { + libs = [ "//flutter/third_party/updater/target/x86_64-linux-android/release/libupdater.a" ] + } else if (target_cpu == "x86") { + libs = [ "//flutter/third_party/updater/target/i686-linux-android/release/libupdater.a" ] + } + } else if (is_ios) { + if (target_cpu == "arm64") { + libs = [ "//flutter/third_party/updater/target/aarch64-apple-ios/release/libupdater.a" ] + } else if (target_cpu == "x64") { + libs = [ "//flutter/third_party/updater/target/x86_64-apple-ios/release/libupdater.a" ] + } + } else if (is_mac) { + if (target_cpu == "arm64") { + libs = [ "//flutter/third_party/updater/target/aarch64-apple-darwin/release/libupdater.a" ] + } else if (target_cpu == "x64") { + libs = [ "//flutter/third_party/updater/target/x86_64-apple-darwin/release/libupdater.a" ] + } + } else if (is_win) { + if (target_cpu == "x64") { + libs = [ + "userenv.lib", + "//flutter/third_party/updater/target/x86_64-pc-windows-msvc/release/updater.lib", + ] + } + } else if (is_linux) { + if (target_cpu == "x64") { + libs = [ "//flutter/third_party/updater/target/x86_64-unknown-linux-gnu/release/libupdater.a" ] + } + } +} + source_set("shorebird") { sources = [ "shorebird.cc", @@ -23,14 +73,13 @@ source_set("shorebird") { deps = [ ":snapshots_data_handle", + ":updater", "//flutter/fml", "//flutter/runtime", "//flutter/runtime:libdart", "//flutter/shell/common", "//flutter/shell/platform/embedder:embedder_headers", ] - - include_dirs = [ "//flutter/updater" ] } if (enable_unittests) { @@ -45,14 +94,14 @@ if (enable_unittests) { "patch_cache_unittests.cc", "shorebird_unittests.cc", "snapshots_data_handle_unittests.cc", + "updater_unittests.cc", ] - # This only includes snapshots_data_handle and not shorebird because - # shorebird fails to link due to a missing updater lib. deps = [ ":shorebird", ":shorebird_fixtures", ":snapshots_data_handle", + ":updater", "//flutter/runtime", "//flutter/runtime/shorebird:patch_cache", "//flutter/testing", diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc index d040e07e0285d..71d336e90f8b0 100644 --- a/engine/src/flutter/shell/common/shorebird/shorebird.cc +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -19,13 +19,12 @@ #include "flutter/runtime/dart_vm.h" #include "flutter/shell/common/shell.h" #include "flutter/shell/common/shorebird/snapshots_data_handle.h" +#include "flutter/shell/common/shorebird/updater.h" #include "flutter/shell/common/switches.h" #include "fml/logging.h" #include "shell/platform/embedder/embedder.h" #include "third_party/dart/runtime/include/dart_tools_api.h" -#include "third_party/updater/library/include/updater.h" - // Namespaced to avoid Google style warnings. namespace flutter { @@ -80,7 +79,7 @@ class FileCallbacksImpl { static void Close(void* file); }; -FileCallbacks ShorebirdFileCallbacks() { +shorebird::FileCallbacks ShorebirdFileCallbacks() { return { .open = FileCallbacksImpl::Open, .read = FileCallbacksImpl::Read, @@ -137,33 +136,22 @@ bool ConfigureShorebird(const ShorebirdConfigArgs& args, {shorebird_updater_dir_name}, fml::FilePermission::kReadWrite); - bool init_result; - // Using a block to make AppParameters lifetime explicit. - { - AppParameters app_parameters; - // Combine version and version_code into a single string. - // We could also pass these separately through to the updater if needed. - auto release_version = args.release_version.version; - if (!args.release_version.build_number.empty()) { - release_version += "+" + args.release_version.build_number; - } - - app_parameters.release_version = release_version.c_str(); - app_parameters.code_cache_dir = code_cache_dir.c_str(); - app_parameters.app_storage_dir = app_storage_dir.c_str(); + // Combine version and version_code into a single string. + // We could also pass these separately through to the updater if needed. + auto release_version = args.release_version.version; + if (!args.release_version.build_number.empty()) { + release_version += "+" + args.release_version.build_number; + } - // https://stackoverflow.com/questions/26032039/convert-vectorstring-into-char-c - std::vector c_paths{}; - c_paths.push_back(args.release_app_library_path.c_str()); - // Do not modify application_library_paths or c_strings will invalidate. + shorebird::AppConfig config; + config.release_version = release_version; + config.original_libapp_paths = {args.release_app_library_path}; + config.app_storage_dir = app_storage_dir; + config.code_cache_dir = code_cache_dir; + config.file_callbacks = ShorebirdFileCallbacks(); + config.yaml_config = args.shorebird_yaml; - app_parameters.original_libapp_paths = c_paths.data(); - app_parameters.original_libapp_paths_size = c_paths.size(); - - // shorebird_init copies from app_parameters and shorebirdYaml. - init_result = shorebird_init(&app_parameters, ShorebirdFileCallbacks(), - args.shorebird_yaml.c_str()); - } + bool init_result = shorebird::Updater::Instance().Init(config); // We do not support synchronous updates on launch, it's a terrible UX. // Users can implement custom check-for-updates using @@ -171,11 +159,10 @@ bool ConfigureShorebird(const ShorebirdConfigArgs& args, // https://github.com/shorebirdtech/shorebird/issues/950 FML_LOG(INFO) << "Checking for active patch"; - shorebird_validate_next_boot_patch(); - char* c_active_path = shorebird_next_boot_patch_path(); - if (c_active_path != NULL) { - patch_path = c_active_path; - shorebird_free_string(c_active_path); + shorebird::Updater::Instance().ValidateNextBootPatch(); + std::string active_path = shorebird::Updater::Instance().NextBootPatchPath(); + if (!active_path.empty()) { + patch_path = active_path; FML_LOG(INFO) << "Shorebird updater: patch path: " << patch_path; } else { FML_LOG(INFO) << "Shorebird updater: no active patch."; @@ -189,9 +176,9 @@ bool ConfigureShorebird(const ShorebirdConfigArgs& args, return false; } - if (shorebird_should_auto_update()) { + if (shorebird::Updater::Instance().ShouldAutoUpdate()) { FML_LOG(INFO) << "Starting Shorebird update"; - shorebird_start_update_thread(); + shorebird::Updater::Instance().StartUpdateThread(); } else { FML_LOG(INFO) << "Shorebird auto_update disabled, not checking for updates."; @@ -226,31 +213,17 @@ void ConfigureShorebird(std::string code_cache_path, {shorebird_updater_dir_name}, fml::FilePermission::kReadWrite); - bool init_result; - // Using a block to make AppParameters lifetime explicit. - { - AppParameters app_parameters; - // Combine version and version_code into a single string. - // We could also pass these separately through to the updater if needed. - auto release_version = version + "+" + version_code; - app_parameters.release_version = release_version.c_str(); - app_parameters.code_cache_dir = code_cache_dir.c_str(); - app_parameters.app_storage_dir = app_storage_dir.c_str(); - - // https://stackoverflow.com/questions/26032039/convert-vectorstring-into-char-c - std::vector c_paths{}; - for (const auto& string : settings.application_library_paths) { - c_paths.push_back(string.c_str()); - } - // Do not modify application_library_paths or c_strings will invalidate. - - app_parameters.original_libapp_paths = c_paths.data(); - app_parameters.original_libapp_paths_size = c_paths.size(); + // Combine version and version_code into a single string. + // We could also pass these separately through to the updater if needed. + shorebird::AppConfig config; + config.release_version = version + "+" + version_code; + config.original_libapp_paths = settings.application_library_paths; + config.app_storage_dir = app_storage_dir; + config.code_cache_dir = code_cache_dir; + config.file_callbacks = ShorebirdFileCallbacks(); + config.yaml_config = shorebird_yaml; - // shorebird_init copies from app_parameters and shorebirdYaml. - init_result = shorebird_init(&app_parameters, ShorebirdFileCallbacks(), - shorebird_yaml.c_str()); - } + bool init_result = shorebird::Updater::Instance().Init(config); // We do not support synchronous updates on launch, it's a terrible UX. // Users can implement custom check-for-updates using @@ -262,11 +235,9 @@ void ConfigureShorebird(std::string code_cache_path, SetBaseSnapshot(settings); #endif - shorebird_validate_next_boot_patch(); - char* c_active_path = shorebird_next_boot_patch_path(); - if (c_active_path != NULL) { - std::string active_path = c_active_path; - shorebird_free_string(c_active_path); + shorebird::Updater::Instance().ValidateNextBootPatch(); + std::string active_path = shorebird::Updater::Instance().NextBootPatchPath(); + if (!active_path.empty()) { FML_LOG(INFO) << "Shorebird updater: active path: " << active_path; #if SHOREBIRD_USE_INTERPRETER @@ -292,9 +263,9 @@ void ConfigureShorebird(std::string code_cache_path, return; } - if (shorebird_should_auto_update()) { + if (shorebird::Updater::Instance().ShouldAutoUpdate()) { FML_LOG(INFO) << "Starting Shorebird update"; - shorebird_start_update_thread(); + shorebird::Updater::Instance().StartUpdateThread(); } else { FML_LOG(INFO) << "Shorebird auto_update disabled, not checking for updates."; diff --git a/engine/src/flutter/shell/common/shorebird/updater.cc b/engine/src/flutter/shell/common/shorebird/updater.cc new file mode 100644 index 0000000000000..84de941ec2d40 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/updater.cc @@ -0,0 +1,166 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/common/shorebird/updater.h" + +#include "flutter/fml/logging.h" + +#if SHOREBIRD_PLATFORM_SUPPORTED +#include "third_party/updater/library/include/updater.h" +#endif + +namespace flutter { +namespace shorebird { + +// Static member definitions +std::unique_ptr Updater::instance_; +std::mutex Updater::instance_mutex_; + +Updater& Updater::Instance() { + std::lock_guard lock(instance_mutex_); + if (!instance_) { +#if SHOREBIRD_PLATFORM_SUPPORTED + instance_ = std::make_unique(); +#else + instance_ = std::make_unique(); +#endif + } + return *instance_; +} + +void Updater::SetInstanceForTesting(std::unique_ptr instance) { + std::lock_guard lock(instance_mutex_); + instance_ = std::move(instance); +} + +void Updater::ResetInstanceForTesting() { + std::lock_guard lock(instance_mutex_); + instance_.reset(); +} + +#if SHOREBIRD_PLATFORM_SUPPORTED +// RealUpdater implementation - wraps the Rust C API + +bool RealUpdater::Init(const AppConfig& config) { + // Convert paths to C strings + std::vector c_paths; + c_paths.reserve(config.original_libapp_paths.size()); + for (const auto& path : config.original_libapp_paths) { + c_paths.push_back(path.c_str()); + } + + AppParameters params; + params.release_version = config.release_version.c_str(); + params.original_libapp_paths = c_paths.data(); + params.original_libapp_paths_size = static_cast(c_paths.size()); + params.app_storage_dir = config.app_storage_dir.c_str(); + params.code_cache_dir = config.code_cache_dir.c_str(); + + // Convert our FileCallbacks to the Rust struct + ::FileCallbacks rust_callbacks; + rust_callbacks.open = config.file_callbacks.open; + rust_callbacks.read = config.file_callbacks.read; + rust_callbacks.seek = config.file_callbacks.seek; + rust_callbacks.close = config.file_callbacks.close; + + return shorebird_init(¶ms, rust_callbacks, config.yaml_config.c_str()); +} + +void RealUpdater::ValidateNextBootPatch() { + shorebird_validate_next_boot_patch(); +} + +std::string RealUpdater::NextBootPatchPath() { + char* c_path = shorebird_next_boot_patch_path(); + if (c_path == nullptr) { + return ""; + } + std::string path(c_path); + shorebird_free_string(c_path); + return path; +} + +void RealUpdater::ReportLaunchStart() { + shorebird_report_launch_start(); +} + +void RealUpdater::ReportLaunchSuccess() { + shorebird_report_launch_success(); +} + +void RealUpdater::ReportLaunchFailure() { + shorebird_report_launch_failure(); +} + +bool RealUpdater::ShouldAutoUpdate() { + return shorebird_should_auto_update(); +} + +void RealUpdater::StartUpdateThread() { + shorebird_start_update_thread(); +} +#endif // SHOREBIRD_PLATFORM_SUPPORTED + +// MockUpdater implementation - for testing + +bool MockUpdater::Init(const AppConfig& config) { + init_count_++; + last_release_version_ = config.release_version; + last_yaml_config_ = config.yaml_config; + call_log_.push_back("Init"); + return init_result_; +} + +void MockUpdater::ValidateNextBootPatch() { + validate_count_++; + call_log_.push_back("ValidateNextBootPatch"); +} + +std::string MockUpdater::NextBootPatchPath() { + call_log_.push_back("NextBootPatchPath"); + return next_boot_patch_path_; +} + +void MockUpdater::ReportLaunchStart() { + launch_start_count_++; + call_log_.push_back("ReportLaunchStart"); +} + +void MockUpdater::ReportLaunchSuccess() { + launch_success_count_++; + call_log_.push_back("ReportLaunchSuccess"); +} + +void MockUpdater::ReportLaunchFailure() { + launch_failure_count_++; + call_log_.push_back("ReportLaunchFailure"); +} + +bool MockUpdater::ShouldAutoUpdate() { + call_log_.push_back("ShouldAutoUpdate"); + return should_auto_update_; +} + +void MockUpdater::StartUpdateThread() { + start_update_thread_count_++; + call_log_.push_back("StartUpdateThread"); +} + +void MockUpdater::Reset() { + init_count_ = 0; + validate_count_ = 0; + launch_start_count_ = 0; + launch_success_count_ = 0; + launch_failure_count_ = 0; + start_update_thread_count_ = 0; + init_result_ = true; + should_auto_update_ = false; + next_boot_patch_path_.clear(); + last_release_version_.clear(); + last_yaml_config_.clear(); + call_log_.clear(); +} + +} // namespace shorebird +} // namespace flutter diff --git a/engine/src/flutter/shell/common/shorebird/updater.h b/engine/src/flutter/shell/common/shorebird/updater.h new file mode 100644 index 0000000000000..f6fa5a9a2f6d9 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/updater.h @@ -0,0 +1,191 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_COMMON_SHOREBIRD_UPDATER_H_ +#define FLUTTER_SHELL_COMMON_SHOREBIRD_UPDATER_H_ + +#include +#include +#include +#include +#include +#include + +namespace flutter { +namespace shorebird { + +/// File callbacks for iOS patch loading. +/// Mirrors the FileCallbacks struct from the Rust updater. +struct FileCallbacks { + void* (*open)(void); + uintptr_t (*read)(void* file_handle, uint8_t* buffer, uintptr_t count); + int64_t (*seek)(void* file_handle, int64_t offset, int32_t whence); + void (*close)(void* file_handle); +}; + +/// Configuration for initializing the Shorebird updater. +struct AppConfig { + /// Version string for this release (e.g., "1.0.0+1"). + std::string release_version; + + /// Paths to the original AOT libraries (libapp.so on Android, App.framework + /// on iOS). + std::vector original_libapp_paths; + + /// Directory for persistent updater state (survives app updates). + std::string app_storage_dir; + + /// Directory for cached artifacts (cleared on app updates). + std::string code_cache_dir; + + /// Callbacks for iOS patch file access (can be null callbacks on Android). + FileCallbacks file_callbacks; + + /// YAML configuration from shorebird.yaml. + std::string yaml_config; +}; + +/// Abstract interface for the Shorebird updater. +/// +/// This abstraction allows for: +/// 1. Mocking in tests without requiring the real Rust library +/// 2. Future migration from Rust to C++ implementation +/// 3. Test instrumentation (call counting, logging) +class Updater { + public: + virtual ~Updater() = default; + + /// Initialize the updater with configuration. + /// @param config Configuration containing release version, paths, and + /// callbacks + /// @return true if initialization succeeded + virtual bool Init(const AppConfig& config) = 0; + + /// Validate the next boot patch. If invalid, falls back to last good state. + virtual void ValidateNextBootPatch() = 0; + + /// Get the path to the patch that will boot on next run. + /// @return Path to patch, or empty string if no patch available + virtual std::string NextBootPatchPath() = 0; + + // Boot lifecycle methods + virtual void ReportLaunchStart() = 0; + virtual void ReportLaunchSuccess() = 0; + virtual void ReportLaunchFailure() = 0; + + // Update checking + virtual bool ShouldAutoUpdate() = 0; + virtual void StartUpdateThread() = 0; + + // Singleton access + static Updater& Instance(); + + // Test support - allows injecting a mock implementation + static void SetInstanceForTesting(std::unique_ptr instance); + static void ResetInstanceForTesting(); + + protected: + Updater() = default; + + private: + static std::unique_ptr instance_; + static std::mutex instance_mutex_; +}; + +/// No-op implementation for unsupported platforms. +/// All methods are safe to call but do nothing. +class NoOpUpdater : public Updater { + public: + NoOpUpdater() = default; + ~NoOpUpdater() override = default; + + bool Init(const AppConfig& config) override { return true; } + void ValidateNextBootPatch() override {} + std::string NextBootPatchPath() override { return ""; } + void ReportLaunchStart() override {} + void ReportLaunchSuccess() override {} + void ReportLaunchFailure() override {} + bool ShouldAutoUpdate() override { return false; } + void StartUpdateThread() override {} +}; + +#if SHOREBIRD_PLATFORM_SUPPORTED +/// Production implementation that wraps the Rust updater C API. +/// Only available on supported platforms (Android, iOS, macOS, Windows, Linux). +class RealUpdater : public Updater { + public: + RealUpdater() = default; + ~RealUpdater() override = default; + + bool Init(const AppConfig& config) override; + void ValidateNextBootPatch() override; + std::string NextBootPatchPath() override; + void ReportLaunchStart() override; + void ReportLaunchSuccess() override; + void ReportLaunchFailure() override; + bool ShouldAutoUpdate() override; + void StartUpdateThread() override; +}; +#endif // SHOREBIRD_PLATFORM_SUPPORTED + +/// Mock implementation for testing. +/// Tracks call counts and can be queried to verify behavior. +class MockUpdater : public Updater { + public: + MockUpdater() = default; + ~MockUpdater() override = default; + + bool Init(const AppConfig& config) override; + void ValidateNextBootPatch() override; + std::string NextBootPatchPath() override; + void ReportLaunchStart() override; + void ReportLaunchSuccess() override; + void ReportLaunchFailure() override; + bool ShouldAutoUpdate() override; + void StartUpdateThread() override; + + // Test accessors + int init_count() const { return init_count_; } + int validate_count() const { return validate_count_; } + int launch_start_count() const { return launch_start_count_; } + int launch_success_count() const { return launch_success_count_; } + int launch_failure_count() const { return launch_failure_count_; } + int start_update_thread_count() const { return start_update_thread_count_; } + const std::vector& call_log() const { return call_log_; } + + // Last init parameters (for verification) + const std::string& last_release_version() const { + return last_release_version_; + } + const std::string& last_yaml_config() const { return last_yaml_config_; } + + // Test configuration + void set_init_result(bool value) { init_result_ = value; } + void set_should_auto_update(bool value) { should_auto_update_ = value; } + void set_next_boot_patch_path(const std::string& path) { + next_boot_patch_path_ = path; + } + + // Reset all counters and logs + void Reset(); + + private: + int init_count_ = 0; + int validate_count_ = 0; + int launch_start_count_ = 0; + int launch_success_count_ = 0; + int launch_failure_count_ = 0; + int start_update_thread_count_ = 0; + bool init_result_ = true; + bool should_auto_update_ = false; + std::string next_boot_patch_path_; + std::string last_release_version_; + std::string last_yaml_config_; + std::vector call_log_; +}; + +} // namespace shorebird +} // namespace flutter + +#endif // FLUTTER_SHELL_COMMON_SHOREBIRD_UPDATER_H_ diff --git a/engine/src/flutter/shell/common/shorebird/updater_unittests.cc b/engine/src/flutter/shell/common/shorebird/updater_unittests.cc new file mode 100644 index 0000000000000..97fb6ef2129f2 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/updater_unittests.cc @@ -0,0 +1,127 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/common/shorebird/updater.h" + +#include + +#include "gtest/gtest.h" + +namespace flutter { +namespace shorebird { +namespace testing { + +class UpdaterTest : public ::testing::Test { + protected: + void SetUp() override { + // Install a mock for each test + auto mock = std::make_unique(); + mock_ = mock.get(); + Updater::SetInstanceForTesting(std::move(mock)); + } + + void TearDown() override { + mock_ = nullptr; + Updater::ResetInstanceForTesting(); + } + + MockUpdater* mock_ = nullptr; +}; + +TEST_F(UpdaterTest, MockUpdaterTracksLaunchStartCalls) { + EXPECT_EQ(mock_->launch_start_count(), 0); + + Updater::Instance().ReportLaunchStart(); + EXPECT_EQ(mock_->launch_start_count(), 1); + + Updater::Instance().ReportLaunchStart(); + EXPECT_EQ(mock_->launch_start_count(), 2); +} + +TEST_F(UpdaterTest, MockUpdaterTracksLaunchSuccessCalls) { + EXPECT_EQ(mock_->launch_success_count(), 0); + + Updater::Instance().ReportLaunchSuccess(); + EXPECT_EQ(mock_->launch_success_count(), 1); +} + +TEST_F(UpdaterTest, MockUpdaterTracksLaunchFailureCalls) { + EXPECT_EQ(mock_->launch_failure_count(), 0); + + Updater::Instance().ReportLaunchFailure(); + EXPECT_EQ(mock_->launch_failure_count(), 1); +} + +TEST_F(UpdaterTest, MockUpdaterTracksShouldAutoUpdate) { + mock_->set_should_auto_update(false); + EXPECT_FALSE(Updater::Instance().ShouldAutoUpdate()); + + mock_->set_should_auto_update(true); + EXPECT_TRUE(Updater::Instance().ShouldAutoUpdate()); +} + +TEST_F(UpdaterTest, MockUpdaterTracksStartUpdateThreadCalls) { + EXPECT_EQ(mock_->start_update_thread_count(), 0); + + Updater::Instance().StartUpdateThread(); + EXPECT_EQ(mock_->start_update_thread_count(), 1); +} + +TEST_F(UpdaterTest, MockUpdaterCallLogRecordsSequence) { + EXPECT_TRUE(mock_->call_log().empty()); + + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ShouldAutoUpdate(); + Updater::Instance().ReportLaunchSuccess(); + + const auto& log = mock_->call_log(); + ASSERT_EQ(log.size(), 3u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ShouldAutoUpdate"); + EXPECT_EQ(log[2], "ReportLaunchSuccess"); +} + +TEST_F(UpdaterTest, MockUpdaterResetClearsState) { + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + mock_->set_should_auto_update(true); + + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_success_count(), 1); + EXPECT_TRUE(mock_->ShouldAutoUpdate()); + + mock_->Reset(); + + EXPECT_EQ(mock_->launch_start_count(), 0); + EXPECT_EQ(mock_->launch_success_count(), 0); + // Check call_log before ShouldAutoUpdate() since the method adds to call_log + EXPECT_TRUE(mock_->call_log().empty()); + EXPECT_FALSE(mock_->ShouldAutoUpdate()); +} + +// Test that demonstrates the std::once_flag pattern works correctly. +// This is the same pattern used in TryLoadFromPatch. +TEST_F(UpdaterTest, OncePerProcessPatternOnlyCallsOnce) { + static std::once_flag test_flag; + int call_count = 0; + + auto simulate_patch_load = [&]() { + std::call_once(test_flag, [&]() { + call_count++; + Updater::Instance().ReportLaunchStart(); + }); + }; + + // Simulate multiple engines loading patches + simulate_patch_load(); // Engine 1 + simulate_patch_load(); // Engine 2 + simulate_patch_load(); // Engine 3 + + EXPECT_EQ(call_count, 1); + EXPECT_EQ(mock_->launch_start_count(), 1); +} + +} // namespace testing +} // namespace shorebird +} // namespace flutter diff --git a/engine/src/flutter/shell/platform/android/BUILD.gn b/engine/src/flutter/shell/platform/android/BUILD.gn index 11b208db449e5..c3b50e380bacf 100644 --- a/engine/src/flutter/shell/platform/android/BUILD.gn +++ b/engine/src/flutter/shell/platform/android/BUILD.gn @@ -185,8 +185,6 @@ source_set("flutter_shell_native_src") { public_configs = [ "//flutter:config" ] - include_dirs = [ "//flutter/updater" ] - defines = [] libs = [ @@ -194,17 +192,6 @@ source_set("flutter_shell_native_src") { "EGL", "GLESv2", ] - if (target_cpu == "arm") { - libs += [ "//flutter/third_party/updater/target/armv7-linux-androideabi/release/libupdater.a" ] - } else if (target_cpu == "arm64") { - libs += [ "//flutter/third_party/updater/target/aarch64-linux-android/release/libupdater.a" ] - } else if (target_cpu == "x64") { - libs += [ "//flutter/third_party/updater/target/x86_64-linux-android/release/libupdater.a" ] - } else if (target_cpu == "x86") { - libs += [ "//flutter/third_party/updater/target/i686-linux-android/release/libupdater.a" ] - } else { - assert(false, "Unsupported target_cpu") - } } action("gen_android_build_config_java") { diff --git a/engine/src/flutter/shell/platform/android/flutter_main.cc b/engine/src/flutter/shell/platform/android/flutter_main.cc index d881ca39efd3f..45e369f5b3a9d 100644 --- a/engine/src/flutter/shell/platform/android/flutter_main.cc +++ b/engine/src/flutter/shell/platform/android/flutter_main.cc @@ -30,8 +30,6 @@ #include "impeller/toolkit/android/proc_table.h" #include "txt/platform.h" -#include "third_party/updater/library/include/updater.h" - namespace flutter { constexpr int kMinimumAndroidApiLevelForImpeller = 29; diff --git a/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn b/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn index fd167854107b1..777b6ae4a64bb 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn +++ b/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn @@ -75,14 +75,6 @@ source_set("flutter_framework_source") { "//build/config/ios:ios_application_extension", ] - if (target_cpu == "arm64") { - libs = [ "//flutter/third_party/updater/target/aarch64-apple-ios/release/libupdater.a" ] - } else if (target_cpu == "x64") { - libs = [ "//flutter/third_party/updater/target/x86_64-apple-ios/release/libupdater.a" ] - } else { - assert(false, "Unsupported target_cpu") - } - sources = [ "framework/Source/FlutterAppDelegate.mm", "framework/Source/FlutterAppDelegate_Internal.h", From 983506a24668a7ee96d2437948925d68bf764fd1 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 30 Jan 2026 18:41:39 -0800 Subject: [PATCH 20/51] fix: start/success reporting (#103) Previously we stopped reporting start on android by accident. This fixes that. I also removed the once-per-process guard since it's not necessary. This should be correctly reporting once-per-shell and let the rust code only handle the first of the calls. Fixes https://github.com/shorebirdtech/shorebird/issues/3488 --- engine/src/flutter/runtime/dart_snapshot.cc | 6 +++ .../flutter/runtime/shorebird/patch_cache.cc | 15 ++----- .../flutter/shell/common/shell_unittests.cc | 42 +++++++++-------- .../common/shorebird/updater_unittests.cc | 45 ++++++++++--------- 4 files changed, 53 insertions(+), 55 deletions(-) diff --git a/engine/src/flutter/runtime/dart_snapshot.cc b/engine/src/flutter/runtime/dart_snapshot.cc index b822d98604712..dc487590f65a7 100644 --- a/engine/src/flutter/runtime/dart_snapshot.cc +++ b/engine/src/flutter/runtime/dart_snapshot.cc @@ -17,6 +17,7 @@ #if SHOREBIRD_USE_INTERPRETER #include "flutter/runtime/shorebird/patch_cache.h" // nogncheck #endif +#include "flutter/shell/common/shorebird/updater.h" // nogncheck namespace flutter { @@ -151,6 +152,11 @@ static std::shared_ptr ResolveIsolateData( true // dontneed_safe ); #else // DART_SNAPSHOT_STATIC_LINK + // Report launch start to pair with ReportLaunchSuccess/ReportLaunchFailure + // in Shell::Shell. Called once per engine, matching the per-engine success/ + // failure calls. The Rust updater no-ops when no patch is booting. + FML_LOG(INFO) << "Reporting launch start for patch"; + shorebird::Updater::Instance().ReportLaunchStart(); #if SHOREBIRD_USE_INTERPRETER // Try loading from a Shorebird patch first. if (auto mapping = TryLoadFromPatch(settings.application_library_paths, diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.cc b/engine/src/flutter/runtime/shorebird/patch_cache.cc index b835d01b98639..8c5cc3a83f182 100644 --- a/engine/src/flutter/runtime/shorebird/patch_cache.cc +++ b/engine/src/flutter/runtime/shorebird/patch_cache.cc @@ -9,7 +9,6 @@ #include "flutter/fml/logging.h" #include "flutter/fml/mapping.h" #include "flutter/runtime/shorebird/patch_mapping.h" -#include "flutter/shell/common/shorebird/updater.h" #include "third_party/dart/runtime/include/dart_api.h" namespace flutter { @@ -152,18 +151,10 @@ std::shared_ptr TryLoadFromPatch( FML_LOG(INFO) << "Loading symbol from patch: " << symbol_name; - // Report launch_start when we're actually about to use a patch. - // This is called at exactly the right time - right before the patched - // snapshot is loaded. We use std::once_flag to ensure it's only called - // once per process, and only for the first symbol (isolate data). - // This fixes the FlutterEngineGroup issue where report_launch_start was - // called too early (in ConfigureShorebird) before any patch was used. - static std::once_flag launch_start_flag; + // ReportLaunchStart is now called from ResolveIsolateData in + // dart_snapshot.cc, which runs before TryLoadFromPatch on all platforms. + if (symbol == kIsolateDataSymbol) { - std::call_once(launch_start_flag, []() { - FML_LOG(INFO) << "Reporting launch start for patch"; - shorebird::Updater::Instance().ReportLaunchStart(); - }); return PatchMapping::CreateIsolateData(cache_entry); } else { FML_CHECK(symbol == kIsolateInstructionsSymbol); diff --git a/engine/src/flutter/shell/common/shell_unittests.cc b/engine/src/flutter/shell/common/shell_unittests.cc index 8a57f10a63341..316791dbb4e69 100644 --- a/engine/src/flutter/shell/common/shell_unittests.cc +++ b/engine/src/flutter/shell/common/shell_unittests.cc @@ -5110,60 +5110,58 @@ TEST_F(ShellTest, ShoulDiscardLayerTreeIfFrameIsSizedIncorrectly) { DestroyShell(std::move(shell), task_runners); } -// Test that Shell creation triggers the Shorebird Updater's ReportLaunchSuccess -// call. This is important for the crash recovery mechanism to work correctly. -TEST_F(ShellTest, ShorebirdUpdaterReportLaunchSuccessOnShellCreation) { - // Install a mock updater before creating the shell +// Test the full boot flow: ReportLaunchStart is called from +// ResolveIsolateData, then ReportLaunchSuccess from the Shell constructor. +// Each shell gets a paired Start+Success. +TEST_F(ShellTest, ShorebirdBootFlowCallsLaunchStartThenSuccess) { auto mock = std::make_unique(); auto* mock_ptr = mock.get(); shorebird::Updater::SetInstanceForTesting(std::move(mock)); - EXPECT_EQ(mock_ptr->launch_success_count(), 0); - EXPECT_EQ(mock_ptr->launch_failure_count(), 0); - auto settings = CreateSettingsForFixture(); auto task_runners = GetTaskRunnersForFixture(); auto shell = CreateShell(settings, task_runners); ASSERT_TRUE(shell); - // Shell constructor should have called ReportLaunchSuccess - EXPECT_EQ(mock_ptr->launch_success_count(), 1); - EXPECT_EQ(mock_ptr->launch_failure_count(), 0); - - // Verify the call was logged const auto& log = mock_ptr->call_log(); - EXPECT_TRUE(std::find(log.begin(), log.end(), "ReportLaunchSuccess") != - log.end()); + ASSERT_EQ(log.size(), 2u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchSuccess"); DestroyShell(std::move(shell), task_runners); - - // Clean up - reset the updater instance shorebird::Updater::ResetInstanceForTesting(); } -// Test that creating multiple shells only calls ReportLaunchSuccess for each -// shell. This verifies that each Shell reports its own launch status. +// Test that each shell gets a paired ReportLaunchStart + ReportLaunchSuccess. TEST_F(ShellTest, ShorebirdUpdaterReportLaunchSuccessForMultipleShells) { auto mock = std::make_unique(); auto* mock_ptr = mock.get(); shorebird::Updater::SetInstanceForTesting(std::move(mock)); - EXPECT_EQ(mock_ptr->launch_success_count(), 0); - auto settings = CreateSettingsForFixture(); - // Create first shell + // Create first shell โ€” gets Start + Success auto task_runners1 = GetTaskRunnersForFixture(); auto shell1 = CreateShell(settings, task_runners1); ASSERT_TRUE(shell1); + EXPECT_EQ(mock_ptr->launch_start_count(), 1); EXPECT_EQ(mock_ptr->launch_success_count(), 1); - // Create second shell + // Create second shell โ€” also gets Start + Success auto task_runners2 = GetTaskRunnersForFixture(); auto shell2 = CreateShell(settings, task_runners2); ASSERT_TRUE(shell2); + EXPECT_EQ(mock_ptr->launch_start_count(), 2); EXPECT_EQ(mock_ptr->launch_success_count(), 2); + // Full call log: Start+Success per shell + const auto& log = mock_ptr->call_log(); + ASSERT_EQ(log.size(), 4u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchSuccess"); + EXPECT_EQ(log[2], "ReportLaunchStart"); + EXPECT_EQ(log[3], "ReportLaunchSuccess"); + DestroyShell(std::move(shell1), task_runners1); DestroyShell(std::move(shell2), task_runners2); diff --git a/engine/src/flutter/shell/common/shorebird/updater_unittests.cc b/engine/src/flutter/shell/common/shorebird/updater_unittests.cc index 97fb6ef2129f2..569673a9e5b1c 100644 --- a/engine/src/flutter/shell/common/shorebird/updater_unittests.cc +++ b/engine/src/flutter/shell/common/shorebird/updater_unittests.cc @@ -4,8 +4,6 @@ #include "flutter/shell/common/shorebird/updater.h" -#include - #include "gtest/gtest.h" namespace flutter { @@ -100,26 +98,31 @@ TEST_F(UpdaterTest, MockUpdaterResetClearsState) { EXPECT_FALSE(mock_->ShouldAutoUpdate()); } -// Test that demonstrates the std::once_flag pattern works correctly. -// This is the same pattern used in TryLoadFromPatch. -TEST_F(UpdaterTest, OncePerProcessPatternOnlyCallsOnce) { - static std::once_flag test_flag; - int call_count = 0; - - auto simulate_patch_load = [&]() { - std::call_once(test_flag, [&]() { - call_count++; - Updater::Instance().ReportLaunchStart(); - }); - }; - - // Simulate multiple engines loading patches - simulate_patch_load(); // Engine 1 - simulate_patch_load(); // Engine 2 - simulate_patch_load(); // Engine 3 - - EXPECT_EQ(call_count, 1); +// ReportLaunchStart and ReportLaunchSuccess are always paired per shell. +// The Rust updater no-ops both when no patch is booting. +TEST_F(UpdaterTest, LaunchStartAndSuccessAreAlwaysPaired) { + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_success_count(), 1); + const auto& log = mock_->call_log(); + ASSERT_EQ(log.size(), 2u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchSuccess"); +} + +// ReportLaunchStart and ReportLaunchFailure are paired on failed boots. +TEST_F(UpdaterTest, LaunchStartAndFailureAreAlwaysPaired) { + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchFailure(); + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_failure_count(), 1); + const auto& log = mock_->call_log(); + ASSERT_EQ(log.size(), 2u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchFailure"); } } // namespace testing From fc5f5f8dbc505e4ee5099221f90ca41ec3a306cb Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 2 Feb 2026 16:17:02 -0800 Subject: [PATCH 21/51] fix: only call shorebird_report_start once (#105) As part of our previous fix for FlutterEngineGroup, we introduced a new bug whereby report_launch_start could be called more than once in a multi-engine scenerio. That would cause confusion about what the current boot patch is, since the current patch is updated as part of report_launch_start. report_launch_start should only be called once per processs, which this change fixes. We still need more end-to-end testing at this layer to prevent bugs like this from sneaking in. --- engine/src/flutter/runtime/dart_snapshot.cc | 9 +-- engine/src/flutter/shell/common/shell.cc | 7 +- .../flutter/shell/common/shell_unittests.cc | 26 ++++--- .../flutter/shell/common/shorebird/updater.cc | 48 +++++++++++-- .../flutter/shell/common/shorebird/updater.h | 65 +++++++++++++---- .../common/shorebird/updater_unittests.cc | 71 ++++++++++++++++--- 6 files changed, 183 insertions(+), 43 deletions(-) diff --git a/engine/src/flutter/runtime/dart_snapshot.cc b/engine/src/flutter/runtime/dart_snapshot.cc index dc487590f65a7..fc70f63a346f2 100644 --- a/engine/src/flutter/runtime/dart_snapshot.cc +++ b/engine/src/flutter/runtime/dart_snapshot.cc @@ -152,10 +152,11 @@ static std::shared_ptr ResolveIsolateData( true // dontneed_safe ); #else // DART_SNAPSHOT_STATIC_LINK - // Report launch start to pair with ReportLaunchSuccess/ReportLaunchFailure - // in Shell::Shell. Called once per engine, matching the per-engine success/ - // failure calls. The Rust updater no-ops when no patch is booting. - FML_LOG(INFO) << "Reporting launch start for patch"; + // Tell the Rust updater we're booting from whatever patch it selected. + // This copies next_boot โ†’ current_boot in the Rust state. The call is + // guarded inside Updater to execute at most once per process โ€” see the + // Updater class comment for why this matters in add-to-app and + // FlutterEngineGroup scenarios. shorebird::Updater::Instance().ReportLaunchStart(); #if SHOREBIRD_USE_INTERPRETER // Try loading from a Shorebird patch first. diff --git a/engine/src/flutter/shell/common/shell.cc b/engine/src/flutter/shell/common/shell.cc index c424f82151e36..87b5d9094f76b 100644 --- a/engine/src/flutter/shell/common/shell.cc +++ b/engine/src/flutter/shell/common/shell.cc @@ -524,7 +524,12 @@ Shell::Shell(DartVMRef vm, is_gpu_disabled_sync_switch_(new fml::SyncSwitch(is_gpu_disabled)), weak_factory_gpu_(nullptr), weak_factory_(this) { - // Report launch status to Shorebird updater for crash recovery tracking. + // Report launch outcome to the Shorebird updater for crash recovery. + // If the VM failed to start, we report failure so the updater can roll + // back the patch. These calls are guarded inside Updater to execute at + // most once per process โ€” only the first Shell's outcome is reported. + // In add-to-app, subsequent engines are silently ignored since they + // boot from the same snapshot that was already reported on. // On unsupported platforms, NoOpUpdater handles these calls gracefully. if (!vm_) { shorebird::Updater::Instance().ReportLaunchFailure(); diff --git a/engine/src/flutter/shell/common/shell_unittests.cc b/engine/src/flutter/shell/common/shell_unittests.cc index 316791dbb4e69..5ed39de31b6be 100644 --- a/engine/src/flutter/shell/common/shell_unittests.cc +++ b/engine/src/flutter/shell/common/shell_unittests.cc @@ -5112,11 +5112,12 @@ TEST_F(ShellTest, ShoulDiscardLayerTreeIfFrameIsSizedIncorrectly) { // Test the full boot flow: ReportLaunchStart is called from // ResolveIsolateData, then ReportLaunchSuccess from the Shell constructor. -// Each shell gets a paired Start+Success. +// Both are guarded to run at most once per process. TEST_F(ShellTest, ShorebirdBootFlowCallsLaunchStartThenSuccess) { auto mock = std::make_unique(); auto* mock_ptr = mock.get(); shorebird::Updater::SetInstanceForTesting(std::move(mock)); + shorebird::Updater::ResetLaunchStateForTesting(); auto settings = CreateSettingsForFixture(); auto task_runners = GetTaskRunnersForFixture(); @@ -5129,14 +5130,20 @@ TEST_F(ShellTest, ShorebirdBootFlowCallsLaunchStartThenSuccess) { EXPECT_EQ(log[1], "ReportLaunchSuccess"); DestroyShell(std::move(shell), task_runners); + shorebird::Updater::ResetLaunchStateForTesting(); shorebird::Updater::ResetInstanceForTesting(); } -// Test that each shell gets a paired ReportLaunchStart + ReportLaunchSuccess. -TEST_F(ShellTest, ShorebirdUpdaterReportLaunchSuccessForMultipleShells) { +// In add-to-app, multiple engines may be created within a single process. +// Only the first engine should report launch start/success to the Rust +// updater. This prevents the updater from promoting a newly-downloaded patch +// to "current_boot" when subsequent engines are still running the original +// snapshot that was selected at process init time. +TEST_F(ShellTest, ShorebirdUpdaterReportsOnlyOnceForMultipleShells) { auto mock = std::make_unique(); auto* mock_ptr = mock.get(); shorebird::Updater::SetInstanceForTesting(std::move(mock)); + shorebird::Updater::ResetLaunchStateForTesting(); auto settings = CreateSettingsForFixture(); @@ -5147,24 +5154,23 @@ TEST_F(ShellTest, ShorebirdUpdaterReportLaunchSuccessForMultipleShells) { EXPECT_EQ(mock_ptr->launch_start_count(), 1); EXPECT_EQ(mock_ptr->launch_success_count(), 1); - // Create second shell โ€” also gets Start + Success + // Create second shell โ€” guarded, no additional Start or Success calls. auto task_runners2 = GetTaskRunnersForFixture(); auto shell2 = CreateShell(settings, task_runners2); ASSERT_TRUE(shell2); - EXPECT_EQ(mock_ptr->launch_start_count(), 2); - EXPECT_EQ(mock_ptr->launch_success_count(), 2); + EXPECT_EQ(mock_ptr->launch_start_count(), 1); + EXPECT_EQ(mock_ptr->launch_success_count(), 1); - // Full call log: Start+Success per shell + // Only one Start+Success pair in the call log. const auto& log = mock_ptr->call_log(); - ASSERT_EQ(log.size(), 4u); + ASSERT_EQ(log.size(), 2u); EXPECT_EQ(log[0], "ReportLaunchStart"); EXPECT_EQ(log[1], "ReportLaunchSuccess"); - EXPECT_EQ(log[2], "ReportLaunchStart"); - EXPECT_EQ(log[3], "ReportLaunchSuccess"); DestroyShell(std::move(shell1), task_runners1); DestroyShell(std::move(shell2), task_runners2); + shorebird::Updater::ResetLaunchStateForTesting(); shorebird::Updater::ResetInstanceForTesting(); } diff --git a/engine/src/flutter/shell/common/shorebird/updater.cc b/engine/src/flutter/shell/common/shorebird/updater.cc index 84de941ec2d40..63d993613c3f9 100644 --- a/engine/src/flutter/shell/common/shorebird/updater.cc +++ b/engine/src/flutter/shell/common/shorebird/updater.cc @@ -16,6 +16,8 @@ namespace shorebird { // Static member definitions std::unique_ptr Updater::instance_; std::mutex Updater::instance_mutex_; +std::atomic Updater::launch_started_{false}; +std::atomic Updater::launch_completed_{false}; Updater& Updater::Instance() { std::lock_guard lock(instance_mutex_); @@ -39,6 +41,40 @@ void Updater::ResetInstanceForTesting() { instance_.reset(); } +void Updater::ResetLaunchStateForTesting() { + launch_started_.store(false); + launch_completed_.store(false); +} + +void Updater::ReportLaunchStart() { + // Guard: only the first engine in a process should promote next_boot โ†’ + // current_boot in the Rust updater. See class-level comment for rationale. + bool expected = false; + if (!launch_started_.compare_exchange_strong(expected, true)) { + return; + } + DoReportLaunchStart(); +} + +void Updater::ReportLaunchSuccess() { + // Guard: only report success once per process. Subsequent engines reuse + // the same patch and don't need to re-confirm the boot. + bool expected = false; + if (!launch_completed_.compare_exchange_strong(expected, true)) { + return; + } + DoReportLaunchSuccess(); +} + +void Updater::ReportLaunchFailure() { + // Guard: only report failure once per process. + bool expected = false; + if (!launch_completed_.compare_exchange_strong(expected, true)) { + return; + } + DoReportLaunchFailure(); +} + #if SHOREBIRD_PLATFORM_SUPPORTED // RealUpdater implementation - wraps the Rust C API @@ -81,15 +117,15 @@ std::string RealUpdater::NextBootPatchPath() { return path; } -void RealUpdater::ReportLaunchStart() { +void RealUpdater::DoReportLaunchStart() { shorebird_report_launch_start(); } -void RealUpdater::ReportLaunchSuccess() { +void RealUpdater::DoReportLaunchSuccess() { shorebird_report_launch_success(); } -void RealUpdater::ReportLaunchFailure() { +void RealUpdater::DoReportLaunchFailure() { shorebird_report_launch_failure(); } @@ -122,17 +158,17 @@ std::string MockUpdater::NextBootPatchPath() { return next_boot_patch_path_; } -void MockUpdater::ReportLaunchStart() { +void MockUpdater::DoReportLaunchStart() { launch_start_count_++; call_log_.push_back("ReportLaunchStart"); } -void MockUpdater::ReportLaunchSuccess() { +void MockUpdater::DoReportLaunchSuccess() { launch_success_count_++; call_log_.push_back("ReportLaunchSuccess"); } -void MockUpdater::ReportLaunchFailure() { +void MockUpdater::DoReportLaunchFailure() { launch_failure_count_++; call_log_.push_back("ReportLaunchFailure"); } diff --git a/engine/src/flutter/shell/common/shorebird/updater.h b/engine/src/flutter/shell/common/shorebird/updater.h index f6fa5a9a2f6d9..aeb2d1f2d90cc 100644 --- a/engine/src/flutter/shell/common/shorebird/updater.h +++ b/engine/src/flutter/shell/common/shorebird/updater.h @@ -5,6 +5,7 @@ #ifndef FLUTTER_SHELL_COMMON_SHOREBIRD_UPDATER_H_ #define FLUTTER_SHELL_COMMON_SHOREBIRD_UPDATER_H_ +#include #include #include #include @@ -52,6 +53,29 @@ struct AppConfig { /// 1. Mocking in tests without requiring the real Rust library /// 2. Future migration from Rust to C++ implementation /// 3. Test instrumentation (call counting, logging) +/// +/// ## Launch lifecycle (start/success/failure) +/// +/// The Rust updater uses a start/success/failure protocol to detect crashes: +/// - `ReportLaunchStart` copies `next_boot` โ†’ `current_boot` in the Rust +/// state. If the app crashes before `ReportLaunchSuccess`, the updater +/// assumes the patch caused the crash and rolls back on the next launch. +/// +/// These calls are guarded to execute at most once per process because: +/// 1. The Rust updater is a process-global singleton โ€” calling +/// `report_launch_start` multiple times would repeatedly copy `next_boot` +/// โ†’ `current_boot`, which could promote a newly-downloaded (but not yet +/// booted) patch to "current" even though the running engine loaded the +/// old snapshot. +/// 2. In add-to-app, multiple FlutterEngines may be created and destroyed +/// within a single process. Each engine creation resolves snapshots and +/// constructs a Shell, but we must only report launch start/success once +/// โ€” for the first engine that actually boots. Without this guard, a +/// background update that completes between engine creations would get +/// promoted to "current" by the second engine's `ReportLaunchStart`, +/// even though that engine is still running the old snapshot. +/// +/// Tests can call `ResetLaunchStateForTesting()` to re-enable the guards. class Updater { public: virtual ~Updater() = default; @@ -69,10 +93,12 @@ class Updater { /// @return Path to patch, or empty string if no patch available virtual std::string NextBootPatchPath() = 0; - // Boot lifecycle methods - virtual void ReportLaunchStart() = 0; - virtual void ReportLaunchSuccess() = 0; - virtual void ReportLaunchFailure() = 0; + // Boot lifecycle methods โ€” guarded to run at most once per process. + // Callers may call these freely; subsequent calls after the first are + // silently ignored. + void ReportLaunchStart(); + void ReportLaunchSuccess(); + void ReportLaunchFailure(); // Update checking virtual bool ShouldAutoUpdate() = 0; @@ -85,12 +111,25 @@ class Updater { static void SetInstanceForTesting(std::unique_ptr instance); static void ResetInstanceForTesting(); + /// Resets the once-per-process launch guards so tests can verify + /// start/success/failure calls on fresh Updater instances. + static void ResetLaunchStateForTesting(); + protected: Updater() = default; + // Subclass hooks โ€” called by the public guarded methods above. + virtual void DoReportLaunchStart() = 0; + virtual void DoReportLaunchSuccess() = 0; + virtual void DoReportLaunchFailure() = 0; + private: static std::unique_ptr instance_; static std::mutex instance_mutex_; + + // Once-per-process guards for launch lifecycle. + static std::atomic launch_started_; + static std::atomic launch_completed_; }; /// No-op implementation for unsupported platforms. @@ -103,9 +142,9 @@ class NoOpUpdater : public Updater { bool Init(const AppConfig& config) override { return true; } void ValidateNextBootPatch() override {} std::string NextBootPatchPath() override { return ""; } - void ReportLaunchStart() override {} - void ReportLaunchSuccess() override {} - void ReportLaunchFailure() override {} + void DoReportLaunchStart() override {} + void DoReportLaunchSuccess() override {} + void DoReportLaunchFailure() override {} bool ShouldAutoUpdate() override { return false; } void StartUpdateThread() override {} }; @@ -121,9 +160,9 @@ class RealUpdater : public Updater { bool Init(const AppConfig& config) override; void ValidateNextBootPatch() override; std::string NextBootPatchPath() override; - void ReportLaunchStart() override; - void ReportLaunchSuccess() override; - void ReportLaunchFailure() override; + void DoReportLaunchStart() override; + void DoReportLaunchSuccess() override; + void DoReportLaunchFailure() override; bool ShouldAutoUpdate() override; void StartUpdateThread() override; }; @@ -139,9 +178,9 @@ class MockUpdater : public Updater { bool Init(const AppConfig& config) override; void ValidateNextBootPatch() override; std::string NextBootPatchPath() override; - void ReportLaunchStart() override; - void ReportLaunchSuccess() override; - void ReportLaunchFailure() override; + void DoReportLaunchStart() override; + void DoReportLaunchSuccess() override; + void DoReportLaunchFailure() override; bool ShouldAutoUpdate() override; void StartUpdateThread() override; diff --git a/engine/src/flutter/shell/common/shorebird/updater_unittests.cc b/engine/src/flutter/shell/common/shorebird/updater_unittests.cc index 569673a9e5b1c..def93050e885d 100644 --- a/engine/src/flutter/shell/common/shorebird/updater_unittests.cc +++ b/engine/src/flutter/shell/common/shorebird/updater_unittests.cc @@ -13,42 +13,56 @@ namespace testing { class UpdaterTest : public ::testing::Test { protected: void SetUp() override { - // Install a mock for each test + // Install a mock for each test and reset the once-per-process guards + // so each test starts with a clean slate. auto mock = std::make_unique(); mock_ = mock.get(); Updater::SetInstanceForTesting(std::move(mock)); + Updater::ResetLaunchStateForTesting(); } void TearDown() override { mock_ = nullptr; Updater::ResetInstanceForTesting(); + Updater::ResetLaunchStateForTesting(); } MockUpdater* mock_ = nullptr; }; -TEST_F(UpdaterTest, MockUpdaterTracksLaunchStartCalls) { +// ReportLaunchStart is guarded to run at most once per process. +// The second call should be silently ignored. +TEST_F(UpdaterTest, ReportLaunchStartOnlyCallsOnce) { EXPECT_EQ(mock_->launch_start_count(), 0); Updater::Instance().ReportLaunchStart(); EXPECT_EQ(mock_->launch_start_count(), 1); + // Second call is a no-op due to the once-per-process guard. Updater::Instance().ReportLaunchStart(); - EXPECT_EQ(mock_->launch_start_count(), 2); + EXPECT_EQ(mock_->launch_start_count(), 1); } -TEST_F(UpdaterTest, MockUpdaterTracksLaunchSuccessCalls) { +TEST_F(UpdaterTest, ReportLaunchSuccessOnlyCallsOnce) { EXPECT_EQ(mock_->launch_success_count(), 0); Updater::Instance().ReportLaunchSuccess(); EXPECT_EQ(mock_->launch_success_count(), 1); + + // Second call is a no-op. + Updater::Instance().ReportLaunchSuccess(); + EXPECT_EQ(mock_->launch_success_count(), 1); } -TEST_F(UpdaterTest, MockUpdaterTracksLaunchFailureCalls) { +TEST_F(UpdaterTest, ReportLaunchFailureOnlyCallsOnce) { EXPECT_EQ(mock_->launch_failure_count(), 0); Updater::Instance().ReportLaunchFailure(); EXPECT_EQ(mock_->launch_failure_count(), 1); + + // Second call is a no-op. + Updater::Instance().ReportLaunchFailure(); + EXPECT_EQ(mock_->launch_failure_count(), 1); } TEST_F(UpdaterTest, MockUpdaterTracksShouldAutoUpdate) { @@ -98,9 +112,9 @@ TEST_F(UpdaterTest, MockUpdaterResetClearsState) { EXPECT_FALSE(mock_->ShouldAutoUpdate()); } -// ReportLaunchStart and ReportLaunchSuccess are always paired per shell. +// ReportLaunchStart and ReportLaunchSuccess are paired once per process. // The Rust updater no-ops both when no patch is booting. -TEST_F(UpdaterTest, LaunchStartAndSuccessAreAlwaysPaired) { +TEST_F(UpdaterTest, LaunchStartAndSuccessArePairedOncePerProcess) { Updater::Instance().ReportLaunchStart(); Updater::Instance().ReportLaunchSuccess(); @@ -112,8 +126,8 @@ TEST_F(UpdaterTest, LaunchStartAndSuccessAreAlwaysPaired) { EXPECT_EQ(log[1], "ReportLaunchSuccess"); } -// ReportLaunchStart and ReportLaunchFailure are paired on failed boots. -TEST_F(UpdaterTest, LaunchStartAndFailureAreAlwaysPaired) { +// ReportLaunchStart and ReportLaunchFailure are paired once per process. +TEST_F(UpdaterTest, LaunchStartAndFailureArePairedOncePerProcess) { Updater::Instance().ReportLaunchStart(); Updater::Instance().ReportLaunchFailure(); @@ -125,6 +139,45 @@ TEST_F(UpdaterTest, LaunchStartAndFailureAreAlwaysPaired) { EXPECT_EQ(log[1], "ReportLaunchFailure"); } +// Simulates the add-to-app scenario: multiple engines call ReportLaunchStart +// and ReportLaunchSuccess, but only the first should actually reach the +// updater. This prevents the Rust updater from promoting a newly-downloaded +// patch to "current_boot" when subsequent engines are still running the +// original snapshot. +TEST_F(UpdaterTest, MultipleEnginesOnlyReportOnce) { + // First engine boots. + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + + // Second engine boots โ€” these should be no-ops. + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_success_count(), 1); + + const auto& log = mock_->call_log(); + ASSERT_EQ(log.size(), 2u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchSuccess"); +} + +// ResetLaunchStateForTesting re-enables the guards, allowing tests to +// verify launch calls on a fresh state. +TEST_F(UpdaterTest, ResetLaunchStateReenablesGuards) { + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_success_count(), 1); + + Updater::ResetLaunchStateForTesting(); + + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + EXPECT_EQ(mock_->launch_start_count(), 2); + EXPECT_EQ(mock_->launch_success_count(), 2); +} + } // namespace testing } // namespace shorebird } // namespace flutter From 3327d226310952b022d58d224d6c923f93f597b0 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 2 Feb 2026 16:56:02 -0800 Subject: [PATCH 22/51] feat: show logs when running shorebird_tests (#106) --- .../shorebird_tests/test/shorebird_tests.dart | 46 ++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/packages/shorebird_tests/test/shorebird_tests.dart b/packages/shorebird_tests/test/shorebird_tests.dart index 4bead2de4503e..e332d8714d34a 100644 --- a/packages/shorebird_tests/test/shorebird_tests.dart +++ b/packages/shorebird_tests/test/shorebird_tests.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'package:archive/archive_io.dart'; @@ -23,12 +24,19 @@ File get _flutterBinaryFile => File( ); /// Runs a flutter command using the correct binary ([_flutterBinaryFile]) with the given arguments. +/// +/// Streams stdout and stderr to the test output in real time so that +/// CI logs show progress even if the process hangs or times out. Future _runFlutterCommand( List arguments, { required Directory workingDirectory, Map? environment, -}) { - return Process.run( +}) async { + final String command = 'flutter ${arguments.join(' ')}'; + print('[$command] starting...'); + final stopwatch = Stopwatch()..start(); + + final Process process = await Process.start( _flutterBinaryFile.absolute.path, arguments, workingDirectory: workingDirectory.path, @@ -37,6 +45,40 @@ Future _runFlutterCommand( if (environment != null) ...environment, }, ); + + final StringBuffer stdoutBuffer = StringBuffer(); + final StringBuffer stderrBuffer = StringBuffer(); + + process.stdout.transform(utf8.decoder).listen((String data) { + stdoutBuffer.write(data); + // Print each line with a prefix so it's easy to identify in CI logs. + for (final String line in data.split('\n')) { + if (line.isNotEmpty) { + print(' [$command] $line'); + } + } + }); + + process.stderr.transform(utf8.decoder).listen((String data) { + stderrBuffer.write(data); + for (final String line in data.split('\n')) { + if (line.isNotEmpty) { + print(' [$command] (stderr) $line'); + } + } + }); + + final int exitCode = await process.exitCode; + stopwatch.stop(); + print('[$command] completed in ${stopwatch.elapsed} ' + '(exit code $exitCode)'); + + return ProcessResult( + process.pid, + exitCode, + stdoutBuffer.toString(), + stderrBuffer.toString(), + ); } Future _createFlutterProject(Directory projectDirectory) async { From ffe21f73e89de6cf7da1929583c275b921993d4a Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 2 Feb 2026 21:07:55 -0800 Subject: [PATCH 23/51] perf: speed up shorebird_tests with template project and Gradle cache (#108) - Create a template Flutter project once in setUpAll and copy it per test, avoiding repeated `flutter create` calls - Run a warm-up `flutter build apk` in setUpAll (outside per-test timeout) to prime the Gradle cache - Add actions/cache for ~/.gradle so subsequent CI runs start warm - Add VERBOSE env var and failure output logging from #107 --- .github/workflows/shorebird_ci.yml | 11 ++ .../shorebird_tests/test/android_test.dart | 2 + packages/shorebird_tests/test/base_test.dart | 2 + packages/shorebird_tests/test/ios_test.dart | 2 + .../shorebird_tests/test/shorebird_tests.dart | 108 ++++++++++++++---- 5 files changed, 100 insertions(+), 25 deletions(-) diff --git a/.github/workflows/shorebird_ci.yml b/.github/workflows/shorebird_ci.yml index 9597d6a599344..690db437da585 100644 --- a/.github/workflows/shorebird_ci.yml +++ b/.github/workflows/shorebird_ci.yml @@ -38,6 +38,17 @@ jobs: distribution: "zulu" java-version: "17" + - name: ๐Ÿ“ฆ Cache Gradle + if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + - name: ๐Ÿฆ Run Flutter Tools Tests # TODO(eseidel): Find a nice way to run this on windows. if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} diff --git a/packages/shorebird_tests/test/android_test.dart b/packages/shorebird_tests/test/android_test.dart index b24cce0ee9bf6..1e0a6da4f3824 100644 --- a/packages/shorebird_tests/test/android_test.dart +++ b/packages/shorebird_tests/test/android_test.dart @@ -3,6 +3,8 @@ import 'package:test/test.dart'; import 'shorebird_tests.dart'; void main() { + setUpAll(warmUpTemplateProject); + group('shorebird android projects', () { testWithShorebirdProject('can build an apk', (projectDirectory) async { await projectDirectory.runFlutterBuildApk(); diff --git a/packages/shorebird_tests/test/base_test.dart b/packages/shorebird_tests/test/base_test.dart index 14dba7ca6cc85..ce9b0564b1396 100644 --- a/packages/shorebird_tests/test/base_test.dart +++ b/packages/shorebird_tests/test/base_test.dart @@ -3,6 +3,8 @@ import 'package:test/test.dart'; import 'shorebird_tests.dart'; void main() { + setUpAll(warmUpTemplateProject); + group('shorebird helpers', () { testWithShorebirdProject('can build a base project', (projectDirectory) async { diff --git a/packages/shorebird_tests/test/ios_test.dart b/packages/shorebird_tests/test/ios_test.dart index 3fe6e1652c3e5..bb694405ae946 100644 --- a/packages/shorebird_tests/test/ios_test.dart +++ b/packages/shorebird_tests/test/ios_test.dart @@ -3,6 +3,8 @@ import 'package:test/test.dart'; import 'shorebird_tests.dart'; void main() { + setUpAll(warmUpTemplateProject); + group( 'shorebird ios projects', () { diff --git a/packages/shorebird_tests/test/shorebird_tests.dart b/packages/shorebird_tests/test/shorebird_tests.dart index e332d8714d34a..4100c6c5aca50 100644 --- a/packages/shorebird_tests/test/shorebird_tests.dart +++ b/packages/shorebird_tests/test/shorebird_tests.dart @@ -91,46 +91,104 @@ Future _createFlutterProject(Directory projectDirectory) async { } } -@isTest -Future testWithShorebirdProject(String name, - FutureOr Function(Directory projectDirectory) testFn) async { - test( - name, - () async { - final parentDirectory = Directory.systemTemp.createTempSync(); - final projectDirectory = Directory( - path.join( - parentDirectory.path, - 'shorebird_test', - ), - )..createSync(); +/// Cached template project directory, created once and reused across tests. +/// +/// This avoids running `flutter create` for every test, which saves +/// significant time (especially the first Gradle/SDK download). +Directory? _templateProject; - try { - await _createFlutterProject(projectDirectory); +/// Creates (or returns the cached) template Flutter project with +/// shorebird.yaml configured. The first call runs `flutter create` and +/// `flutter build apk` to warm up Gradle caches. +/// +/// Call this from `setUpAll` so the expensive setup runs outside per-test +/// timeouts. +Future warmUpTemplateProject() => _getTemplateProject(); + +Future _getTemplateProject() async { + if (_templateProject != null) { + return _templateProject!; + } + + final Directory templateDir = Directory( + path.join(Directory.systemTemp.createTempSync().path, 'shorebird_template'), + )..createSync(); - projectDirectory.pubspecFile.writeAsStringSync(''' -${projectDirectory.pubspecFile.readAsStringSync()} + await _createFlutterProject(templateDir); + + templateDir.pubspecFile.writeAsStringSync(''' +${templateDir.pubspecFile.readAsStringSync()} assets: - shorebird.yaml '''); - File( - path.join( - projectDirectory.path, - 'shorebird.yaml', - ), - ).writeAsStringSync(''' + File( + path.join(templateDir.path, 'shorebird.yaml'), + ).writeAsStringSync(''' app_id: "123" '''); + // Warm up the Gradle cache with a throwaway build so subsequent + // per-test builds are fast and don't hit the per-test timeout. + // Skip if Gradle cache is already populated (e.g., from GHA cache restore). + final Directory gradleCache = Directory( + path.join(Platform.environment['HOME'] ?? '', '.gradle', 'caches'), + ); + final bool hasGradleCache = + gradleCache.existsSync() && gradleCache.listSync().isNotEmpty; + if (hasGradleCache) { + print('[warmup] Gradle cache exists, skipping warm-up build'); + } else { + await _runFlutterCommand( + ['build', 'apk'], + workingDirectory: templateDir, + ); + } + + _templateProject = templateDir; + return templateDir; +} + +/// Copies the template project to a fresh directory for test isolation. +Future _copyTemplateProject() async { + final Directory template = await _getTemplateProject(); + final Directory testDir = Directory( + path.join(Directory.systemTemp.createTempSync().path, 'shorebird_test'), + ); + + // Use platform copy to preserve the full directory tree efficiently. + if (Platform.isWindows) { + await Process.run('xcopy', [ + template.path, + testDir.path, + '/E', + '/I', + '/Q', + ]); + } else { + await Process.run('cp', ['-R', template.path, testDir.path]); + } + + return testDir; +} + +@isTest +Future testWithShorebirdProject(String name, + FutureOr Function(Directory projectDirectory) testFn) async { + test( + name, + () async { + final Directory projectDirectory = await _copyTemplateProject(); + + try { await testFn(projectDirectory); } finally { projectDirectory.deleteSync(recursive: true); } }, timeout: Timeout( - // These tests usually run flutter create, flutter build, etc, which can take a while, - // specially in CI, so setting from the default of 30 seconds to 6 minutes. + // Per-test timeout can be shorter now since the template project + // creation and Gradle warm-up happen outside the test timeout. Duration(minutes: 6), ), ); From fe2749b6e9991f08c2763110d30fc4dd0e7a9f5a Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 2 Feb 2026 21:31:24 -0800 Subject: [PATCH 24/51] fix: throw on copy failure in _copyTemplateProject (#109) --- packages/shorebird_tests/test/shorebird_tests.dart | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/shorebird_tests/test/shorebird_tests.dart b/packages/shorebird_tests/test/shorebird_tests.dart index 4100c6c5aca50..409f3963e95e1 100644 --- a/packages/shorebird_tests/test/shorebird_tests.dart +++ b/packages/shorebird_tests/test/shorebird_tests.dart @@ -157,8 +157,9 @@ Future _copyTemplateProject() async { ); // Use platform copy to preserve the full directory tree efficiently. + final ProcessResult result; if (Platform.isWindows) { - await Process.run('xcopy', [ + result = await Process.run('xcopy', [ template.path, testDir.path, '/E', @@ -166,7 +167,10 @@ Future _copyTemplateProject() async { '/Q', ]); } else { - await Process.run('cp', ['-R', template.path, testDir.path]); + result = await Process.run('cp', ['-R', template.path, testDir.path]); + } + if (result.exitCode != 0) { + throw Exception('Failed to copy template project: ${result.stderr}'); } return testDir; From b7b0a78a7ebed6a88621eda165cdc85a3b919466 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Mon, 2 Feb 2026 22:28:57 -0800 Subject: [PATCH 25/51] chore: split CI into parallel jobs (#110) * chore: split CI into parallel jobs Split the single CI job into three parallel jobs: 1. flutter-tools-tests: Runs on ubuntu + macOS (unchanged) 2. shorebird-android-tests: Runs on Ubuntu only (faster runners) 3. shorebird-ios-tests: Runs on macOS only (requires Xcode) This improves CI performance by: - Running all jobs in parallel instead of sequentially - Moving Android tests off macOS to faster Ubuntu runners - Removing Windows from the matrix (nothing was running there anyway) Expected speedup on macOS: ~5 minutes (no longer runs Android tests) * Add Android smoke test on macOS Run a single "can build an apk" test on macOS to catch any platform-specific issues with Android builds on macOS. * Add comment explaining why Windows is excluded --- .github/workflows/shorebird_ci.yml | 83 ++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 17 deletions(-) diff --git a/.github/workflows/shorebird_ci.yml b/.github/workflows/shorebird_ci.yml index 690db437da585..7dbedfa88a2fa 100644 --- a/.github/workflows/shorebird_ci.yml +++ b/.github/workflows/shorebird_ci.yml @@ -11,25 +11,45 @@ on: - shorebird/dev jobs: - test: + # NOTE: Windows is not included because shorebird_tests depends on + # flutter_flavorizr which requires Xcode. Additionally, flutter_tools + # tests on Windows would need additional setup work. + + flutter-tools-tests: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} - name: ๐Ÿฆ Shorebird Test + name: ๐Ÿ› ๏ธ Flutter Tools Tests (${{ matrix.os }}) + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - name: ๐Ÿฆ Run Flutter Tools Tests + run: ../../bin/flutter test test/general.shard + working-directory: packages/flutter_tools + + shorebird-android-tests: + # Android tests run on Ubuntu (faster runners, no macOS needed) + runs-on: ubuntu-latest + + name: ๐Ÿค– Shorebird Android Tests steps: - name: ๐Ÿ“š Git Checkout uses: actions/checkout@v4 with: - # Fetch all branches and tags to ensure that Flutter can determine its version fetch-depth: 0 - # TODO(eseidel): shorebird_tests seems to assume flutter is available - # yet it doesn't seem to set it up here? - name: ๐ŸŽฏ Setup Dart uses: dart-lang/setup-dart@v1 @@ -39,7 +59,6 @@ jobs: java-version: "17" - name: ๐Ÿ“ฆ Cache Gradle - if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} uses: actions/cache@v4 with: path: | @@ -49,15 +68,45 @@ jobs: restore-keys: | ${{ runner.os }}-gradle- - - name: ๐Ÿฆ Run Flutter Tools Tests - # TODO(eseidel): Find a nice way to run this on windows. - if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} - run: ../../bin/flutter test test/general.shard - working-directory: packages/flutter_tools + - name: ๐Ÿค– Run Android Tests + run: dart test test/base_test.dart test/android_test.dart + working-directory: packages/shorebird_tests + + shorebird-ios-tests: + # iOS tests require macOS for Xcode + runs-on: macos-latest + + name: ๐ŸŽ Shorebird iOS + Android Smoke Tests + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - uses: actions/setup-java@v4 + with: + distribution: "zulu" + java-version: "17" + + - name: ๐Ÿ“ฆ Cache Gradle + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: ๐ŸŽ Run iOS Tests + run: dart test test/base_test.dart test/ios_test.dart + working-directory: packages/shorebird_tests - - name: ๐Ÿฆ Run Shorebird Tests - # TODO(felangel): These tests have a dependency on pkg:flutter_flavorizr which - # requires XCode -- therefore they don't work on Windows. - if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} - run: dart test + - name: ๐Ÿค– Run Android Smoke Test (macOS) + # Quick sanity check that Android builds work on macOS too + run: dart test test/android_test.dart --name "can build an apk" working-directory: packages/shorebird_tests From f73b6ef6809316b9e5863e4527c213afc178beaa Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Tue, 3 Feb 2026 20:40:35 -0800 Subject: [PATCH 26/51] feat: add sharded CI build runner (#111) * feat: add sharded CI build runner Adds a Dart-based shard runner for parallel engine builds: - JSON configs for Linux, macOS, Windows shards - run_shard.dart: executes gn/ninja/rust builds - compose.dart: assembles iOS/macOS frameworks - GCS upload/download for artifact staging * refactor: parse compose.json into typed objects Move from reading compose.json directly as Map to using proper ComposeConfig and ComposeDef model classes. This follows a more idiomatic Dart pattern. * feat: add finalize.dart for manifest generation and uploads Implements the finalize job logic that: - Downloads artifacts from GCS staging - Generates artifacts_manifest.yaml - Uploads to production GCS bucket (download.shorebird.dev) Ports upload logic from linux_upload.sh, mac_upload.sh, and generate_manifest.sh into a single Dart script. * fix: correct cargo-ndk invocation for Android Rust builds - Use --target flag (not -t) with Rust target triples - Set ANDROID_NDK_HOME to engine's hermetic NDK - Build all Android targets in a single cargo ndk command - Remove incorrect _androidArch helper function This matches the behavior of linux_build.sh. * test: add unit tests for config parsing Tests cover: - PlatformConfig: single-step shards, multi-step shards, compose_input - BuildStep: gn_ninja and rust step types - ComposeConfig: compose definitions, requires, script, args - Error handling for unknown shards/compose names * feat: add artifacts field to shard configs Define artifacts declaratively in JSON configs instead of hardcoding upload paths in Dart. Each artifact specifies: - src: source path relative to out/ - dst: destination path with $engine placeholder - zip: whether to zip before upload (for directories like dart-sdk) - content_hash: whether to also upload to content-hash path (for Dart SDK) This makes the config self-describing and aligns with Flutter's ci/builders/*.json pattern of explicit artifact declarations. * refactor: read shard names from JSON configs instead of hardcoding Load PlatformConfig for each platform to get shard names dynamically, rather than maintaining a duplicate list in finalize.dart. * feat(ci): add manifest generation and bucket configuration - Extract generateManifest to lib/manifest.dart with tests - Refactor finalize.dart to use artifacts from JSON configs - Add --bucket flag for test uploads to alternate buckets - Add compare_buckets.dart for validating uploads against production * chore: add pubspec.lock for shard_runner * chore: allow shard_runner pubspec.lock in gitignore * chore: use local .gitignore for shard_runner pubspec.lock * refactor: load manifest from template file Move manifest content to artifacts_manifest.template.yaml and update generateManifest to load from template file with sync IO. * fix: fail finalize on download errors instead of continuing A missing shard download means incomplete artifacts. Better to fail loudly than silently upload an incomplete build. * fix: fail on gsutil/zip errors instead of warning Upload and zip failures should halt the build, not silently continue with missing artifacts. * refactor: clean up shard_runner CLI and config parsing - Add shared runChecked() helper to eliminate duplicated Process.run + exit code check patterns across config.dart, gcs.dart, finalize.dart, and compose.dart - Add @immutable annotations to all data classes (via package:meta) - Remove implicit single-step shard shorthand; all shards now use explicit steps arrays in JSON configs - Convert all async file IO to sync equivalents (existsSync, etc.) - Make --engine-src and --run-id mandatory CLI args, removing hidden defaults and GITHUB_RUN_ID env var fallback - Restructure compose.json to use explicit flags/path_args instead of a single args list that guessed flag vs path semantics - Collect outDirs from config upfront rather than accumulating during execution * ci: add shard_runner tests to shorebird_ci workflow - Add analysis_options.yaml (package:lints/recommended with strict mode) - Add shard-runner-tests job with format, analyze, and test steps - Fix stale await on sync PlatformConfig.load in compare_buckets.dart - Reformat all files to Dart standard (80 char width) --- .github/workflows/shorebird_ci.yml | 28 ++ shorebird/ci/artifacts_manifest.template.yaml | 65 +++ shorebird/ci/compose.json | 30 ++ shorebird/ci/shard_runner/.gitignore | 2 + .../ci/shard_runner/analysis_options.yaml | 7 + .../ci/shard_runner/bin/compare_buckets.dart | 158 +++++++ shorebird/ci/shard_runner/bin/compose.dart | 104 +++++ shorebird/ci/shard_runner/bin/finalize.dart | 207 ++++++++++ shorebird/ci/shard_runner/bin/run_shard.dart | 84 ++++ shorebird/ci/shard_runner/lib/cli.dart | 83 ++++ .../ci/shard_runner/lib/compose_config.dart | 76 ++++ shorebird/ci/shard_runner/lib/config.dart | 252 ++++++++++++ shorebird/ci/shard_runner/lib/gcs.dart | 112 +++++ shorebird/ci/shard_runner/lib/manifest.dart | 26 ++ shorebird/ci/shard_runner/lib/process.dart | 36 ++ shorebird/ci/shard_runner/pubspec.lock | 389 ++++++++++++++++++ shorebird/ci/shard_runner/pubspec.yaml | 17 + .../test/compose_config_test.dart | 119 ++++++ .../ci/shard_runner/test/config_test.dart | 264 ++++++++++++ .../ci/shard_runner/test/manifest_test.dart | 208 ++++++++++ shorebird/ci/shards/linux.json | 92 +++++ shorebird/ci/shards/macos.json | 145 +++++++ shorebird/ci/shards/windows.json | 66 +++ 23 files changed, 2570 insertions(+) create mode 100644 shorebird/ci/artifacts_manifest.template.yaml create mode 100644 shorebird/ci/compose.json create mode 100644 shorebird/ci/shard_runner/.gitignore create mode 100644 shorebird/ci/shard_runner/analysis_options.yaml create mode 100644 shorebird/ci/shard_runner/bin/compare_buckets.dart create mode 100644 shorebird/ci/shard_runner/bin/compose.dart create mode 100644 shorebird/ci/shard_runner/bin/finalize.dart create mode 100644 shorebird/ci/shard_runner/bin/run_shard.dart create mode 100644 shorebird/ci/shard_runner/lib/cli.dart create mode 100644 shorebird/ci/shard_runner/lib/compose_config.dart create mode 100644 shorebird/ci/shard_runner/lib/config.dart create mode 100644 shorebird/ci/shard_runner/lib/gcs.dart create mode 100644 shorebird/ci/shard_runner/lib/manifest.dart create mode 100644 shorebird/ci/shard_runner/lib/process.dart create mode 100644 shorebird/ci/shard_runner/pubspec.lock create mode 100644 shorebird/ci/shard_runner/pubspec.yaml create mode 100644 shorebird/ci/shard_runner/test/compose_config_test.dart create mode 100644 shorebird/ci/shard_runner/test/config_test.dart create mode 100644 shorebird/ci/shard_runner/test/manifest_test.dart create mode 100644 shorebird/ci/shards/linux.json create mode 100644 shorebird/ci/shards/macos.json create mode 100644 shorebird/ci/shards/windows.json diff --git a/.github/workflows/shorebird_ci.yml b/.github/workflows/shorebird_ci.yml index 7dbedfa88a2fa..83e1511d102de 100644 --- a/.github/workflows/shorebird_ci.yml +++ b/.github/workflows/shorebird_ci.yml @@ -38,6 +38,34 @@ jobs: run: ../../bin/flutter test test/general.shard working-directory: packages/flutter_tools + shard-runner-tests: + runs-on: ubuntu-latest + + name: ๐Ÿงฉ Shard Runner Tests + + defaults: + run: + working-directory: shorebird/ci/shard_runner + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - name: ๐Ÿ“ฆ Install Dependencies + run: dart pub get + + - name: ๐Ÿ” Check Formatting + run: dart format --output=none --set-exit-if-changed . + + - name: ๐Ÿ”Ž Analyze + run: dart analyze --fatal-infos + + - name: ๐Ÿงช Run Tests + run: dart test + shorebird-android-tests: # Android tests run on Ubuntu (faster runners, no macOS needed) runs-on: ubuntu-latest diff --git a/shorebird/ci/artifacts_manifest.template.yaml b/shorebird/ci/artifacts_manifest.template.yaml new file mode 100644 index 0000000000000..4b5c8a7f0809d --- /dev/null +++ b/shorebird/ci/artifacts_manifest.template.yaml @@ -0,0 +1,65 @@ +# Template for artifacts_manifest.yaml +# This file is processed by shard_runner:finalize +# Variable substitution: {{flutter_engine_revision}} is replaced at generation time +# The $engine placeholder is kept as-is for Shorebird's artifact proxy + +flutter_engine_revision: {{flutter_engine_revision}} +storage_bucket: download.shorebird.dev +artifact_overrides: + # Android release artifacts + - flutter_infra_release/flutter/$engine/android-arm64-release/artifacts.zip + - flutter_infra_release/flutter/$engine/android-arm64-release/darwin-x64.zip + - flutter_infra_release/flutter/$engine/android-arm64-release/linux-x64.zip + - flutter_infra_release/flutter/$engine/android-arm64-release/symbols.zip + - flutter_infra_release/flutter/$engine/android-arm64-release/windows-x64.zip + + - flutter_infra_release/flutter/$engine/android-arm-release/artifacts.zip + - flutter_infra_release/flutter/$engine/android-arm-release/darwin-x64.zip + - flutter_infra_release/flutter/$engine/android-arm-release/linux-x64.zip + - flutter_infra_release/flutter/$engine/android-arm-release/symbols.zip + - flutter_infra_release/flutter/$engine/android-arm-release/windows-x64.zip + + - flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip + - flutter_infra_release/flutter/$engine/android-x64-release/darwin-x64.zip + - flutter_infra_release/flutter/$engine/android-x64-release/linux-x64.zip + - flutter_infra_release/flutter/$engine/android-x64-release/symbols.zip + - flutter_infra_release/flutter/$engine/android-x64-release/windows-x64.zip + + # engine_stamp.json + - flutter_infra_release/flutter/$engine/engine_stamp.json + + # Dart SDK + - flutter_infra_release/flutter/$engine/dart-sdk-darwin-arm64.zip + - flutter_infra_release/flutter/$engine/dart-sdk-darwin-x64.zip + - flutter_infra_release/flutter/$engine/dart-sdk-linux-x64.zip + - flutter_infra_release/flutter/$engine/dart-sdk-windows-x64.zip + + # Maven artifacts + - download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom + - download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar + - download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom + - download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar + - download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom + - download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.jar + - download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom + - download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.jar + + # Common release artifacts + - flutter_infra_release/flutter/$engine/flutter_patched_sdk_product.zip + + # iOS release artifacts + - flutter_infra_release/flutter/$engine/ios-release/artifacts.zip + - flutter_infra_release/flutter/$engine/ios-release/Flutter.dSYM.zip + + # Linux release artifacts + - flutter_infra_release/flutter/$engine/linux-x64/artifacts.zip + - flutter_infra_release/flutter/$engine/linux-x64-release/linux-x64-flutter-gtk.zip + + # macOS release artifacts + - flutter_infra_release/flutter/$engine/darwin-x64-release/artifacts.zip + - flutter_infra_release/flutter/$engine/darwin-x64-release/framework.zip + - flutter_infra_release/flutter/$engine/darwin-x64-release/gen_snapshot.zip + + # Windows release artifacts + - flutter_infra_release/flutter/$engine/windows-x64/artifacts.zip + - flutter_infra_release/flutter/$engine/windows-x64-release/windows-x64-flutter.zip diff --git a/shorebird/ci/compose.json b/shorebird/ci/compose.json new file mode 100644 index 0000000000000..1f6ca744d1e5d --- /dev/null +++ b/shorebird/ci/compose.json @@ -0,0 +1,30 @@ +{ + "ios-framework": { + "requires": ["ios-release", "ios-release-ext", "ios-sim-x64", "ios-sim-x64-ext", "ios-sim-arm64", "ios-sim-arm64-ext"], + "script": "flutter/sky/tools/create_ios_framework.py", + "flags": ["--dsym", "--strip"], + "path_args": { + "--arm64-out-dir": "ios_release", + "--simulator-x64-out-dir": "ios_debug_sim", + "--simulator-arm64-out-dir": "ios_debug_sim_arm64" + } + }, + "macos-framework": { + "requires": ["mac-arm64", "mac-x64"], + "script": "flutter/sky/tools/create_macos_framework.py", + "flags": ["--dsym", "--strip", "--zip"], + "path_args": { + "--arm64-out-dir": "mac_release_arm64", + "--x64-out-dir": "mac_release" + } + }, + "macos-gen-snapshot": { + "requires": ["mac-arm64", "mac-x64"], + "script": "flutter/sky/tools/create_macos_gen_snapshots.py", + "flags": ["--zip"], + "path_args": { + "--arm64-path": "mac_release_arm64/universal/gen_snapshot_arm64", + "--x64-path": "mac_release/universal/gen_snapshot_x64" + } + } +} diff --git a/shorebird/ci/shard_runner/.gitignore b/shorebird/ci/shard_runner/.gitignore new file mode 100644 index 0000000000000..929f3d7e632c0 --- /dev/null +++ b/shorebird/ci/shard_runner/.gitignore @@ -0,0 +1,2 @@ +# Override root .gitignore to track our lockfile +!pubspec.lock diff --git a/shorebird/ci/shard_runner/analysis_options.yaml b/shorebird/ci/shard_runner/analysis_options.yaml new file mode 100644 index 0000000000000..d04adaf90938a --- /dev/null +++ b/shorebird/ci/shard_runner/analysis_options.yaml @@ -0,0 +1,7 @@ +include: package:lints/recommended.yaml + +analyzer: + language: + strict-casts: true + strict-inference: true + strict-raw-types: true diff --git a/shorebird/ci/shard_runner/bin/compare_buckets.dart b/shorebird/ci/shard_runner/bin/compare_buckets.dart new file mode 100644 index 0000000000000..958c238bb0230 --- /dev/null +++ b/shorebird/ci/shard_runner/bin/compare_buckets.dart @@ -0,0 +1,158 @@ +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:path/path.dart' as p; +import 'package:shard_runner/config.dart'; + +/// Compares artifacts between two GCS buckets for a given engine revision. +/// +/// Usage: dart run shard_runner:compare_buckets [options] +/// +/// Example: +/// dart run shard_runner:compare_buckets \ +/// --engine-revision abc123 \ +/// --test-bucket shorebird-build-test \ +/// --production-bucket download.shorebird.dev +Future main(List args) async { + final ArgParser parser = ArgParser() + ..addOption('engine-revision', + abbr: 'r', help: 'Engine revision (git hash)', mandatory: true) + ..addOption('test-bucket', + abbr: 't', help: 'Test bucket to compare', mandatory: true) + ..addOption('production-bucket', + abbr: 'p', + help: 'Production bucket (default: download.shorebird.dev)', + defaultsTo: 'download.shorebird.dev') + ..addOption('config-dir', + abbr: 'c', help: 'Config directory containing shards/*.json') + ..addFlag('verbose', abbr: 'v', help: 'Show detailed output') + ..addFlag('help', abbr: 'h', negatable: false, help: 'Show this help'); + + final ArgResults results = parser.parse(args); + + if (results['help'] as bool) { + print('Usage: dart run shard_runner:compare_buckets [options]'); + print(''); + print('Compares artifacts between test and production GCS buckets.'); + print('Uses gsutil hash to compare file checksums (MD5 + CRC32C).'); + print(''); + print(parser.usage); + exit(0); + } + + final String engineRevision = results['engine-revision'] as String; + final String testBucket = results['test-bucket'] as String; + final String productionBucket = results['production-bucket'] as String; + final String? configDirPath = results['config-dir'] as String?; + final bool verbose = results['verbose'] as bool; + + // Find config directory + final String configDir = configDirPath ?? + p.join(p.dirname(p.dirname(Platform.script.toFilePath())), 'shards'); + + print('=' * 60); + print('Compare Buckets'); + print('=' * 60); + print('Engine revision: $engineRevision'); + print('Test bucket: $testBucket'); + print('Production bucket: $productionBucket'); + print('Config dir: $configDir'); + print(''); + + // Load configs to get artifact paths + const List platforms = ['linux', 'macos', 'windows']; + final Map configs = {}; + for (final String platform in platforms) { + configs[platform] = PlatformConfig.load(platform, configDir); + } + + // Collect all artifact paths + final List artifacts = []; + for (final PlatformConfig config in configs.values) { + for (final ShardDef shard in config.shards.values) { + for (final ArtifactDef artifact in shard.artifacts) { + final String dstPath = + artifact.dst.replaceAll(r'$engine', engineRevision); + artifacts.add(dstPath); + } + } + } + + // Add manifest + artifacts.add('shorebird/$engineRevision/artifacts_manifest.yaml'); + + print('Comparing ${artifacts.length} artifacts...\n'); + + int matches = 0; + int mismatches = 0; + int missing = 0; + + for (final String artifact in artifacts) { + final String testUri = 'gs://$testBucket/$artifact'; + final String prodUri = 'gs://$productionBucket/$artifact'; + + // Get hash from test bucket + final String? testHash = await _getHash(testUri); + if (testHash == null) { + if (verbose) print('[MISSING] $artifact (not in test bucket)'); + missing++; + continue; + } + + // Get hash from production bucket + final String? prodHash = await _getHash(prodUri); + if (prodHash == null) { + if (verbose) print('[MISSING] $artifact (not in production bucket)'); + missing++; + continue; + } + + // Compare hashes + if (testHash == prodHash) { + if (verbose) print('[OK] $artifact'); + matches++; + } else { + print('[MISMATCH] $artifact'); + print(' Test: $testHash'); + print(' Prod: $prodHash'); + mismatches++; + } + } + + print(''); + print('=' * 60); + print('Results:'); + print(' Matches: $matches'); + print(' Mismatches: $mismatches'); + print(' Missing: $missing'); + print('=' * 60); + + if (mismatches > 0) { + print('\nWARNING: Found $mismatches mismatched artifacts!'); + exit(1); + } else if (missing > 0) { + print('\nWARNING: Found $missing missing artifacts.'); + exit(2); + } else { + print('\nSUCCESS: All artifacts match!'); + } +} + +/// Gets the MD5 hash of a GCS object using gsutil hash. +Future _getHash(String uri) async { + final ProcessResult result = + await Process.run('gsutil', ['hash', uri]); + if (result.exitCode != 0) { + return null; + } + + // Parse output for MD5 hash + // Example output: + // Hashes [hex] for gs://bucket/path: + // Hash (crc32c): abc123== + // Hash (md5): xyz789== + final String output = result.stdout as String; + final RegExpMatch? md5Match = + RegExp(r'Hash \(md5\):\s+(\S+)').firstMatch(output); + return md5Match?.group(1); +} diff --git a/shorebird/ci/shard_runner/bin/compose.dart b/shorebird/ci/shard_runner/bin/compose.dart new file mode 100644 index 0000000000000..73cdb44393bb7 --- /dev/null +++ b/shorebird/ci/shard_runner/bin/compose.dart @@ -0,0 +1,104 @@ +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:path/path.dart' as p; +import 'package:shard_runner/cli.dart'; +import 'package:shard_runner/compose_config.dart'; +import 'package:shard_runner/gcs.dart'; +import 'package:shard_runner/process.dart'; + +/// Composes artifacts from multiple shards into final outputs. +/// +/// Usage: dart run shard_runner:compose [options] +/// +/// Example: +/// dart run shard_runner:compose ios-framework --engine-src ~/.engine_checkout/engine/src +Future main(List args) async { + final ArgParser parser = ArgParser(); + CliConfig.addCommonOptions(parser, includeUpload: false); + parser.addFlag('download', + defaultsTo: true, help: 'Download artifacts from GCS staging'); + + final ArgResults results = parser.parse(args); + + if (results['help'] as bool || results.rest.isEmpty) { + print('Usage: dart run shard_runner:compose [options]'); + print(''); + print('Compose names: ios-framework, macos-framework, macos-gen-snapshot'); + print(''); + print(parser.usage); + exit(results['help'] as bool ? 0 : 1); + } + + final String composeName = results.rest[0]; + final CliConfig cli = + CliConfig.fromArgs(results, scriptPath: Platform.script.toFilePath()); + final bool shouldDownload = results['download'] as bool; + + cli.printHeader('Compose Runner', { + 'Compose:': composeName, + 'Download:': shouldDownload.toString(), + }); + + // Load compose config + final ComposeConfig config; + try { + config = ComposeConfig.load(cli.configDir); + } on FileSystemException catch (e) { + print('Error: ${e.message} at ${e.path}'); + exit(1); + } + + final ComposeDef composeDef; + try { + composeDef = config.getCompose(composeName); + } on ArgumentError catch (e) { + print('Error: ${e.message}'); + exit(1); + } + + print('\n[Compose] Requires shards: ${composeDef.requires.join(', ')}'); + + // Download artifacts from each required shard + if (shouldDownload) { + for (final String shard in composeDef.requires) { + print('\n[Download] Fetching $shard artifacts...'); + await downloadFromStaging( + runId: cli.runId, + platform: 'macos', // Compose only runs for macOS currently + shard: shard, + destDir: p.join(cli.engineSrc, 'out'), + ); + } + } + + // Build script arguments + final String outDir = p.join(cli.engineSrc, 'out', 'release'); + + // Ensure output directory exists + Directory(outDir).createSync(recursive: true); + + // Build script arguments: expand path_args to absolute paths, pass flags as-is. + final List expandedArgs = ['--dst', outDir]; + for (final MapEntry entry in composeDef.pathArgs.entries) { + expandedArgs + .addAll([entry.key, p.join(cli.engineSrc, 'out', entry.value)]); + } + expandedArgs.addAll(composeDef.flags); + + // Run the composition script + print('\n[Compose] Running ${composeDef.script}...'); + print('[Compose] Args: ${expandedArgs.join(' ')}'); + + await runChecked( + 'python3', + [p.join(cli.engineSrc, composeDef.script), ...expandedArgs], + workingDirectory: cli.engineSrc, + description: 'Compose ($composeName)', + ); + + print('[Compose] Complete'); + print('\n${'='.padRight(60, '=')}'); + print('Compose $composeName completed successfully'); + print('='.padRight(60, '=')); +} diff --git a/shorebird/ci/shard_runner/bin/finalize.dart b/shorebird/ci/shard_runner/bin/finalize.dart new file mode 100644 index 0000000000000..5a6dae8a923ee --- /dev/null +++ b/shorebird/ci/shard_runner/bin/finalize.dart @@ -0,0 +1,207 @@ +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:path/path.dart' as p; +import 'package:shard_runner/cli.dart'; +import 'package:shard_runner/config.dart'; +import 'package:shard_runner/gcs.dart'; +import 'package:shard_runner/manifest.dart'; +import 'package:shard_runner/process.dart'; + +/// Finalizes a sharded build by generating manifest and uploading artifacts. +/// +/// Usage: dart run shard_runner:finalize [options] +/// +/// Example: +/// dart run shard_runner:finalize --engine-revision abc123 +Future main(List args) async { + final ArgParser parser = ArgParser() + ..addOption('engine-revision', + abbr: 'r', help: 'Engine revision (git hash)', mandatory: true) + ..addOption('base-engine-revision', + help: 'Base Flutter engine revision for manifest') + ..addOption('content-hash', help: 'Content-aware hash for Dart SDK') + ..addOption('bucket', + abbr: 'b', + help: 'GCS bucket for uploads (default: download.shorebird.dev)', + defaultsTo: 'download.shorebird.dev') + ..addFlag('download', + defaultsTo: true, help: 'Download artifacts from GCS staging') + ..addFlag('upload', + defaultsTo: true, help: 'Upload artifacts to GCS bucket') + ..addFlag('help', abbr: 'h', negatable: false, help: 'Show this help'); + + CliConfig.addCommonOptions(parser, includeUpload: false); + + final ArgResults results = parser.parse(args); + + if (results['help'] as bool) { + print('Usage: dart run shard_runner:finalize [options]'); + print(''); + print(parser.usage); + exit(0); + } + + final CliConfig cli = + CliConfig.fromArgs(results, scriptPath: Platform.script.toFilePath()); + final String engineRevision = results['engine-revision'] as String; + final String baseEngineRevision = + results['base-engine-revision'] as String? ?? engineRevision; + final String? contentHash = results['content-hash'] as String?; + final String bucket = results['bucket'] as String; + final bool shouldDownload = results['download'] as bool; + final bool shouldUpload = results['upload'] as bool; + + cli.printHeader('Finalize Build', { + 'Engine:': engineRevision, + 'Base Engine:': baseEngineRevision, + 'Bucket:': bucket, + 'Download:': shouldDownload.toString(), + 'Upload:': shouldUpload.toString(), + }); + + // Load shard configs for each platform + const List platforms = ['linux', 'macos', 'windows']; + final Map configs = {}; + for (final String platform in platforms) { + configs[platform] = PlatformConfig.load(platform, cli.configDir); + } + + // Download all artifacts from staging + if (shouldDownload) { + final String outDir = p.join(cli.engineSrc, 'out'); + Directory(outDir).createSync(recursive: true); + + for (final String platform in platforms) { + final PlatformConfig config = configs[platform]!; + for (final String shardName in config.shards.keys) { + print('\n[Download] Fetching $platform/$shardName...'); + await downloadFromStaging( + runId: cli.runId, + platform: platform, + shard: shardName, + destDir: outDir, + ); + } + } + } + + // Generate manifest + print('\n[Manifest] Generating artifacts_manifest.yaml...'); + final String manifest = + generateManifest(baseEngineRevision, configDir: cli.configDir); + final File manifestFile = + File(p.join(cli.engineSrc, 'artifacts_manifest.yaml')); + manifestFile.writeAsStringSync(manifest); + print('[Manifest] Written to ${manifestFile.path}'); + + // Upload to production + if (shouldUpload) { + print('\n[Upload] Uploading to $bucket...'); + await uploadToProduction( + engineSrc: cli.engineSrc, + engineRevision: engineRevision, + contentHash: contentHash, + configs: configs, + bucket: bucket, + ); + } + + print('\n${'=' * 60}'); + print('Finalize completed successfully'); + print('=' * 60); +} + +/// Production storage bucket name (without gs:// prefix). +const String productionBucket = 'download.shorebird.dev'; + +/// Uploads artifacts to a GCS bucket based on config definitions. +Future uploadToProduction({ + required String engineSrc, + required String engineRevision, + required String? contentHash, + required Map configs, + required String bucket, +}) async { + final String outDir = p.join(engineSrc, 'out'); + final String bucketUri = 'gs://$bucket'; + + // Helper to run gsutil cp + Future gscp(String src, String dest) async { + print('[Upload] $src -> $dest'); + await runChecked('gsutil', ['cp', src, dest], + description: 'gsutil cp $src'); + } + + // Helper to zip a directory and upload + Future zipAndUpload(String srcPath, String dest) async { + final String tempZip = '$srcPath.zip'; + print('[Zip] Creating $tempZip...'); + await runChecked( + 'zip', + ['-r', tempZip, '.'], + workingDirectory: srcPath, + description: 'zip $tempZip', + ); + await gscp(tempZip, dest); + File(tempZip).deleteSync(); + } + + // Process artifacts from all configs + for (final MapEntry entry in configs.entries) { + final String platform = entry.key; + final PlatformConfig config = entry.value; + + for (final MapEntry shardEntry in config.shards.entries) { + final String shardName = shardEntry.key; + final ShardDef shard = shardEntry.value; + + print('\n[Upload] Processing $platform/$shardName...'); + + for (final ArtifactDef artifact in shard.artifacts) { + // Resolve source path + final String srcPath = p.join(outDir, artifact.src); + + // Resolve destination path (replace $engine with actual revision) + final String dstPath = + artifact.dst.replaceAll(r'$engine', engineRevision); + final String fullDest = '$bucketUri/$dstPath'; + + // Check if source exists + final File srcFile = File(srcPath); + final Directory srcDir = Directory(srcPath); + final bool srcExists = srcFile.existsSync() || srcDir.existsSync(); + + if (!srcExists) { + print('[Skip] $srcPath (not found)'); + continue; + } + + // Handle zip flag + if (artifact.zip && srcDir.existsSync()) { + await zipAndUpload(srcPath, fullDest); + } else { + await gscp(srcPath, fullDest); + } + + // Handle content-hash uploads (for Dart SDK) + if (artifact.contentHash && contentHash != null) { + final String contentDstPath = + artifact.dst.replaceAll(r'$engine', contentHash); + final String contentFullDest = '$bucketUri/$contentDstPath'; + await gscp(srcPath, contentFullDest); + } + } + } + } + + // Upload manifest + final String manifestFile = p.join(engineSrc, 'artifacts_manifest.yaml'); + if (File(manifestFile).existsSync()) { + final String manifestDest = + '$bucketUri/shorebird/$engineRevision/artifacts_manifest.yaml'; + await gscp(manifestFile, manifestDest); + } + + print('\n[Upload] Production upload complete'); +} diff --git a/shorebird/ci/shard_runner/bin/run_shard.dart b/shorebird/ci/shard_runner/bin/run_shard.dart new file mode 100644 index 0000000000000..ed8c4cddac435 --- /dev/null +++ b/shorebird/ci/shard_runner/bin/run_shard.dart @@ -0,0 +1,84 @@ +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:shard_runner/cli.dart'; +import 'package:shard_runner/config.dart'; +import 'package:shard_runner/gcs.dart'; + +/// Runs a single build shard. +/// +/// Usage: dart run shard_runner:run_shard [options] +/// +/// Example: +/// dart run shard_runner:run_shard linux android-arm64 --engine-src ~/.engine_checkout/engine/src +Future main(List args) async { + final ArgParser parser = ArgParser(); + CliConfig.addCommonOptions(parser); + + final ArgResults results = parser.parse(args); + + if (results['help'] as bool || results.rest.length < 2) { + print( + 'Usage: dart run shard_runner:run_shard [options]'); + print(''); + print('Platforms: linux, macos, windows'); + print(''); + print(parser.usage); + exit(results['help'] as bool ? 0 : 1); + } + + final String platform = results.rest[0]; + final String shard = results.rest[1]; + final CliConfig cli = + CliConfig.fromArgs(results, scriptPath: Platform.script.toFilePath()); + + cli.printHeader('Shard Runner', { + 'Platform:': platform, + 'Shard:': shard, + 'Upload:': cli.shouldUpload.toString(), + }); + + cli.verifyEngineSrc(); + + // Load config + print('\n[Config] Loading $platform.json...'); + final PlatformConfig config = PlatformConfig.load(platform, cli.configDir); + final ShardDef shardDef = config.getShard(shard); + + print('[Config] Found ${shardDef.steps.length} step(s)'); + + // Collect output directories from GnNinja steps for upload + final List outDirs = [ + for (final BuildStep step in shardDef.steps) + if (step is GnNinjaStep) step.outDir, + ]; + + // Execute steps + final Stopwatch stopwatch = Stopwatch()..start(); + + for (int i = 0; i < shardDef.steps.length; i++) { + final BuildStep step = shardDef.steps[i]; + print( + '\n[${'Step ${i + 1}/${shardDef.steps.length}'}] ${step.runtimeType}'); + + await step.execute(cli.engineSrc); + } + + stopwatch.stop(); + print('\n[Build] Complete in ${stopwatch.elapsed}'); + + // Upload to GCS staging + if (cli.shouldUpload && outDirs.isNotEmpty) { + await uploadToStaging( + runId: cli.runId, + platform: platform, + shard: shard, + engineSrc: cli.engineSrc, + outDirs: outDirs, + ); + } + + print('\n${'='.padRight(60, '=')}'); + print('Shard $platform/$shard completed successfully'); + print('='.padRight(60, '=')); +} diff --git a/shorebird/ci/shard_runner/lib/cli.dart b/shorebird/ci/shard_runner/lib/cli.dart new file mode 100644 index 0000000000000..23a2a0b99a5a3 --- /dev/null +++ b/shorebird/ci/shard_runner/lib/cli.dart @@ -0,0 +1,83 @@ +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as p; + +/// Common CLI configuration used by shard runner scripts. +@immutable +class CliConfig { + CliConfig({ + required this.engineSrc, + required this.configDir, + required this.runId, + required this.shouldUpload, + }); + + /// Parses common options from ArgResults. + /// + /// [scriptPath] should be Platform.script.toFilePath() from the calling script. + factory CliConfig.fromArgs(ArgResults results, {required String scriptPath}) { + final String engineSrc = p.canonicalize(results['engine-src'] as String); + + // Config directory defaults to shorebird/ci (grandparent of bin/*.dart) + final String configDir = results['config-dir'] as String? ?? + p.dirname(p.dirname(p.dirname(scriptPath))); + + final String runId = results['run-id'] as String; + + final bool shouldUpload = + !results.options.contains('upload') || results['upload'] as bool; + + return CliConfig( + engineSrc: engineSrc, + configDir: configDir, + runId: runId, + shouldUpload: shouldUpload, + ); + } + final String engineSrc; + final String configDir; + final String runId; + final bool shouldUpload; + + /// Creates common argument parser options. + static void addCommonOptions(ArgParser parser, {bool includeUpload = true}) { + parser + ..addOption('engine-src', + abbr: 'e', help: 'Path to engine/src directory', mandatory: true) + ..addOption('run-id', + help: 'Build run identifier (use "local" for local development)', + mandatory: true) + ..addOption('config-dir', + abbr: 'c', help: 'Path to config directory (shorebird/ci)') + ..addFlag('help', abbr: 'h', negatable: false, help: 'Show this help'); + + if (includeUpload) { + parser.addFlag('upload', + defaultsTo: true, help: 'Upload artifacts to GCS staging'); + } + } + + /// Prints a standard header with configuration info. + void printHeader(String title, Map extra) { + print('='.padRight(60, '=')); + print(title); + print('='.padRight(60, '=')); + print('Engine: $engineSrc'); + print('Config: $configDir'); + print('Run ID: $runId'); + for (final MapEntry entry in extra.entries) { + print('${entry.key.padRight(12)}${entry.value}'); + } + print('='.padRight(60, '=')); + } + + /// Verifies that the engine source directory exists. + void verifyEngineSrc() { + if (!Directory(engineSrc).existsSync()) { + print('Error: Engine source not found at $engineSrc'); + exit(1); + } + } +} diff --git a/shorebird/ci/shard_runner/lib/compose_config.dart b/shorebird/ci/shard_runner/lib/compose_config.dart new file mode 100644 index 0000000000000..927748d326484 --- /dev/null +++ b/shorebird/ci/shard_runner/lib/compose_config.dart @@ -0,0 +1,76 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as p; + +/// Configuration for all compose operations. +@immutable +class ComposeConfig { + ComposeConfig({required this.composes}); + + factory ComposeConfig.fromJson(Map json) { + return ComposeConfig( + composes: json.map( + (String key, value) => + MapEntry(key, ComposeDef.fromJson(value as Map)), + ), + ); + } + final Map composes; + + static ComposeConfig load(String configDir) { + final File file = File(p.join(configDir, 'compose.json')); + if (!file.existsSync()) { + throw FileSystemException('compose.json not found', file.path); + } + final String content = file.readAsStringSync(); + final Map json = + jsonDecode(content) as Map; + return ComposeConfig.fromJson(json); + } + + ComposeDef getCompose(String name) { + final ComposeDef? compose = composes[name]; + if (compose == null) { + throw ArgumentError( + 'Unknown compose: $name. Available: ${composes.keys.join(', ')}'); + } + return compose; + } +} + +/// Definition of a single compose operation. +@immutable +class ComposeDef { + ComposeDef({ + required this.requires, + required this.script, + this.flags = const [], + this.pathArgs = const {}, + }); + + factory ComposeDef.fromJson(Map json) { + return ComposeDef( + requires: (json['requires'] as List).cast(), + script: json['script'] as String, + flags: (json['flags'] as List?)?.cast() ?? [], + pathArgs: (json['path_args'] as Map?) + ?.cast() ?? + {}, + ); + } + + /// Shards that must complete before this compose can run. + final List requires; + + /// Path to the Python script to execute (relative to engine/src). + final String script; + + /// Boolean flags to pass to the script (e.g., --dsym, --strip, --zip). + final List flags; + + /// Arguments whose values are paths relative to out/ (e.g., --arm64-out-dir: ios_release). + /// These are expanded to absolute paths at runtime. + final Map pathArgs; +} diff --git a/shorebird/ci/shard_runner/lib/config.dart b/shorebird/ci/shard_runner/lib/config.dart new file mode 100644 index 0000000000000..378e463f96712 --- /dev/null +++ b/shorebird/ci/shard_runner/lib/config.dart @@ -0,0 +1,252 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as p; +import 'package:shard_runner/process.dart'; + +/// Configuration for all shards on a platform. +@immutable +class PlatformConfig { + PlatformConfig({required this.shards}); + + factory PlatformConfig.fromJson(Map json) { + return PlatformConfig( + shards: json.map( + (String key, value) => + MapEntry(key, ShardDef.fromJson(value as Map)), + ), + ); + } + final Map shards; + + ShardDef getShard(String name) { + final ShardDef? shard = shards[name]; + if (shard == null) { + throw ArgumentError( + 'Unknown shard: $name. Available: ${shards.keys.join(', ')}'); + } + return shard; + } + + static PlatformConfig load(String platform, String configDir) { + final File file = File(p.join(configDir, 'shards', '$platform.json')); + if (!file.existsSync()) { + throw FileSystemException('Config file not found', file.path); + } + final String content = file.readAsStringSync(); + final Map json = + jsonDecode(content) as Map; + return PlatformConfig.fromJson(json); + } +} + +/// Definition of a single build shard. +@immutable +class ShardDef { + ShardDef( + {required this.steps, + this.composeInput, + this.artifacts = const []}); + + factory ShardDef.fromJson(Map json) { + final List steps = (json['steps'] as List) + .map((s) => BuildStep.fromJson(s as Map)) + .toList(); + + final List artifacts = (json['artifacts'] as List?) + ?.map((a) => ArtifactDef.fromJson(a as Map)) + .toList() ?? + []; + + return ShardDef( + steps: steps, + composeInput: json['compose_input'] as String?, + artifacts: artifacts, + ); + } + + /// Build steps to execute. For simple shards, this is a single GnNinja step. + final List steps; + + /// If set, this shard contributes to a compose operation. + final String? composeInput; + + /// Artifacts produced by this shard (paths relative to out_dir). + /// Used by finalize to know what to upload. + final List artifacts; +} + +/// Definition of an artifact to upload. +@immutable +class ArtifactDef { + ArtifactDef({ + required this.src, + required this.dst, + this.zip = false, + this.contentHash = false, + }); + + factory ArtifactDef.fromJson(Map json) { + return ArtifactDef( + src: json['src'] as String, + dst: json['dst'] as String, + zip: json['zip'] as bool? ?? false, + contentHash: json['content_hash'] as bool? ?? false, + ); + } + + /// Source path relative to out/ (or out// for single-step shards) + final String src; + + /// Destination path (relative to storage bucket root). + /// Supports placeholders: $engine (engine hash) + final String dst; + + /// If true, zip the source directory before uploading. + final bool zip; + + /// If true, also upload to content-hash path (for Dart SDK). + final bool contentHash; +} + +/// Base class for build steps. +sealed class BuildStep { + factory BuildStep.fromJson(Map json) { + final String type = json['type'] as String; + return switch (type) { + 'gn_ninja' => GnNinjaStep.fromJson(json), + 'rust' => RustStep.fromJson(json), + _ => throw ArgumentError('Unknown step type: $type'), + }; + } + Future execute(String engineSrc); +} + +/// A GN + Ninja build step. +@immutable +class GnNinjaStep implements BuildStep { + GnNinjaStep({ + required this.gnArgs, + required this.ninjaTargets, + required this.outDir, + }); + + factory GnNinjaStep.fromJson(Map json) { + return GnNinjaStep( + gnArgs: (json['gn_args'] as List).cast(), + ninjaTargets: (json['ninja_targets'] as List).cast(), + outDir: json['out_dir'] as String, + ); + } + final List gnArgs; + final List ninjaTargets; + final String outDir; + + @override + Future execute(String engineSrc) async { + // Import gn.dart functions + await _runGn(engineSrc, gnArgs, outDir); + await _runNinja(engineSrc, outDir, ninjaTargets); + } +} + +/// A Rust/Cargo build step. +@immutable +class RustStep implements BuildStep { + RustStep({required this.targets}); + + factory RustStep.fromJson(Map json) { + return RustStep( + targets: (json['targets'] as List).cast(), + ); + } + final List targets; + + @override + Future execute(String engineSrc) async { + await _runRust(engineSrc, targets); + } +} + +// Internal execution functions (to be moved to separate files) +Future _runGn(String engineSrc, List args, String outDir) async { + print('[GN] Building $outDir with args: ${args.join(' ')}'); + await runChecked( + 'python3', + [ + p.join(engineSrc, 'flutter', 'tools', 'gn'), + '--no-rbe', + '--no-enable-unittests', + '--target-dir', + outDir, + ...args, + ], + workingDirectory: engineSrc, + description: 'GN ($outDir)', + ); + print('[GN] Complete'); +} + +Future _runNinja( + String engineSrc, String outDir, List targets) async { + print('[Ninja] Building ${targets.join(' ')} in out/$outDir'); + await runChecked( + 'ninja', + [ + '-C', + p.join(engineSrc, 'out', outDir), + ...targets, + ], + workingDirectory: engineSrc, + description: 'Ninja ($outDir)', + ); + print('[Ninja] Complete'); +} + +Future _runRust(String engineSrc, List targets) async { + final String updaterPath = + p.join(engineSrc, 'flutter', 'third_party', 'updater', 'library'); + + // Separate Android and non-Android targets + final List androidTargets = + targets.where((String t) => t.contains('android')).toList(); + final List otherTargets = + targets.where((String t) => !t.contains('android')).toList(); + + // Build all Android targets together with cargo-ndk + if (androidTargets.isNotEmpty) { + print('[Rust] Building Android targets: ${androidTargets.join(', ')}'); + + final List args = ['ndk']; + for (final String target in androidTargets) { + args.addAll(['--target', target]); + } + args.addAll(['build', '--release']); + + await runChecked( + 'cargo', + args, + workingDirectory: updaterPath, + environment: { + 'ANDROID_NDK_HOME': + p.join(engineSrc, 'flutter', 'third_party', 'android_tools', 'ndk'), + }, + description: 'Cargo ndk (${androidTargets.join(', ')})', + ); + } + + // Build non-Android targets individually + for (final String target in otherTargets) { + print('[Rust] Building for target: $target'); + + await runChecked( + 'cargo', + ['build', '--release', '--target', target], + workingDirectory: updaterPath, + description: 'Cargo ($target)', + ); + } + + print('[Rust] Complete'); +} diff --git a/shorebird/ci/shard_runner/lib/gcs.dart b/shorebird/ci/shard_runner/lib/gcs.dart new file mode 100644 index 0000000000000..c4263658bc275 --- /dev/null +++ b/shorebird/ci/shard_runner/lib/gcs.dart @@ -0,0 +1,112 @@ +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:shard_runner/process.dart'; + +/// Staging bucket for intermediate artifacts. +const String stagingBucket = 'gs://shorebird-build-staging'; + +/// Uploads shard artifacts to GCS staging bucket. +/// +/// Artifacts are uploaded to: +/// gs://shorebird-build-staging/builds/{runId}/{platform}/{shard}/ +Future uploadToStaging({ + required String runId, + required String platform, + required String shard, + required String engineSrc, + required List outDirs, +}) async { + final String stagingRoot = '$stagingBucket/builds/$runId/$platform/$shard'; + + print('[GCS] Uploading to $stagingRoot'); + + for (final String outDir in outDirs) { + final String outPath = p.join(engineSrc, 'out', outDir); + if (!Directory(outPath).existsSync()) { + print('[GCS] Skipping $outDir (not found)'); + continue; + } + + // Create a tarball of the out directory + final String tarFile = '$outDir.tar.gz'; + print('[GCS] Creating $tarFile...'); + + await runChecked( + 'tar', + ['-czf', tarFile, '-C', p.join(engineSrc, 'out'), outDir], + workingDirectory: engineSrc, + description: 'tar create $tarFile', + ); + + // Upload to GCS + print('[GCS] Uploading $tarFile...'); + await runChecked( + 'gsutil', + ['-m', 'cp', p.join(engineSrc, tarFile), '$stagingRoot/'], + description: 'gsutil upload $tarFile', + ); + + // Clean up local tarball + File(p.join(engineSrc, tarFile)).deleteSync(); + } + + // Upload status file + final File statusFile = File(p.join(engineSrc, 'status.json')); + statusFile.writeAsStringSync('{"status": "success", "shard": "$shard"}'); + await runChecked('gsutil', ['cp', statusFile.path, '$stagingRoot/'], + description: 'gsutil upload status.json'); + statusFile.deleteSync(); + + print('[GCS] Upload complete'); +} + +/// Downloads artifacts from GCS staging bucket. +Future downloadFromStaging({ + required String runId, + required String platform, + required String shard, + required String destDir, +}) async { + final String stagingRoot = '$stagingBucket/builds/$runId/$platform/$shard'; + + print('[GCS] Downloading from $stagingRoot'); + + // List files in the staging location + final ProcessResult lsResult = await runChecked( + 'gsutil', + ['ls', stagingRoot], + description: 'gsutil ls $stagingRoot', + ); + + final List files = (lsResult.stdout as String) + .split('\n') + .where((String f) => f.endsWith('.tar.gz')) + .toList(); + + for (final String file in files) { + final String fileName = p.basename(file); + print('[GCS] Downloading $fileName...'); + + // Download + await runChecked( + 'gsutil', + ['cp', file, p.join(destDir, fileName)], + description: 'gsutil download $fileName', + ); + + // Extract + print('[GCS] Extracting $fileName...'); + await runChecked( + 'tar', + ['-xzf', fileName], + workingDirectory: destDir, + description: 'tar extract $fileName', + ); + + // Clean up tarball + File(p.join(destDir, fileName)).deleteSync(); + } + + print('[GCS] Download complete'); +} diff --git a/shorebird/ci/shard_runner/lib/manifest.dart b/shorebird/ci/shard_runner/lib/manifest.dart new file mode 100644 index 0000000000000..db5c3a5789248 --- /dev/null +++ b/shorebird/ci/shard_runner/lib/manifest.dart @@ -0,0 +1,26 @@ +import 'dart:io'; + +import 'package:path/path.dart' as p; + +/// Generates the artifacts manifest YAML from template. +/// +/// The manifest maps a Shorebird engine revision to a Flutter engine revision +/// and lists all artifact paths that should be proxied. +/// +/// [flutterEngineRevision] is the base Flutter engine revision this build is based on. +/// [configDir] is the path to the ci/ directory containing the template. +String generateManifest( + String flutterEngineRevision, { + required String configDir, +}) { + final templatePath = p.join(configDir, 'artifacts_manifest.template.yaml'); + final templateFile = File(templatePath); + + if (!templateFile.existsSync()) { + throw ArgumentError('Manifest template not found: $templatePath'); + } + + final template = templateFile.readAsStringSync(); + return template.replaceAll( + '{{flutter_engine_revision}}', flutterEngineRevision); +} diff --git a/shorebird/ci/shard_runner/lib/process.dart b/shorebird/ci/shard_runner/lib/process.dart new file mode 100644 index 0000000000000..897714d0d3d16 --- /dev/null +++ b/shorebird/ci/shard_runner/lib/process.dart @@ -0,0 +1,36 @@ +import 'dart:io'; + +/// Runs a process and throws if it exits with a non-zero code. +/// +/// Returns the [ProcessResult] for callers that need stdout/stderr. +Future runChecked( + String executable, + List arguments, { + String? workingDirectory, + Map? environment, + String? description, +}) async { + final ProcessResult result = await Process.run( + executable, + arguments, + workingDirectory: workingDirectory, + environment: environment, + ); + + if (result.exitCode != 0) { + final String desc = description ?? '$executable ${arguments.join(' ')}'; + final String stderr = (result.stderr as String).trim(); + final String stdout = (result.stdout as String).trim(); + final StringBuffer message = + StringBuffer('$desc failed (exit ${result.exitCode})'); + if (stdout.isNotEmpty) { + message.write('\nSTDOUT: $stdout'); + } + if (stderr.isNotEmpty) { + message.write('\nSTDERR: $stderr'); + } + throw Exception(message.toString()); + } + + return result; +} diff --git a/shorebird/ci/shard_runner/pubspec.lock b/shorebird/ci/shard_runner/pubspec.lock new file mode 100644 index 0000000000000..14ad0cf1c27c0 --- /dev/null +++ b/shorebird/ci/shard_runner/pubspec.lock @@ -0,0 +1,389 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + _fe_analyzer_shared: + dependency: transitive + description: + name: _fe_analyzer_shared + sha256: "796d97d925add7ffcdf5595f33a2066a6e3cee97971e6dbef09b76b7880fd760" + url: "https://pub.dev" + source: hosted + version: "94.0.0" + analyzer: + dependency: transitive + description: + name: analyzer + sha256: "9c8ebb304d72c0a0c8764344627529d9503fc83d7d73e43ed727dc532f822e4b" + url: "https://pub.dev" + source: hosted + version: "10.0.2" + args: + dependency: "direct main" + description: + name: args + sha256: d0481093c50b1da8910eb0bb301626d4d8eb7284aa739614d2b394ee09e3ea04 + url: "https://pub.dev" + source: hosted + version: "2.7.0" + async: + dependency: transitive + description: + name: async + sha256: "758e6d74e971c3e5aceb4110bfd6698efc7f501675bcfe0c775459a8140750eb" + url: "https://pub.dev" + source: hosted + version: "2.13.0" + boolean_selector: + dependency: transitive + description: + name: boolean_selector + sha256: "8aab1771e1243a5063b8b0ff68042d67334e3feab9e95b9490f9a6ebf73b42ea" + url: "https://pub.dev" + source: hosted + version: "2.1.2" + cli_config: + dependency: transitive + description: + name: cli_config + sha256: ac20a183a07002b700f0c25e61b7ee46b23c309d76ab7b7640a028f18e4d99ec + url: "https://pub.dev" + source: hosted + version: "0.2.0" + collection: + dependency: "direct main" + description: + name: collection + sha256: "2f5709ae4d3d59dd8f7cd309b4e023046b57d8a6c82130785d2b0e5868084e76" + url: "https://pub.dev" + source: hosted + version: "1.19.1" + convert: + dependency: transitive + description: + name: convert + sha256: b30acd5944035672bc15c6b7a8b47d773e41e2f17de064350988c5d02adb1c68 + url: "https://pub.dev" + source: hosted + version: "3.1.2" + coverage: + dependency: transitive + description: + name: coverage + sha256: "5da775aa218eaf2151c721b16c01c7676fbfdd99cebba2bf64e8b807a28ff94d" + url: "https://pub.dev" + source: hosted + version: "1.15.0" + crypto: + dependency: transitive + description: + name: crypto + sha256: c8ea0233063ba03258fbcf2ca4d6dadfefe14f02fab57702265467a19f27fadf + url: "https://pub.dev" + source: hosted + version: "3.0.7" + file: + dependency: transitive + description: + name: file + sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4 + url: "https://pub.dev" + source: hosted + version: "7.0.1" + frontend_server_client: + dependency: transitive + description: + name: frontend_server_client + sha256: f64a0333a82f30b0cca061bc3d143813a486dc086b574bfb233b7c1372427694 + url: "https://pub.dev" + source: hosted + version: "4.0.0" + glob: + dependency: transitive + description: + name: glob + sha256: c3f1ee72c96f8f78935e18aa8cecced9ab132419e8625dc187e1c2408efc20de + url: "https://pub.dev" + source: hosted + version: "2.1.3" + http_multi_server: + dependency: transitive + description: + name: http_multi_server + sha256: aa6199f908078bb1c5efb8d8638d4ae191aac11b311132c3ef48ce352fb52ef8 + url: "https://pub.dev" + source: hosted + version: "3.2.2" + http_parser: + dependency: transitive + description: + name: http_parser + sha256: "178d74305e7866013777bab2c3d8726205dc5a4dd935297175b19a23a2e66571" + url: "https://pub.dev" + source: hosted + version: "4.1.2" + io: + dependency: transitive + description: + name: io + sha256: dfd5a80599cf0165756e3181807ed3e77daf6dd4137caaad72d0b7931597650b + url: "https://pub.dev" + source: hosted + version: "1.0.5" + lints: + dependency: "direct dev" + description: + name: lints + sha256: cbf8d4b858bb0134ef3ef87841abdf8d63bfc255c266b7bf6b39daa1085c4290 + url: "https://pub.dev" + source: hosted + version: "3.0.0" + logging: + dependency: transitive + description: + name: logging + sha256: c8245ada5f1717ed44271ed1c26b8ce85ca3228fd2ffdb75468ab01979309d61 + url: "https://pub.dev" + source: hosted + version: "1.3.0" + matcher: + dependency: transitive + description: + name: matcher + sha256: "12956d0ad8390bbcc63ca2e1469c0619946ccb52809807067a7020d57e647aa6" + url: "https://pub.dev" + source: hosted + version: "0.12.18" + meta: + dependency: "direct main" + description: + name: meta + sha256: "9f29b9bcc8ee287b1a31e0d01be0eae99a930dbffdaecf04b3f3d82a969f296f" + url: "https://pub.dev" + source: hosted + version: "1.18.1" + mime: + dependency: transitive + description: + name: mime + sha256: "41a20518f0cb1256669420fdba0cd90d21561e560ac240f26ef8322e45bb7ed6" + url: "https://pub.dev" + source: hosted + version: "2.0.0" + node_preamble: + dependency: transitive + description: + name: node_preamble + sha256: "6e7eac89047ab8a8d26cf16127b5ed26de65209847630400f9aefd7cd5c730db" + url: "https://pub.dev" + source: hosted + version: "2.0.2" + package_config: + dependency: transitive + description: + name: package_config + sha256: f096c55ebb7deb7e384101542bfba8c52696c1b56fca2eb62827989ef2353bbc + url: "https://pub.dev" + source: hosted + version: "2.2.0" + path: + dependency: "direct main" + description: + name: path + sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5" + url: "https://pub.dev" + source: hosted + version: "1.9.1" + pool: + dependency: transitive + description: + name: pool + sha256: "978783255c543aa3586a1b3c21f6e9d720eb315376a915872c61ef8b5c20177d" + url: "https://pub.dev" + source: hosted + version: "1.5.2" + pub_semver: + dependency: transitive + description: + name: pub_semver + sha256: "5bfcf68ca79ef689f8990d1160781b4bad40a3bd5e5218ad4076ddb7f4081585" + url: "https://pub.dev" + source: hosted + version: "2.2.0" + shelf: + dependency: transitive + description: + name: shelf + sha256: e7dd780a7ffb623c57850b33f43309312fc863fb6aa3d276a754bb299839ef12 + url: "https://pub.dev" + source: hosted + version: "1.4.2" + shelf_packages_handler: + dependency: transitive + description: + name: shelf_packages_handler + sha256: "89f967eca29607c933ba9571d838be31d67f53f6e4ee15147d5dc2934fee1b1e" + url: "https://pub.dev" + source: hosted + version: "3.0.2" + shelf_static: + dependency: transitive + description: + name: shelf_static + sha256: c87c3875f91262785dade62d135760c2c69cb217ac759485334c5857ad89f6e3 + url: "https://pub.dev" + source: hosted + version: "1.1.3" + shelf_web_socket: + dependency: transitive + description: + name: shelf_web_socket + sha256: "3632775c8e90d6c9712f883e633716432a27758216dfb61bd86a8321c0580925" + url: "https://pub.dev" + source: hosted + version: "3.0.0" + source_map_stack_trace: + dependency: transitive + description: + name: source_map_stack_trace + sha256: c0713a43e323c3302c2abe2a1cc89aa057a387101ebd280371d6a6c9fa68516b + url: "https://pub.dev" + source: hosted + version: "2.1.2" + source_maps: + dependency: transitive + description: + name: source_maps + sha256: "190222579a448b03896e0ca6eca5998fa810fda630c1d65e2f78b3f638f54812" + url: "https://pub.dev" + source: hosted + version: "0.10.13" + source_span: + dependency: transitive + description: + name: source_span + sha256: "254ee5351d6cb365c859e20ee823c3bb479bf4a293c22d17a9f1bf144ce86f7c" + url: "https://pub.dev" + source: hosted + version: "1.10.1" + stack_trace: + dependency: transitive + description: + name: stack_trace + sha256: "8b27215b45d22309b5cddda1aa2b19bdfec9df0e765f2de506401c071d38d1b1" + url: "https://pub.dev" + source: hosted + version: "1.12.1" + stream_channel: + dependency: transitive + description: + name: stream_channel + sha256: "969e04c80b8bcdf826f8f16579c7b14d780458bd97f56d107d3950fdbeef059d" + url: "https://pub.dev" + source: hosted + version: "2.1.4" + string_scanner: + dependency: transitive + description: + name: string_scanner + sha256: "921cd31725b72fe181906c6a94d987c78e3b98c2e205b397ea399d4054872b43" + url: "https://pub.dev" + source: hosted + version: "1.4.1" + term_glyph: + dependency: transitive + description: + name: term_glyph + sha256: "7f554798625ea768a7518313e58f83891c7f5024f88e46e7182a4558850a4b8e" + url: "https://pub.dev" + source: hosted + version: "1.2.2" + test: + dependency: "direct dev" + description: + name: test + sha256: "54c516bbb7cee2754d327ad4fca637f78abfc3cbcc5ace83b3eda117e42cd71a" + url: "https://pub.dev" + source: hosted + version: "1.29.0" + test_api: + dependency: transitive + description: + name: test_api + sha256: "93167629bfc610f71560ab9312acdda4959de4df6fac7492c89ff0d3886f6636" + url: "https://pub.dev" + source: hosted + version: "0.7.9" + test_core: + dependency: transitive + description: + name: test_core + sha256: "394f07d21f0f2255ec9e3989f21e54d3c7dc0e6e9dbce160e5a9c1a6be0e2943" + url: "https://pub.dev" + source: hosted + version: "0.6.15" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006 + url: "https://pub.dev" + source: hosted + version: "1.4.0" + vm_service: + dependency: transitive + description: + name: vm_service + sha256: "45caa6c5917fa127b5dbcfbd1fa60b14e583afdc08bfc96dda38886ca252eb60" + url: "https://pub.dev" + source: hosted + version: "15.0.2" + watcher: + dependency: transitive + description: + name: watcher + sha256: "1398c9f081a753f9226febe8900fce8f7d0a67163334e1c94a2438339d79d635" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + web: + dependency: transitive + description: + name: web + sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a" + url: "https://pub.dev" + source: hosted + version: "1.1.1" + web_socket: + dependency: transitive + description: + name: web_socket + sha256: "34d64019aa8e36bf9842ac014bb5d2f5586ca73df5e4d9bf5c936975cae6982c" + url: "https://pub.dev" + source: hosted + version: "1.0.1" + web_socket_channel: + dependency: transitive + description: + name: web_socket_channel + sha256: d645757fb0f4773d602444000a8131ff5d48c9e47adfe9772652dd1a4f2d45c8 + url: "https://pub.dev" + source: hosted + version: "3.0.3" + webkit_inspection_protocol: + dependency: transitive + description: + name: webkit_inspection_protocol + sha256: "87d3f2333bb240704cd3f1c6b5b7acd8a10e7f0bc28c28dcf14e782014f4a572" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + yaml: + dependency: transitive + description: + name: yaml + sha256: b9da305ac7c39faa3f030eccd175340f968459dae4af175130b3fc47e40d76ce + url: "https://pub.dev" + source: hosted + version: "3.1.3" +sdks: + dart: ">=3.9.0 <4.0.0" diff --git a/shorebird/ci/shard_runner/pubspec.yaml b/shorebird/ci/shard_runner/pubspec.yaml new file mode 100644 index 0000000000000..8e0834004a88b --- /dev/null +++ b/shorebird/ci/shard_runner/pubspec.yaml @@ -0,0 +1,17 @@ +name: shard_runner +description: Dart-based build shard runner for Shorebird CI +version: 0.1.0 +publish_to: none + +environment: + sdk: ^3.0.0 + +dependencies: + args: ^2.4.0 + collection: ^1.18.0 + meta: ^1.11.0 + path: ^1.8.0 + +dev_dependencies: + lints: ^3.0.0 + test: ^1.24.0 diff --git a/shorebird/ci/shard_runner/test/compose_config_test.dart b/shorebird/ci/shard_runner/test/compose_config_test.dart new file mode 100644 index 0000000000000..24868a6affcc5 --- /dev/null +++ b/shorebird/ci/shard_runner/test/compose_config_test.dart @@ -0,0 +1,119 @@ +import 'package:test/test.dart'; +import 'package:shard_runner/compose_config.dart'; + +void main() { + group('ComposeConfig', () { + test('parses compose definitions', () { + final Map json = { + 'ios-framework': { + 'requires': ['ios-release', 'ios-sim-x64', 'ios-sim-arm64'], + 'script': 'flutter/sky/tools/create_ios_framework.py', + 'flags': ['--dsym', '--strip'], + 'path_args': { + '--arm64-out-dir': 'ios_release', + }, + }, + 'macos-framework': { + 'requires': ['mac-arm64', 'mac-x64'], + 'script': 'flutter/sky/tools/create_macos_framework.py', + 'flags': ['--zip'], + }, + }; + + final ComposeConfig config = ComposeConfig.fromJson(json); + + expect(config.composes.length, 2); + expect(config.composes.containsKey('ios-framework'), true); + expect(config.composes.containsKey('macos-framework'), true); + }); + + test('getCompose returns correct definition', () { + final Map json = { + 'ios-framework': { + 'requires': ['ios-release'], + 'script': 'create_ios_framework.py', + }, + }; + + final ComposeConfig config = ComposeConfig.fromJson(json); + final ComposeDef compose = config.getCompose('ios-framework'); + + expect(compose.requires, ['ios-release']); + expect(compose.script, 'create_ios_framework.py'); + }); + + test('getCompose throws for unknown name', () { + final ComposeConfig config = + ComposeConfig(composes: {}); + + expect( + () => config.getCompose('nonexistent'), + throwsA(isA()), + ); + }); + }); + + group('ComposeDef', () { + test('parses all fields', () { + final Map json = { + 'requires': ['shard-a', 'shard-b'], + 'script': 'path/to/script.py', + 'flags': ['--dsym', '--strip'], + 'path_args': { + '--arm64-out-dir': 'ios_release', + '--x64-out-dir': 'ios_debug_sim', + }, + }; + + final ComposeDef compose = ComposeDef.fromJson(json); + + expect(compose.requires, ['shard-a', 'shard-b']); + expect(compose.script, 'path/to/script.py'); + expect(compose.flags, ['--dsym', '--strip']); + expect(compose.pathArgs, { + '--arm64-out-dir': 'ios_release', + '--x64-out-dir': 'ios_debug_sim', + }); + }); + + test('defaults flags and path_args when missing', () { + final Map json = { + 'requires': ['shard-a'], + 'script': 'script.py', + }; + + final ComposeDef compose = ComposeDef.fromJson(json); + + expect(compose.flags, isEmpty); + expect(compose.pathArgs, isEmpty); + }); + + test('parses ios-framework config correctly', () { + final Map json = { + 'requires': [ + 'ios-release', + 'ios-release-ext', + 'ios-sim-x64', + 'ios-sim-x64-ext', + 'ios-sim-arm64', + 'ios-sim-arm64-ext', + ], + 'script': 'flutter/sky/tools/create_ios_framework.py', + 'flags': ['--dsym', '--strip'], + 'path_args': { + '--arm64-out-dir': 'ios_release', + '--simulator-x64-out-dir': 'ios_debug_sim', + '--simulator-arm64-out-dir': 'ios_debug_sim_arm64', + }, + }; + + final ComposeDef compose = ComposeDef.fromJson(json); + + expect(compose.requires.length, 6); + expect(compose.script, 'flutter/sky/tools/create_ios_framework.py'); + expect(compose.flags, ['--dsym', '--strip']); + expect(compose.pathArgs.keys, contains('--arm64-out-dir')); + expect(compose.pathArgs['--arm64-out-dir'], 'ios_release'); + }); + }); +} diff --git a/shorebird/ci/shard_runner/test/config_test.dart b/shorebird/ci/shard_runner/test/config_test.dart new file mode 100644 index 0000000000000..ec3d508864758 --- /dev/null +++ b/shorebird/ci/shard_runner/test/config_test.dart @@ -0,0 +1,264 @@ +import 'package:test/test.dart'; +import 'package:shard_runner/config.dart'; + +void main() { + group('PlatformConfig', () { + test('parses single-step shard', () { + final json = { + 'android-arm64': { + 'steps': [ + { + 'type': 'gn_ninja', + 'gn_args': [ + '--android', + '--android-cpu=arm64', + '--runtime-mode=release' + ], + 'ninja_targets': ['default', 'gen_snapshot'], + 'out_dir': 'android_release_arm64', + }, + ], + }, + }; + + final config = PlatformConfig.fromJson(json); + + expect(config.shards.length, 1); + expect(config.shards.containsKey('android-arm64'), true); + + final shard = config.getShard('android-arm64'); + expect(shard.steps.length, 1); + expect(shard.steps.first, isA()); + expect(shard.artifacts, isEmpty); + + final step = shard.steps.first as GnNinjaStep; + expect(step.gnArgs, + ['--android', '--android-cpu=arm64', '--runtime-mode=release']); + expect(step.ninjaTargets, ['default', 'gen_snapshot']); + expect(step.outDir, 'android_release_arm64'); + }); + + test('parses shard with artifacts', () { + final json = { + 'android-arm64': { + 'steps': [ + { + 'type': 'gn_ninja', + 'gn_args': ['--android'], + 'ninja_targets': ['default'], + 'out_dir': 'android_release_arm64', + }, + ], + 'artifacts': [ + { + 'src': 'zip_archives/artifacts.zip', + 'dst': 'flutter_infra/\$engine/artifacts.zip' + }, + {'src': 'maven.pom', 'dst': 'maven/\$engine/maven.pom'}, + ], + }, + }; + + final config = PlatformConfig.fromJson(json); + final shard = config.getShard('android-arm64'); + + expect(shard.artifacts.length, 2); + expect(shard.artifacts[0].src, 'zip_archives/artifacts.zip'); + expect(shard.artifacts[0].dst, 'flutter_infra/\$engine/artifacts.zip'); + expect(shard.artifacts[1].src, 'maven.pom'); + }); + + test('parses multi-step shard', () { + final json = { + 'host': { + 'steps': [ + { + 'type': 'rust', + 'targets': ['aarch64-linux-android', 'x86_64-unknown-linux-gnu'], + }, + { + 'type': 'gn_ninja', + 'gn_args': ['--runtime-mode=release'], + 'ninja_targets': ['dart_sdk'], + 'out_dir': 'host_release', + }, + ], + }, + }; + + final config = PlatformConfig.fromJson(json); + final shard = config.getShard('host'); + + expect(shard.steps.length, 2); + expect(shard.steps[0], isA()); + expect(shard.steps[1], isA()); + + final rustStep = shard.steps[0] as RustStep; + expect(rustStep.targets, + ['aarch64-linux-android', 'x86_64-unknown-linux-gnu']); + + final gnStep = shard.steps[1] as GnNinjaStep; + expect(gnStep.outDir, 'host_release'); + }); + + test('parses compose_input', () { + final json = { + 'ios-release': { + 'steps': [ + { + 'type': 'gn_ninja', + 'gn_args': ['--ios', '--runtime-mode=release'], + 'ninja_targets': ['flutter_framework'], + 'out_dir': 'ios_release', + }, + ], + 'compose_input': 'ios-framework', + }, + }; + + final config = PlatformConfig.fromJson(json); + final shard = config.getShard('ios-release'); + + expect(shard.composeInput, 'ios-framework'); + }); + + test('getShard throws for unknown shard', () { + final config = PlatformConfig(shards: {}); + + expect( + () => config.getShard('nonexistent'), + throwsA(isA()), + ); + }); + }); + + group('BuildStep.fromJson', () { + test('parses gn_ninja type', () { + final json = { + 'type': 'gn_ninja', + 'gn_args': ['--android'], + 'ninja_targets': ['default'], + 'out_dir': 'out_dir', + }; + + final step = BuildStep.fromJson(json); + expect(step, isA()); + }); + + test('parses rust type', () { + final json = { + 'type': 'rust', + 'targets': ['x86_64-unknown-linux-gnu'], + }; + + final step = BuildStep.fromJson(json); + expect(step, isA()); + }); + + test('throws for unknown type', () { + final json = { + 'type': 'unknown', + }; + + expect( + () => BuildStep.fromJson(json), + throwsA(isA()), + ); + }); + }); + + group('GnNinjaStep', () { + test('fromJson parses all fields', () { + final json = { + 'type': 'gn_ninja', + 'gn_args': ['--android', '--runtime-mode=release'], + 'ninja_targets': ['default', 'gen_snapshot'], + 'out_dir': 'android_release', + }; + + final step = GnNinjaStep.fromJson(json); + + expect(step.gnArgs, ['--android', '--runtime-mode=release']); + expect(step.ninjaTargets, ['default', 'gen_snapshot']); + expect(step.outDir, 'android_release'); + }); + }); + + group('RustStep', () { + test('fromJson parses targets', () { + final json = { + 'type': 'rust', + 'targets': [ + 'aarch64-linux-android', + 'armv7-linux-androideabi', + 'x86_64-unknown-linux-gnu', + ], + }; + + final step = RustStep.fromJson(json); + + expect(step.targets, [ + 'aarch64-linux-android', + 'armv7-linux-androideabi', + 'x86_64-unknown-linux-gnu', + ]); + }); + }); + + group('ArtifactDef', () { + test('fromJson parses src and dst', () { + final json = { + 'src': 'zip_archives/artifacts.zip', + 'dst': 'flutter_infra/\$engine/artifacts.zip', + }; + + final artifact = ArtifactDef.fromJson(json); + + expect(artifact.src, 'zip_archives/artifacts.zip'); + expect(artifact.dst, 'flutter_infra/\$engine/artifacts.zip'); + expect(artifact.zip, false); + expect(artifact.contentHash, false); + }); + + test('fromJson parses zip flag', () { + final json = { + 'src': 'dart-sdk', + 'dst': 'flutter_infra/dart-sdk.zip', + 'zip': true, + }; + + final artifact = ArtifactDef.fromJson(json); + + expect(artifact.zip, true); + }); + + test('fromJson parses content_hash flag', () { + final json = { + 'src': 'dart-sdk', + 'dst': 'flutter_infra/dart-sdk.zip', + 'content_hash': true, + }; + + final artifact = ArtifactDef.fromJson(json); + + expect(artifact.contentHash, true); + }); + + test('fromJson parses all flags together', () { + final json = { + 'src': 'host_release/dart-sdk', + 'dst': 'flutter_infra/flutter/\$engine/dart-sdk-linux-x64.zip', + 'zip': true, + 'content_hash': true, + }; + + final artifact = ArtifactDef.fromJson(json); + + expect(artifact.src, 'host_release/dart-sdk'); + expect(artifact.dst, + 'flutter_infra/flutter/\$engine/dart-sdk-linux-x64.zip'); + expect(artifact.zip, true); + expect(artifact.contentHash, true); + }); + }); +} diff --git a/shorebird/ci/shard_runner/test/manifest_test.dart b/shorebird/ci/shard_runner/test/manifest_test.dart new file mode 100644 index 0000000000000..1e8e930eb2458 --- /dev/null +++ b/shorebird/ci/shard_runner/test/manifest_test.dart @@ -0,0 +1,208 @@ +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; +import 'package:shard_runner/manifest.dart'; + +void main() { + group('generateManifest', () { + // Path to the ci/ directory where the template lives + // Tests run from shard_runner/, so go up one level to ci/ + final String configDir = p.normalize(p.join(Directory.current.path, '..')); + + test('generates valid YAML structure', () { + final manifest = generateManifest('abc123def456', configDir: configDir); + + expect(manifest, contains('flutter_engine_revision: abc123def456')); + expect(manifest, contains('storage_bucket: download.shorebird.dev')); + expect(manifest, contains('artifact_overrides:')); + }); + + test('includes Android release artifacts', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + // Android arm64 + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-arm64-release/artifacts.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-arm64-release/linux-x64.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-arm64-release/darwin-x64.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-arm64-release/windows-x64.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-arm64-release/symbols.zip')); + + // Android arm32 + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-arm-release/artifacts.zip')); + + // Android x64 + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip')); + }); + + test('includes Dart SDK for all platforms', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/dart-sdk-darwin-arm64.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/dart-sdk-darwin-x64.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/dart-sdk-linux-x64.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/dart-sdk-windows-x64.zip')); + }); + + test('includes Maven artifacts', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + // flutter_embedding_release + expect( + manifest, + contains( + r'download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom')); + expect( + manifest, + contains( + r'download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar')); + + // arm64_v8a_release + expect( + manifest, + contains( + r'download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom')); + expect( + manifest, + contains( + r'download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar')); + + // armeabi_v7a_release + expect( + manifest, + contains( + r'download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom')); + + // x86_64_release + expect( + manifest, + contains( + r'download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom')); + }); + + test('includes iOS release artifacts', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/ios-release/artifacts.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/ios-release/Flutter.dSYM.zip')); + }); + + test('includes Linux release artifacts', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/linux-x64/artifacts.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/linux-x64-release/linux-x64-flutter-gtk.zip')); + }); + + test('includes macOS release artifacts', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/darwin-x64-release/artifacts.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/darwin-x64-release/framework.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/darwin-x64-release/gen_snapshot.zip')); + }); + + test('includes Windows release artifacts', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/windows-x64/artifacts.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/windows-x64-release/windows-x64-flutter.zip')); + }); + + test('includes engine_stamp.json', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect(manifest, + contains(r'flutter_infra_release/flutter/$engine/engine_stamp.json')); + }); + + test('includes flutter_patched_sdk_product', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/flutter_patched_sdk_product.zip')); + }); + + test('uses \$engine placeholder (not hardcoded hash)', () { + final manifest = generateManifest('abc123', configDir: configDir); + + // The flutter_engine_revision should use the actual hash + expect(manifest, contains('flutter_engine_revision: abc123')); + + // But artifact paths should use $engine placeholder + expect(manifest, contains(r'$engine')); + // Should NOT contain the actual hash in artifact paths + expect(manifest.split('flutter_engine_revision:')[1], + isNot(contains('abc123/'))); + }); + + test('throws when template file not found', () { + expect( + () => generateManifest('test-hash', configDir: '/nonexistent'), + throwsA(isA()), + ); + }); + }); +} diff --git a/shorebird/ci/shards/linux.json b/shorebird/ci/shards/linux.json new file mode 100644 index 0000000000000..453ede1e851bc --- /dev/null +++ b/shorebird/ci/shards/linux.json @@ -0,0 +1,92 @@ +{ + "android-arm64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=arm64", "--runtime-mode=release"], + "ninja_targets": ["default", "gen_snapshot"], + "out_dir": "android_release_arm64" + } + ], + "artifacts": [ + {"src": "zip_archives/android-arm64-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/artifacts.zip"}, + {"src": "zip_archives/android-arm64-release/linux-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/linux-x64.zip"}, + {"src": "zip_archives/android-arm64-release/symbols.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/symbols.zip"}, + {"src": "arm64_v8a_release.pom", "dst": "download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom"}, + {"src": "arm64_v8a_release.jar", "dst": "download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar"}, + {"src": "arm64_v8a_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.maven-metadata.xml"} + ] + }, + "android-arm32": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--android", "--runtime-mode=release"], + "ninja_targets": ["default", "gen_snapshot"], + "out_dir": "android_release" + } + ], + "artifacts": [ + {"src": "zip_archives/android-arm-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/artifacts.zip"}, + {"src": "zip_archives/android-arm-release/linux-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/linux-x64.zip"}, + {"src": "zip_archives/android-arm-release/symbols.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/symbols.zip"}, + {"src": "armeabi_v7a_release.pom", "dst": "download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom"}, + {"src": "armeabi_v7a_release.jar", "dst": "download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.jar"}, + {"src": "armeabi_v7a_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.maven-metadata.xml"}, + {"src": "flutter_embedding_release.pom", "dst": "download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom"}, + {"src": "flutter_embedding_release.jar", "dst": "download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar"}, + {"src": "flutter_embedding_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.maven-metadata.xml"} + ] + }, + "android-x64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=x64", "--runtime-mode=release"], + "ninja_targets": ["default", "gen_snapshot"], + "out_dir": "android_release_x64" + } + ], + "artifacts": [ + {"src": "zip_archives/android-x64-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip"}, + {"src": "zip_archives/android-x64-release/linux-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/linux-x64.zip"}, + {"src": "zip_archives/android-x64-release/symbols.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/symbols.zip"}, + {"src": "x86_64_release.pom", "dst": "download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom"}, + {"src": "x86_64_release.jar", "dst": "download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.jar"}, + {"src": "x86_64_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.maven-metadata.xml"} + ] + }, + "host": { + "steps": [ + { + "type": "rust", + "targets": [ + "armv7-linux-androideabi", + "aarch64-linux-android", + "i686-linux-android", + "x86_64-linux-android", + "x86_64-unknown-linux-gnu" + ] + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--no-prebuilt-dart-sdk"], + "ninja_targets": ["dart_sdk", "flutter/shell/platform/linux:flutter_gtk", "flutter/build/archives:flutter_patched_sdk"], + "out_dir": "host_release" + }, + { + "type": "gn_ninja", + "gn_args": ["--no-prebuilt-dart-sdk"], + "ninja_targets": ["flutter/build/archives:artifacts"], + "out_dir": "host_debug" + } + ], + "artifacts": [ + {"src": "host_release/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-linux-x64.zip", "zip": true, "content_hash": true}, + {"src": "host_release/zip_archives/flutter_patched_sdk_product.zip", "dst": "flutter_infra_release/flutter/$engine/flutter_patched_sdk_product.zip"}, + {"src": "host_release/zip_archives/linux-x64-release/linux-x64-flutter-gtk.zip", "dst": "flutter_infra_release/flutter/$engine/linux-x64-release/linux-x64-flutter-gtk.zip"}, + {"src": "host_debug/zip_archives/linux-x64/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/linux-x64/artifacts.zip"}, + {"src": "host_release/aot_tools/aot-tools.dill", "dst": "shorebird/$engine/aot-tools.dill"} + ] + } +} diff --git a/shorebird/ci/shards/macos.json b/shorebird/ci/shards/macos.json new file mode 100644 index 0000000000000..a1ebf2d50a26e --- /dev/null +++ b/shorebird/ci/shards/macos.json @@ -0,0 +1,145 @@ +{ + "android": { + "steps": [ + { + "type": "rust", + "targets": [ + "aarch64-apple-ios", + "x86_64-apple-ios", + "aarch64-apple-darwin", + "x86_64-apple-darwin" + ] + }, + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=arm64", "--runtime-mode=release", "--gn-args=host_cpu=\"x64\""], + "ninja_targets": ["flutter/shell/platform/android:gen_snapshot"], + "out_dir": "android_release_arm64" + }, + { + "type": "gn_ninja", + "gn_args": ["--android", "--runtime-mode=release", "--gn-args=host_cpu=\"x64\""], + "ninja_targets": ["flutter/shell/platform/android:gen_snapshot"], + "out_dir": "android_release" + }, + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=x64", "--runtime-mode=release", "--gn-args=host_cpu=\"x64\""], + "ninja_targets": ["flutter/shell/platform/android:gen_snapshot"], + "out_dir": "android_release_x64" + } + ], + "artifacts": [ + {"src": "android_release_arm64/zip_archives/android-arm64-release/darwin-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/darwin-x64.zip"}, + {"src": "android_release/zip_archives/android-arm-release/darwin-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/darwin-x64.zip"}, + {"src": "android_release_x64/zip_archives/android-x64-release/darwin-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/darwin-x64.zip"} + ] + }, + "ios-release": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=release", "--gn-arg=shorebird_runtime=true"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_release" + } + ], + "compose_input": "ios-framework" + }, + "ios-release-ext": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=release", "--darwin-extension-safe", "--xcode-symlinks", "--gn-arg=shorebird_runtime=true"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_release_extension_safe" + } + ], + "compose_input": "ios-framework" + }, + "ios-sim-x64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=debug", "--simulator"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_debug_sim" + } + ], + "compose_input": "ios-framework" + }, + "ios-sim-x64-ext": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=debug", "--darwin-extension-safe", "--simulator"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_debug_sim_extension_safe" + } + ], + "compose_input": "ios-framework" + }, + "ios-sim-arm64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=debug", "--simulator", "--simulator-cpu=arm64"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_debug_sim_arm64" + } + ], + "compose_input": "ios-framework" + }, + "ios-sim-arm64-ext": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=debug", "--darwin-extension-safe", "--simulator", "--simulator-cpu=arm64"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_debug_sim_arm64_extension_safe" + } + ], + "compose_input": "ios-framework" + }, + "mac-arm64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--mac-cpu=arm64", "--no-prebuilt-dart-sdk"], + "ninja_targets": ["dart_sdk"], + "out_dir": "host_release_arm64" + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--mac", "--mac-cpu=arm64"], + "ninja_targets": ["flutter/shell/platform/darwin/macos:zip_macos_flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins", "flutter/build/archives:artifacts"], + "out_dir": "mac_release_arm64" + } + ], + "compose_input": "macos-framework", + "artifacts": [ + {"src": "host_release_arm64/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-darwin-arm64.zip", "zip": true, "content_hash": true} + ] + }, + "mac-x64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--mac-cpu=x64", "--no-prebuilt-dart-sdk"], + "ninja_targets": ["dart_sdk"], + "out_dir": "host_release" + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--mac", "--mac-cpu=x64"], + "ninja_targets": ["flutter/shell/platform/darwin/macos:zip_macos_flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins", "flutter/build/archives:artifacts"], + "out_dir": "mac_release" + } + ], + "compose_input": "macos-framework", + "artifacts": [ + {"src": "host_release/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-darwin-x64.zip", "zip": true, "content_hash": true}, + {"src": "engine_stamp.json", "dst": "flutter_infra_release/flutter/$engine/engine_stamp.json"} + ] + } +} diff --git a/shorebird/ci/shards/windows.json b/shorebird/ci/shards/windows.json new file mode 100644 index 0000000000000..5775e5d0a1ad0 --- /dev/null +++ b/shorebird/ci/shards/windows.json @@ -0,0 +1,66 @@ +{ + "android-arm64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=arm64", "--runtime-mode=release"], + "ninja_targets": ["archive_win_gen_snapshot"], + "out_dir": "android_release_arm64" + } + ], + "artifacts": [ + {"src": "zip_archives/android-arm64-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/windows-x64.zip"} + ] + }, + "android-arm32": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--android", "--runtime-mode=release"], + "ninja_targets": ["archive_win_gen_snapshot"], + "out_dir": "android_release" + } + ], + "artifacts": [ + {"src": "zip_archives/android-arm-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/windows-x64.zip"} + ] + }, + "android-x64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=x64", "--runtime-mode=release"], + "ninja_targets": ["archive_win_gen_snapshot"], + "out_dir": "android_release_x64" + } + ], + "artifacts": [ + {"src": "zip_archives/android-x64-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/windows-x64.zip"} + ] + }, + "host": { + "steps": [ + { + "type": "rust", + "targets": ["x86_64-pc-windows-msvc"] + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--no-prebuilt-dart-sdk"], + "ninja_targets": ["dart_sdk", "flutter/build/archives:windows_flutter", "gen_snapshot", "windows", "flutter/build/archives:artifacts"], + "out_dir": "host_release" + }, + { + "type": "gn_ninja", + "gn_args": ["--no-prebuilt-dart-sdk"], + "ninja_targets": ["flutter/build/archives:artifacts"], + "out_dir": "host_debug" + } + ], + "artifacts": [ + {"src": "host_release/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-windows-x64.zip", "zip": true, "content_hash": true}, + {"src": "host_release/zip_archives/windows-x64-release/windows-x64-flutter.zip", "dst": "flutter_infra_release/flutter/$engine/windows-x64-release/windows-x64-flutter.zip"}, + {"src": "host_debug/zip_archives/windows-x64/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/windows-x64/artifacts.zip"} + ] + } +} From 7da3bf6689da1d4a23480fde2cdbe02af034605c Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Tue, 3 Feb 2026 22:40:28 -0800 Subject: [PATCH 27/51] fix: add Rust build steps to each shard that needs libupdater (#112) Each shard runs on a separate machine, so it needs its own Rust build step for the updater library. Previously only the host/android shards had Rust steps, but all shards that build libflutter need libupdater.a for their specific target triple. --- shorebird/ci/shards/linux.json | 12 ++++++++++++ shorebird/ci/shards/macos.json | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/shorebird/ci/shards/linux.json b/shorebird/ci/shards/linux.json index 453ede1e851bc..51b0aa86f0b6b 100644 --- a/shorebird/ci/shards/linux.json +++ b/shorebird/ci/shards/linux.json @@ -1,6 +1,10 @@ { "android-arm64": { "steps": [ + { + "type": "rust", + "targets": ["aarch64-linux-android"] + }, { "type": "gn_ninja", "gn_args": ["--android", "--android-cpu=arm64", "--runtime-mode=release"], @@ -19,6 +23,10 @@ }, "android-arm32": { "steps": [ + { + "type": "rust", + "targets": ["armv7-linux-androideabi"] + }, { "type": "gn_ninja", "gn_args": ["--android", "--runtime-mode=release"], @@ -40,6 +48,10 @@ }, "android-x64": { "steps": [ + { + "type": "rust", + "targets": ["x86_64-linux-android"] + }, { "type": "gn_ninja", "gn_args": ["--android", "--android-cpu=x64", "--runtime-mode=release"], diff --git a/shorebird/ci/shards/macos.json b/shorebird/ci/shards/macos.json index a1ebf2d50a26e..1132a95365706 100644 --- a/shorebird/ci/shards/macos.json +++ b/shorebird/ci/shards/macos.json @@ -37,6 +37,10 @@ }, "ios-release": { "steps": [ + { + "type": "rust", + "targets": ["aarch64-apple-ios"] + }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=release", "--gn-arg=shorebird_runtime=true"], @@ -48,6 +52,10 @@ }, "ios-release-ext": { "steps": [ + { + "type": "rust", + "targets": ["aarch64-apple-ios"] + }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=release", "--darwin-extension-safe", "--xcode-symlinks", "--gn-arg=shorebird_runtime=true"], @@ -59,6 +67,10 @@ }, "ios-sim-x64": { "steps": [ + { + "type": "rust", + "targets": ["x86_64-apple-ios"] + }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--simulator"], @@ -70,6 +82,10 @@ }, "ios-sim-x64-ext": { "steps": [ + { + "type": "rust", + "targets": ["x86_64-apple-ios"] + }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--darwin-extension-safe", "--simulator"], @@ -81,6 +97,10 @@ }, "ios-sim-arm64": { "steps": [ + { + "type": "rust", + "targets": ["aarch64-apple-ios-sim"] + }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--simulator", "--simulator-cpu=arm64"], @@ -92,6 +112,10 @@ }, "ios-sim-arm64-ext": { "steps": [ + { + "type": "rust", + "targets": ["aarch64-apple-ios-sim"] + }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--darwin-extension-safe", "--simulator", "--simulator-cpu=arm64"], @@ -103,6 +127,10 @@ }, "mac-arm64": { "steps": [ + { + "type": "rust", + "targets": ["aarch64-apple-darwin"] + }, { "type": "gn_ninja", "gn_args": ["--runtime-mode=release", "--mac-cpu=arm64", "--no-prebuilt-dart-sdk"], @@ -123,6 +151,10 @@ }, "mac-x64": { "steps": [ + { + "type": "rust", + "targets": ["x86_64-apple-darwin"] + }, { "type": "gn_ninja", "gn_args": ["--runtime-mode=release", "--mac-cpu=x64", "--no-prebuilt-dart-sdk"], From 94aa3da7f31a11206d5b199bc7504b3d95905088 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Wed, 4 Feb 2026 07:13:01 -0800 Subject: [PATCH 28/51] fix: remove duplicate --help flag in finalize.dart --- shorebird/ci/shard_runner/bin/finalize.dart | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/shorebird/ci/shard_runner/bin/finalize.dart b/shorebird/ci/shard_runner/bin/finalize.dart index 5a6dae8a923ee..6534aedec8162 100644 --- a/shorebird/ci/shard_runner/bin/finalize.dart +++ b/shorebird/ci/shard_runner/bin/finalize.dart @@ -28,8 +28,7 @@ Future main(List args) async { ..addFlag('download', defaultsTo: true, help: 'Download artifacts from GCS staging') ..addFlag('upload', - defaultsTo: true, help: 'Upload artifacts to GCS bucket') - ..addFlag('help', abbr: 'h', negatable: false, help: 'Show this help'); + defaultsTo: true, help: 'Upload artifacts to GCS bucket'); CliConfig.addCommonOptions(parser, includeUpload: false); From 38839fb972fa867636b7792c2a914581f46838a4 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Wed, 4 Feb 2026 19:56:33 -0800 Subject: [PATCH 29/51] fix: resolve .cmd executables on Windows for gsutil/gcloud On Windows, gcloud SDK tools like gsutil are installed as .cmd files. When Dart's Process.run is called without runInShell, it doesn't resolve these .cmd extensions. This adds a helper that explicitly checks for .cmd versions in PATH on Windows. --- shorebird/ci/shard_runner/lib/process.dart | 31 +++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/shorebird/ci/shard_runner/lib/process.dart b/shorebird/ci/shard_runner/lib/process.dart index 897714d0d3d16..aefb1a118fc8b 100644 --- a/shorebird/ci/shard_runner/lib/process.dart +++ b/shorebird/ci/shard_runner/lib/process.dart @@ -1,5 +1,32 @@ import 'dart:io'; +/// Resolves an executable name for Windows, appending .cmd if needed. +/// +/// On Windows, many tools like gsutil and gcloud are installed as .cmd files. +/// This function checks if a .cmd version exists and uses it. +String _resolveExecutable(String executable) { + if (!Platform.isWindows) return executable; + + // Don't modify if it already has an extension + if (executable.endsWith('.exe') || executable.endsWith('.cmd') || + executable.endsWith('.bat')) { + return executable; + } + + // Check if .cmd version exists in PATH + final String? path = Platform.environment['PATH']; + if (path == null) return executable; + + for (final String dir in path.split(';')) { + final File cmdFile = File('$dir\\$executable.cmd'); + if (cmdFile.existsSync()) { + return '$executable.cmd'; + } + } + + return executable; +} + /// Runs a process and throws if it exits with a non-zero code. /// /// Returns the [ProcessResult] for callers that need stdout/stderr. @@ -10,8 +37,10 @@ Future runChecked( Map? environment, String? description, }) async { + final String resolvedExecutable = _resolveExecutable(executable); + final ProcessResult result = await Process.run( - executable, + resolvedExecutable, arguments, workingDirectory: workingDirectory, environment: environment, From fb3dc6dc6ddd085f3d0e27830e5e0784ae5336da Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Wed, 4 Feb 2026 20:19:43 -0800 Subject: [PATCH 30/51] style: format process.dart --- shorebird/ci/shard_runner/lib/process.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shorebird/ci/shard_runner/lib/process.dart b/shorebird/ci/shard_runner/lib/process.dart index aefb1a118fc8b..1fb251b609d97 100644 --- a/shorebird/ci/shard_runner/lib/process.dart +++ b/shorebird/ci/shard_runner/lib/process.dart @@ -8,7 +8,8 @@ String _resolveExecutable(String executable) { if (!Platform.isWindows) return executable; // Don't modify if it already has an extension - if (executable.endsWith('.exe') || executable.endsWith('.cmd') || + if (executable.endsWith('.exe') || + executable.endsWith('.cmd') || executable.endsWith('.bat')) { return executable; } From 2ab12169e0735283e4220b54efcbdae20725e44d Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Wed, 4 Feb 2026 22:05:22 -0800 Subject: [PATCH 31/51] fix: remove Rust steps from iOS simulator shards Simulators don't currently support Shorebird's Rust components. Remove the rust build steps from ios-sim-x64, ios-sim-x64-ext, ios-sim-arm64, and ios-sim-arm64-ext shards. --- shorebird/ci/shards/macos.json | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/shorebird/ci/shards/macos.json b/shorebird/ci/shards/macos.json index 1132a95365706..1112d0a5f01ee 100644 --- a/shorebird/ci/shards/macos.json +++ b/shorebird/ci/shards/macos.json @@ -67,10 +67,6 @@ }, "ios-sim-x64": { "steps": [ - { - "type": "rust", - "targets": ["x86_64-apple-ios"] - }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--simulator"], @@ -82,10 +78,6 @@ }, "ios-sim-x64-ext": { "steps": [ - { - "type": "rust", - "targets": ["x86_64-apple-ios"] - }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--darwin-extension-safe", "--simulator"], @@ -97,10 +89,6 @@ }, "ios-sim-arm64": { "steps": [ - { - "type": "rust", - "targets": ["aarch64-apple-ios-sim"] - }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--simulator", "--simulator-cpu=arm64"], @@ -112,10 +100,6 @@ }, "ios-sim-arm64-ext": { "steps": [ - { - "type": "rust", - "targets": ["aarch64-apple-ios-sim"] - }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--darwin-extension-safe", "--simulator", "--simulator-cpu=arm64"], From 501efe530e33951f4cbb38959cab609589a700d7 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Wed, 4 Feb 2026 23:09:12 -0800 Subject: [PATCH 32/51] fix: restore Rust for x64 iOS simulator shards The x64 simulator shards need libupdater.a built for x86_64-apple-ios. The arm64 simulator shards don't require Rust (mac_build.sh doesn't build aarch64-apple-ios-sim either). --- shorebird/ci/shards/macos.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/shorebird/ci/shards/macos.json b/shorebird/ci/shards/macos.json index 1112d0a5f01ee..db68dc3e1a2f8 100644 --- a/shorebird/ci/shards/macos.json +++ b/shorebird/ci/shards/macos.json @@ -67,6 +67,10 @@ }, "ios-sim-x64": { "steps": [ + { + "type": "rust", + "targets": ["x86_64-apple-ios"] + }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--simulator"], @@ -78,6 +82,10 @@ }, "ios-sim-x64-ext": { "steps": [ + { + "type": "rust", + "targets": ["x86_64-apple-ios"] + }, { "type": "gn_ninja", "gn_args": ["--ios", "--runtime-mode=debug", "--darwin-extension-safe", "--simulator"], From 43f07060bbcc59a8ca66dac7a5918931336a357e Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Tue, 10 Feb 2026 17:36:32 -0800 Subject: [PATCH 33/51] chore: bump dart sdk to shorebird dev 3.11.0 (b8847b5a781cfacf7edd199482278a51aa36cf5e) --- DEPS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPS b/DEPS index 752fa14dfb943..cddae06f064c2 100644 --- a/DEPS +++ b/DEPS @@ -16,7 +16,7 @@ vars = { 'skia_git': 'https://skia.googlesource.com', 'llvm_git': 'https://llvm.googlesource.com', 'skia_revision': '837be28dd21818d2169e1e6789e89e771a55f01d', - "dart_sdk_revision": "b65ce89c8057d6880e00693a7b0ecd7b9e5f61ca", + "dart_sdk_revision": "b8847b5a781cfacf7edd199482278a51aa36cf5e", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", "updater_rev": "8691c8f60e69f8eb1f35361f4a22e8c9a7fdf93c", From 75c771dfdac466a6f51553f17f14d11e389ae660 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Tue, 10 Feb 2026 19:22:10 -0800 Subject: [PATCH 34/51] fix: update NDK path for unmodified Android SDK CIPD package The upstream Flutter commit c0b808c9ed3 changed the Android SDK CIPD package to an "unmodified" layout where the NDK lives at android_tools/sdk/ndk/ instead of android_tools/ndk. Update all build scripts to dynamically discover the NDK version directory. --- shorebird/ci/internal/linux_build.sh | 3 ++- shorebird/ci/internal/linux_test_build.sh | 2 +- shorebird/ci/shard_runner/lib/config.dart | 10 ++++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/shorebird/ci/internal/linux_build.sh b/shorebird/ci/internal/linux_build.sh index 06d995f2fc5d2..9f8cbf13baa1b 100755 --- a/shorebird/ci/internal/linux_build.sh +++ b/shorebird/ci/internal/linux_build.sh @@ -25,7 +25,8 @@ cd $UPDATER_SRC/library # previous iterations of cargo-ndk required the version to be passed as # -p , but that no longer seems needed. # We always use the hermetic NDK from the engine repo. -ANDROID_NDK_HOME="$ENGINE_SRC/flutter/third_party/android_tools/ndk" \ +# The "unmodified" CIPD package keeps the NDK at the standard Android SDK path. +ANDROID_NDK_HOME=$(echo "$ENGINE_SRC/flutter/third_party/android_tools/sdk/ndk"/*) \ cargo ndk \ --target armv7-linux-androideabi \ --target aarch64-linux-android \ diff --git a/shorebird/ci/internal/linux_test_build.sh b/shorebird/ci/internal/linux_test_build.sh index be3c54da547ec..5d5fb9b0b2538 100755 --- a/shorebird/ci/internal/linux_test_build.sh +++ b/shorebird/ci/internal/linux_test_build.sh @@ -10,7 +10,7 @@ cd $ENGINE_SRC UPDATER_SRC=$ENGINE_SRC/flutter/third_party/updater (cd $UPDATER_SRC && - ANDROID_NDK_HOME="$ENGINE_SRC/flutter/third_party/android_tools/ndk" \ + ANDROID_NDK_HOME=$(echo "$ENGINE_SRC/flutter/third_party/android_tools/sdk/ndk"/*) \ cargo ndk \ --target armv7-linux-androideabi \ --target aarch64-linux-android \ diff --git a/shorebird/ci/shard_runner/lib/config.dart b/shorebird/ci/shard_runner/lib/config.dart index 378e463f96712..7cbcd01e681ff 100644 --- a/shorebird/ci/shard_runner/lib/config.dart +++ b/shorebird/ci/shard_runner/lib/config.dart @@ -224,13 +224,19 @@ Future _runRust(String engineSrc, List targets) async { } args.addAll(['build', '--release']); + // The "unmodified" CIPD package keeps the NDK at the standard Android + // SDK path: android_tools/sdk/ndk/. + final Directory ndkParent = Directory( + p.join(engineSrc, 'flutter', 'third_party', 'android_tools', 'sdk', 'ndk'), + ); + final String ndkHome = ndkParent.listSync().whereType().first.path; + await runChecked( 'cargo', args, workingDirectory: updaterPath, environment: { - 'ANDROID_NDK_HOME': - p.join(engineSrc, 'flutter', 'third_party', 'android_tools', 'ndk'), + 'ANDROID_NDK_HOME': ndkHome, }, description: 'Cargo ndk (${androidTargets.join(', ')})', ); From 1d90b19bf2a2643d87171b066d37bfbff295c544 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Tue, 10 Feb 2026 19:42:08 -0800 Subject: [PATCH 35/51] fix: update dart_io_api reference to common_embedder_dart_io The upstream Dart SDK renamed the dart_io_api GN target to common_embedder_dart_io. Update shell/testing/BUILD.gn to match. --- engine/src/flutter/shell/testing/BUILD.gn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/flutter/shell/testing/BUILD.gn b/engine/src/flutter/shell/testing/BUILD.gn index 496843c1b6469..0ed88d93ed52a 100644 --- a/engine/src/flutter/shell/testing/BUILD.gn +++ b/engine/src/flutter/shell/testing/BUILD.gn @@ -41,7 +41,7 @@ executable("testing") { deps = [ "$dart_src/runtime:libdart_jit", - "$dart_src/runtime/bin:dart_io_api", + "$dart_src/runtime/bin:common_embedder_dart_io", "$dart_src/runtime/bin:elf_loader", "//flutter/assets", "//flutter/common", From 0da2fc3d72ba3175613010de1744726b7d95526a Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Wed, 11 Feb 2026 16:35:46 -0800 Subject: [PATCH 36/51] chore: bump dart sdk to 896b3245b1a503560adc6ba8487f6ec42a0716a9 --- DEPS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPS b/DEPS index cddae06f064c2..bea045b501dc7 100644 --- a/DEPS +++ b/DEPS @@ -16,7 +16,7 @@ vars = { 'skia_git': 'https://skia.googlesource.com', 'llvm_git': 'https://llvm.googlesource.com', 'skia_revision': '837be28dd21818d2169e1e6789e89e771a55f01d', - "dart_sdk_revision": "b8847b5a781cfacf7edd199482278a51aa36cf5e", + "dart_sdk_revision": "896b3245b1a503560adc6ba8487f6ec42a0716a9", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", "updater_rev": "8691c8f60e69f8eb1f35361f4a22e8c9a7fdf93c", From 01760795bb3c0c62694cdfe4c9bb85efde3a3842 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Wed, 11 Feb 2026 16:52:54 -0800 Subject: [PATCH 37/51] chore: bump dart sdk to 4e44af9d1888a6d7ee3cd6ffe1c52b5e97881360 --- DEPS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPS b/DEPS index bea045b501dc7..9a7a161c608bb 100644 --- a/DEPS +++ b/DEPS @@ -16,7 +16,7 @@ vars = { 'skia_git': 'https://skia.googlesource.com', 'llvm_git': 'https://llvm.googlesource.com', 'skia_revision': '837be28dd21818d2169e1e6789e89e771a55f01d', - "dart_sdk_revision": "896b3245b1a503560adc6ba8487f6ec42a0716a9", + "dart_sdk_revision": "4e44af9d1888a6d7ee3cd6ffe1c52b5e97881360", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", "updater_rev": "8691c8f60e69f8eb1f35361f4a22e8c9a7fdf93c", From 511f6791a013df1c693819412176d469350b5118 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Sun, 22 Feb 2026 23:52:29 -0800 Subject: [PATCH 38/51] chore: bump dart sdk version to a5f0d8808753f5c4990a63e7ee5e93dc2bd978e9 --- DEPS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPS b/DEPS index 9a7a161c608bb..30c1f85130a87 100644 --- a/DEPS +++ b/DEPS @@ -16,7 +16,7 @@ vars = { 'skia_git': 'https://skia.googlesource.com', 'llvm_git': 'https://llvm.googlesource.com', 'skia_revision': '837be28dd21818d2169e1e6789e89e771a55f01d', - "dart_sdk_revision": "4e44af9d1888a6d7ee3cd6ffe1c52b5e97881360", + "dart_sdk_revision": "a5f0d8808753f5c4990a63e7ee5e93dc2bd978e9", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", "updater_rev": "8691c8f60e69f8eb1f35361f4a22e8c9a7fdf93c", From 558c0bb287e4275fec0bb216160e0e5755e120f7 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Tue, 24 Feb 2026 18:58:09 -0800 Subject: [PATCH 39/51] chore: bump dart sdk version to fc0de217def061c060a0dd6e42ff98428b18e52e --- DEPS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPS b/DEPS index 30c1f85130a87..de50c5c822f23 100644 --- a/DEPS +++ b/DEPS @@ -16,7 +16,7 @@ vars = { 'skia_git': 'https://skia.googlesource.com', 'llvm_git': 'https://llvm.googlesource.com', 'skia_revision': '837be28dd21818d2169e1e6789e89e771a55f01d', - "dart_sdk_revision": "a5f0d8808753f5c4990a63e7ee5e93dc2bd978e9", + "dart_sdk_revision": "fc0de217def061c060a0dd6e42ff98428b18e52e", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", "updater_rev": "8691c8f60e69f8eb1f35361f4a22e8c9a7fdf93c", From 6fc33abb3acf37bbc4e607d265d5d245acd85f08 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Wed, 25 Feb 2026 02:40:07 -0800 Subject: [PATCH 40/51] fix: format shard_runner config.dart --- shorebird/ci/shard_runner/lib/config.dart | 68 ++++++++++++++--------- 1 file changed, 42 insertions(+), 26 deletions(-) diff --git a/shorebird/ci/shard_runner/lib/config.dart b/shorebird/ci/shard_runner/lib/config.dart index 7cbcd01e681ff..29448199d1797 100644 --- a/shorebird/ci/shard_runner/lib/config.dart +++ b/shorebird/ci/shard_runner/lib/config.dart @@ -24,7 +24,8 @@ class PlatformConfig { final ShardDef? shard = shards[name]; if (shard == null) { throw ArgumentError( - 'Unknown shard: $name. Available: ${shards.keys.join(', ')}'); + 'Unknown shard: $name. Available: ${shards.keys.join(', ')}', + ); } return shard; } @@ -44,17 +45,19 @@ class PlatformConfig { /// Definition of a single build shard. @immutable class ShardDef { - ShardDef( - {required this.steps, - this.composeInput, - this.artifacts = const []}); + ShardDef({ + required this.steps, + this.composeInput, + this.artifacts = const [], + }); factory ShardDef.fromJson(Map json) { final List steps = (json['steps'] as List) .map((s) => BuildStep.fromJson(s as Map)) .toList(); - final List artifacts = (json['artifacts'] as List?) + final List artifacts = + (json['artifacts'] as List?) ?.map((a) => ArtifactDef.fromJson(a as Map)) .toList() ?? []; @@ -157,9 +160,7 @@ class RustStep implements BuildStep { RustStep({required this.targets}); factory RustStep.fromJson(Map json) { - return RustStep( - targets: (json['targets'] as List).cast(), - ); + return RustStep(targets: (json['targets'] as List).cast()); } final List targets; @@ -189,15 +190,14 @@ Future _runGn(String engineSrc, List args, String outDir) async { } Future _runNinja( - String engineSrc, String outDir, List targets) async { + String engineSrc, + String outDir, + List targets, +) async { print('[Ninja] Building ${targets.join(' ')} in out/$outDir'); await runChecked( 'ninja', - [ - '-C', - p.join(engineSrc, 'out', outDir), - ...targets, - ], + ['-C', p.join(engineSrc, 'out', outDir), ...targets], workingDirectory: engineSrc, description: 'Ninja ($outDir)', ); @@ -205,14 +205,21 @@ Future _runNinja( } Future _runRust(String engineSrc, List targets) async { - final String updaterPath = - p.join(engineSrc, 'flutter', 'third_party', 'updater', 'library'); + final String updaterPath = p.join( + engineSrc, + 'flutter', + 'third_party', + 'updater', + 'library', + ); // Separate Android and non-Android targets - final List androidTargets = - targets.where((String t) => t.contains('android')).toList(); - final List otherTargets = - targets.where((String t) => !t.contains('android')).toList(); + final List androidTargets = targets + .where((String t) => t.contains('android')) + .toList(); + final List otherTargets = targets + .where((String t) => !t.contains('android')) + .toList(); // Build all Android targets together with cargo-ndk if (androidTargets.isNotEmpty) { @@ -227,17 +234,26 @@ Future _runRust(String engineSrc, List targets) async { // The "unmodified" CIPD package keeps the NDK at the standard Android // SDK path: android_tools/sdk/ndk/. final Directory ndkParent = Directory( - p.join(engineSrc, 'flutter', 'third_party', 'android_tools', 'sdk', 'ndk'), + p.join( + engineSrc, + 'flutter', + 'third_party', + 'android_tools', + 'sdk', + 'ndk', + ), ); - final String ndkHome = ndkParent.listSync().whereType().first.path; + final String ndkHome = ndkParent + .listSync() + .whereType() + .first + .path; await runChecked( 'cargo', args, workingDirectory: updaterPath, - environment: { - 'ANDROID_NDK_HOME': ndkHome, - }, + environment: {'ANDROID_NDK_HOME': ndkHome}, description: 'Cargo ndk (${androidTargets.join(', ')})', ); } From 85009ad6ecb058715c24af4763f1bdb595913a19 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Mon, 2 Mar 2026 15:12:00 -0800 Subject: [PATCH 41/51] chore: bump dart-sdk to 12401cce210d164318a7becd9742abfe8391538f --- DEPS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPS b/DEPS index de50c5c822f23..807b551a7f916 100644 --- a/DEPS +++ b/DEPS @@ -16,7 +16,7 @@ vars = { 'skia_git': 'https://skia.googlesource.com', 'llvm_git': 'https://llvm.googlesource.com', 'skia_revision': '837be28dd21818d2169e1e6789e89e771a55f01d', - "dart_sdk_revision": "fc0de217def061c060a0dd6e42ff98428b18e52e", + "dart_sdk_revision": "12401cce210d164318a7becd9742abfe8391538f", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", "updater_rev": "8691c8f60e69f8eb1f35361f4a22e8c9a7fdf93c", From cd234ef8ece6e82da6bec9b9d811e818e8d8373a Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Wed, 4 Mar 2026 20:39:16 -0800 Subject: [PATCH 42/51] chore: bump engine version to 85009ad6ecb058715c24af4763f1bdb595913a19 --- bin/internal/engine.version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/internal/engine.version b/bin/internal/engine.version index fcaec1b3622e3..1bd1b99310421 100644 --- a/bin/internal/engine.version +++ b/bin/internal/engine.version @@ -1 +1 @@ -e4b8dca3f1b4ede4c30371002441c88c12187ed6 +85009ad6ecb058715c24af4763f1bdb595913a19 From cf75cce899a0a7217b56d24b8b6125aa084c7346 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Wed, 11 Mar 2026 19:04:24 -0700 Subject: [PATCH 43/51] chore: update dart-sdk to 67f30e6191fc5d7c924d6abe0435a0104aa0d2e2 --- DEPS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPS b/DEPS index 807b551a7f916..c7ba029f88709 100644 --- a/DEPS +++ b/DEPS @@ -16,7 +16,7 @@ vars = { 'skia_git': 'https://skia.googlesource.com', 'llvm_git': 'https://llvm.googlesource.com', 'skia_revision': '837be28dd21818d2169e1e6789e89e771a55f01d', - "dart_sdk_revision": "12401cce210d164318a7becd9742abfe8391538f", + "dart_sdk_revision": "67f30e6191fc5d7c924d6abe0435a0104aa0d2e2", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", "updater_rev": "8691c8f60e69f8eb1f35361f4a22e8c9a7fdf93c", From 460658b4883d1cf53d98c58ad6f6a38a4e21694d Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Wed, 11 Mar 2026 22:54:35 -0700 Subject: [PATCH 44/51] chore: bump engine to cf75cce899a0a7217b56d24b8b6125aa084c7346 --- bin/internal/engine.version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/internal/engine.version b/bin/internal/engine.version index 1bd1b99310421..d47a7ed22873c 100644 --- a/bin/internal/engine.version +++ b/bin/internal/engine.version @@ -1 +1 @@ -85009ad6ecb058715c24af4763f1bdb595913a19 +cf75cce899a0a7217b56d24b8b6125aa084c7346 From ad4602afd68fd52e0370141ad798ce0831348375 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Wed, 11 Mar 2026 23:02:48 -0700 Subject: [PATCH 45/51] fix: format shard_runner config.dart for stable SDK formatter --- shorebird/ci/shard_runner/lib/config.dart | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/shorebird/ci/shard_runner/lib/config.dart b/shorebird/ci/shard_runner/lib/config.dart index 29448199d1797..9fecdabe1cfd2 100644 --- a/shorebird/ci/shard_runner/lib/config.dart +++ b/shorebird/ci/shard_runner/lib/config.dart @@ -56,8 +56,7 @@ class ShardDef { .map((s) => BuildStep.fromJson(s as Map)) .toList(); - final List artifacts = - (json['artifacts'] as List?) + final List artifacts = (json['artifacts'] as List?) ?.map((a) => ArtifactDef.fromJson(a as Map)) .toList() ?? []; @@ -214,12 +213,10 @@ Future _runRust(String engineSrc, List targets) async { ); // Separate Android and non-Android targets - final List androidTargets = targets - .where((String t) => t.contains('android')) - .toList(); - final List otherTargets = targets - .where((String t) => !t.contains('android')) - .toList(); + final List androidTargets = + targets.where((String t) => t.contains('android')).toList(); + final List otherTargets = + targets.where((String t) => !t.contains('android')).toList(); // Build all Android targets together with cargo-ndk if (androidTargets.isNotEmpty) { @@ -243,11 +240,8 @@ Future _runRust(String engineSrc, List targets) async { 'ndk', ), ); - final String ndkHome = ndkParent - .listSync() - .whereType() - .first - .path; + final String ndkHome = + ndkParent.listSync().whereType().first.path; await runChecked( 'cargo', From 6ea12f228d602b669129431de1ea8a798079de5b Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 13 Mar 2026 11:13:46 -0700 Subject: [PATCH 46/51] fix: Update iOS dSYM filename in artifact manifest (#113) The manifest template still referenced Flutter.dSYM.zip but mac_upload.sh uploads Flutter.framework.dSYM.zip (the new name as of Flutter 3.27.0). Update the template, generate script, and test to match what's actually uploaded. Relates to https://github.com/shorebirdtech/shorebird/issues/3035 --- shorebird/ci/artifacts_manifest.template.yaml | 2 +- shorebird/ci/internal/generate_manifest.sh | 2 +- shorebird/ci/shard_runner/test/manifest_test.dart | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shorebird/ci/artifacts_manifest.template.yaml b/shorebird/ci/artifacts_manifest.template.yaml index 4b5c8a7f0809d..33223fced2de8 100644 --- a/shorebird/ci/artifacts_manifest.template.yaml +++ b/shorebird/ci/artifacts_manifest.template.yaml @@ -49,7 +49,7 @@ artifact_overrides: # iOS release artifacts - flutter_infra_release/flutter/$engine/ios-release/artifacts.zip - - flutter_infra_release/flutter/$engine/ios-release/Flutter.dSYM.zip + - flutter_infra_release/flutter/$engine/ios-release/Flutter.framework.dSYM.zip # Linux release artifacts - flutter_infra_release/flutter/$engine/linux-x64/artifacts.zip diff --git a/shorebird/ci/internal/generate_manifest.sh b/shorebird/ci/internal/generate_manifest.sh index 4c95de6baf098..ee9ef7ddfdf67 100755 --- a/shorebird/ci/internal/generate_manifest.sh +++ b/shorebird/ci/internal/generate_manifest.sh @@ -76,7 +76,7 @@ artifact_overrides: # iOS release artifacts # Includes unified Flutter.framework for device and simulator (debug) - flutter_infra_release/flutter/\$engine/ios-release/artifacts.zip - - flutter_infra_release/flutter/\$engine/ios-release/Flutter.dSYM.zip + - flutter_infra_release/flutter/\$engine/ios-release/Flutter.framework.dSYM.zip # Linux release artifacts - flutter_infra_release/flutter/\$engine/linux-x64/artifacts.zip diff --git a/shorebird/ci/shard_runner/test/manifest_test.dart b/shorebird/ci/shard_runner/test/manifest_test.dart index 1e8e930eb2458..a5606ae9b2f54 100644 --- a/shorebird/ci/shard_runner/test/manifest_test.dart +++ b/shorebird/ci/shard_runner/test/manifest_test.dart @@ -123,7 +123,7 @@ void main() { expect( manifest, contains( - r'flutter_infra_release/flutter/$engine/ios-release/Flutter.dSYM.zip')); + r'flutter_infra_release/flutter/$engine/ios-release/Flutter.framework.dSYM.zip')); }); test('includes Linux release artifacts', () { From 9899da7fccd15f22e047bd2a87e9ed6d6f09daeb Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Fri, 13 Mar 2026 12:59:39 -0700 Subject: [PATCH 47/51] chore: bump dart-sdk to 912ee5b7a1b5c7a41079ea55888031ebf9381ddf --- DEPS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPS b/DEPS index c7ba029f88709..794c24a84375c 100644 --- a/DEPS +++ b/DEPS @@ -16,7 +16,7 @@ vars = { 'skia_git': 'https://skia.googlesource.com', 'llvm_git': 'https://llvm.googlesource.com', 'skia_revision': '837be28dd21818d2169e1e6789e89e771a55f01d', - "dart_sdk_revision": "67f30e6191fc5d7c924d6abe0435a0104aa0d2e2", + "dart_sdk_revision": "912ee5b7a1b5c7a41079ea55888031ebf9381ddf", "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", "updater_git": "https://github.com/shorebirdtech/updater.git", "updater_rev": "8691c8f60e69f8eb1f35361f4a22e8c9a7fdf93c", From 569450f3ff908f0df75af06d810ae3259b1f77b2 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Fri, 13 Mar 2026 15:48:55 -0700 Subject: [PATCH 48/51] chore: bump engine to 9899da7fccd15f22e047bd2a87e9ed6d6f09daeb --- bin/internal/engine.version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/internal/engine.version b/bin/internal/engine.version index d47a7ed22873c..3ce4086c4a81c 100644 --- a/bin/internal/engine.version +++ b/bin/internal/engine.version @@ -1 +1 @@ -cf75cce899a0a7217b56d24b8b6125aa084c7346 +9899da7fccd15f22e047bd2a87e9ed6d6f09daeb From ccd7e81763f867eec7283164c4dca21c8f8898e0 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Fri, 13 Mar 2026 22:55:38 -0700 Subject: [PATCH 49/51] feat: Add --trace flag to flutter build apk for build profiling Adds a --trace option that produces a Chrome Trace Event Format JSON file showing where time is spent across all build layers (flutter tool, Gradle, flutter assemble targets). The output can be viewed in Perfetto at https://ui.perfetto.dev. --- .../gradle/src/main/kotlin/FlutterPlugin.kt | 3 + .../src/main/kotlin/tasks/BaseFlutterTask.kt | 4 + .../kotlin/tasks/BaseFlutterTaskHelper.kt | 3 + .../flutter_tools/lib/src/android/gradle.dart | 73 ++++++ .../flutter_tools/lib/src/build_info.dart | 7 + .../lib/src/build_system/build_system.dart | 6 + .../lib/src/build_system/build_trace.dart | 102 ++++++++ .../lib/src/commands/assemble.dart | 34 +++ .../lib/src/commands/build_apk.dart | 1 + .../lib/src/runner/flutter_command.dart | 18 ++ .../hermetic/assemble_test.dart | 37 +++ .../build_system/build_trace_test.dart | 229 ++++++++++++++++++ 12 files changed, 517 insertions(+) create mode 100644 packages/flutter_tools/lib/src/build_system/build_trace.dart create mode 100644 packages/flutter_tools/test/general.shard/build_system/build_trace_test.dart diff --git a/packages/flutter_tools/gradle/src/main/kotlin/FlutterPlugin.kt b/packages/flutter_tools/gradle/src/main/kotlin/FlutterPlugin.kt index 9dc036df7b154..c640ed6236fc3 100644 --- a/packages/flutter_tools/gradle/src/main/kotlin/FlutterPlugin.kt +++ b/packages/flutter_tools/gradle/src/main/kotlin/FlutterPlugin.kt @@ -597,6 +597,8 @@ class FlutterPlugin : Plugin { val dartDefinesValue: String? = project.findProperty("dart-defines")?.toString() val performanceMeasurementFileValue: String? = project.findProperty("performance-measurement-file")?.toString() + val traceFileValue: String? = + project.findProperty("trace-file")?.toString() val codeSizeDirectoryValue: String? = project.findProperty("code-size-directory")?.toString() val deferredComponentsValue: Boolean = @@ -694,6 +696,7 @@ class FlutterPlugin : Plugin { dartObfuscation = dartObfuscationValue dartDefines = dartDefinesValue performanceMeasurementFile = performanceMeasurementFileValue + traceFile = traceFileValue codeSizeDirectory = codeSizeDirectoryValue deferredComponents = deferredComponentsValue validateDeferredComponents = validateDeferredComponentsValue diff --git a/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTask.kt b/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTask.kt index 1858a139d0e96..dc011b7ecf22d 100644 --- a/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTask.kt +++ b/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTask.kt @@ -111,6 +111,10 @@ open class BaseFlutterTask : DefaultTask() { @Input var performanceMeasurementFile: String? = null + @Optional + @Input + var traceFile: String? = null + @Optional @Input var deferredComponents: Boolean? = null diff --git a/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTaskHelper.kt b/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTaskHelper.kt index 2d16640d324fd..7a5da3346db54 100644 --- a/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTaskHelper.kt +++ b/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTaskHelper.kt @@ -110,6 +110,9 @@ object BaseFlutterTaskHelper { baseFlutterTask.performanceMeasurementFile?.let { args("--performance-measurement-file=$it") } + baseFlutterTask.traceFile?.let { + args("--trace-file=$it") + } args("-dTargetFile=${baseFlutterTask.targetPath}") args("-dTargetPlatform=android") args("-dBuildMode=${baseFlutterTask.buildMode}") diff --git a/packages/flutter_tools/lib/src/android/gradle.dart b/packages/flutter_tools/lib/src/android/gradle.dart index b132b4cbe530a..7938e9edc82d5 100644 --- a/packages/flutter_tools/lib/src/android/gradle.dart +++ b/packages/flutter_tools/lib/src/android/gradle.dart @@ -24,6 +24,7 @@ import '../base/project_migrator.dart'; import '../base/terminal.dart'; import '../base/utils.dart'; import '../build_info.dart'; +import '../build_system/build_trace.dart'; import '../cache.dart'; import '../convert.dart'; import '../flutter_manifest.dart'; @@ -480,6 +481,11 @@ class AndroidGradleBuilder implements AndroidBuilder { } // Assembly work starts here. + final int buildStartMicros = DateTime.now().microsecondsSinceEpoch; + final BuildTracer? tracer = androidBuildInfo.buildInfo.traceFilePath != null + ? BuildTracer() + : null; + final BuildInfo buildInfo = androidBuildInfo.buildInfo; final String assembleTask = isBuildingBundle ? getBundleTaskFor(buildInfo) @@ -556,6 +562,20 @@ class AndroidGradleBuilder implements AndroidBuilder { } } options.addAll(androidBuildInfo.buildInfo.toGradleConfig()); + // Pass trace file path as a Gradle property for flutter assemble. + // We use an intermediate file that gradle.dart will merge into the final trace. + final String? assembleTraceFilePath; + if (buildInfo.traceFilePath != null) { + assembleTraceFilePath = _fileSystem.path.join( + project.android.buildDirectory.path, + 'intermediates', + 'flutter', + 'flutter_assemble_trace.json', + ); + options.add('-Ptrace-file=$assembleTraceFilePath'); + } else { + assembleTraceFilePath = null; + } if (buildInfo.fileSystemRoots.isNotEmpty) { options.add('-Pfilesystem-roots=${buildInfo.fileSystemRoots.join('|')}'); } @@ -565,10 +585,21 @@ class AndroidGradleBuilder implements AndroidBuilder { if (androidBuildInfo.splitPerAbi) { options.add('-Psplit-per-abi=true'); } + final int preGradleEndMicros = DateTime.now().microsecondsSinceEpoch; + tracer?.addCompleteEvent( + name: 'pre-gradle setup', + cat: 'flutter', + tid: 1, + startMicros: buildStartMicros, + endMicros: preGradleEndMicros, + ); + late Stopwatch sw; + late int gradleStartMicros; final int exitCode = await _runGradleTask( assembleTask, preRunTask: () { + gradleStartMicros = DateTime.now().microsecondsSinceEpoch; sw = Stopwatch()..start(); }, postRunTask: () { @@ -588,6 +619,23 @@ class AndroidGradleBuilder implements AndroidBuilder { gradleExecutablePath: gradleExecutablePath, ); + // Record Gradle span and merge assemble trace if tracing is enabled. + if (tracer != null) { + final int gradleEndMicros = DateTime.now().microsecondsSinceEpoch; + tracer.addCompleteEvent( + name: 'gradle $assembleTask', + cat: 'gradle', + tid: 2, + startMicros: gradleStartMicros, + endMicros: gradleEndMicros, + ); + // Merge the assemble trace file that flutter assemble wrote. + if (assembleTraceFilePath != null) { + final File assembleTraceFile = _fileSystem.file(assembleTraceFilePath); + tracer.mergeEventsFromFile(assembleTraceFile); + } + } + if (exitCode != 0) { throwToolExit( 'Gradle task $assembleTask failed with exit code $exitCode', @@ -595,6 +643,31 @@ class AndroidGradleBuilder implements AndroidBuilder { ); } + // Write the build trace if tracing is enabled. + if (tracer != null && buildInfo.traceFilePath != null) { + final int postGradleEndMicros = DateTime.now().microsecondsSinceEpoch; + tracer.addCompleteEvent( + name: 'post-gradle processing', + cat: 'flutter', + tid: 1, + startMicros: gradleStartMicros + sw.elapsedMicroseconds, + endMicros: postGradleEndMicros, + ); + tracer.addCompleteEvent( + name: isBuildingBundle ? 'flutter build appbundle' : 'flutter build apk', + cat: 'flutter', + tid: 1, + startMicros: buildStartMicros, + endMicros: postGradleEndMicros, + ); + final File traceFile = _fileSystem.file(buildInfo.traceFilePath); + tracer.writeToFile(traceFile); + _logger.printStatus( + 'Build trace written to ${buildInfo.traceFilePath}. ' + 'View at https://ui.perfetto.dev', + ); + } + if (isBuildingBundle) { final File bundleFile = findBundleFile(project, buildInfo, _logger, _analytics); diff --git a/packages/flutter_tools/lib/src/build_info.dart b/packages/flutter_tools/lib/src/build_info.dart index 6867be4c62a33..79dda712a19e0 100644 --- a/packages/flutter_tools/lib/src/build_info.dart +++ b/packages/flutter_tools/lib/src/build_info.dart @@ -44,6 +44,7 @@ class BuildInfo { List? dartExperiments, required this.treeShakeIcons, this.performanceMeasurementFile, + this.traceFilePath, required this.packageConfigPath, this.codeSizeDirectory, this.androidGradleDaemon = true, @@ -135,6 +136,10 @@ class BuildInfo { /// rerun tasks. final String? performanceMeasurementFile; + /// The path to a file where a Chrome Trace Event Format JSON trace will be + /// written for build profiling. + final String? traceFilePath; + /// If provided, an output directory where one or more v8-style heap snapshots /// will be written for code size profiling. final String? codeSizeDirectory; @@ -354,6 +359,8 @@ class BuildInfo { 'TREE_SHAKE_ICONS': treeShakeIcons.toString(), if (performanceMeasurementFile != null) 'PERFORMANCE_MEASUREMENT_FILE': performanceMeasurementFile!, + if (traceFilePath != null) + 'TRACE_FILE': traceFilePath!, 'PACKAGE_CONFIG': packageConfigPath, 'CODE_SIZE_DIRECTORY': ?codeSizeDirectory, 'FLAVOR': ?flavor, diff --git a/packages/flutter_tools/lib/src/build_system/build_system.dart b/packages/flutter_tools/lib/src/build_system/build_system.dart index 7e26475c408a8..82f406f82c099 100644 --- a/packages/flutter_tools/lib/src/build_system/build_system.dart +++ b/packages/flutter_tools/lib/src/build_system/build_system.dart @@ -882,6 +882,7 @@ class _BuildInstance { Future _invokeInternal(Node node) async { final PoolResource resource = await resourcePool.request(); + final int startTimeMicroseconds = DateTime.now().microsecondsSinceEpoch; final stopwatch = Stopwatch()..start(); var succeeded = true; var skipped = false; @@ -982,6 +983,7 @@ class _BuildInstance { skipped: skipped, succeeded: succeeded, analyticsName: node.target.analyticsName, + startTimeMicroseconds: startTimeMicroseconds, ); } return succeeded; @@ -1011,6 +1013,7 @@ class PerformanceMeasurement { required this.skipped, required this.succeeded, required this.analyticsName, + required this.startTimeMicroseconds, }); final int elapsedMilliseconds; @@ -1018,6 +1021,9 @@ class PerformanceMeasurement { final bool skipped; final bool succeeded; final String analyticsName; + + /// Wall-clock start time in microseconds since epoch. + final int startTimeMicroseconds; } /// Check if there are any dependency cycles in the target. diff --git a/packages/flutter_tools/lib/src/build_system/build_trace.dart b/packages/flutter_tools/lib/src/build_system/build_trace.dart new file mode 100644 index 0000000000000..131dec4c06c24 --- /dev/null +++ b/packages/flutter_tools/lib/src/build_system/build_trace.dart @@ -0,0 +1,102 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import '../base/file_system.dart'; +import '../convert.dart'; + +/// A single event in a Chrome Trace Event Format trace. +class BuildTraceEvent { + BuildTraceEvent({ + required this.name, + required this.cat, + required this.ts, + required this.dur, + this.pid = 1, + required this.tid, + this.args, + }); + + factory BuildTraceEvent.fromJson(Map json) { + return BuildTraceEvent( + name: json['name']! as String, + cat: json['cat']! as String, + ts: json['ts']! as int, + dur: json['dur']! as int, + pid: json['pid'] as int? ?? 1, + tid: json['tid']! as int, + args: json['args'] as Map?, + ); + } + + final String name; + final String cat; + final int ts; + final int dur; + final int pid; + final int tid; + final Map? args; + + Map toJson() { + return { + 'ph': 'X', + 'name': name, + 'cat': cat, + 'ts': ts, + 'dur': dur, + 'pid': pid, + 'tid': tid, + if (args != null) 'args': args, + }; + } +} + +/// Collects [BuildTraceEvent]s and writes them as a Chrome Trace Event Format +/// JSON array. +class BuildTracer { + final List _events = []; + + /// Adds a complete event (`ph: "X"`) to the trace. + void addCompleteEvent({ + required String name, + required String cat, + required int tid, + required int startMicros, + required int endMicros, + Map? args, + }) { + _events.add(BuildTraceEvent( + name: name, + cat: cat, + tid: tid, + ts: startMicros, + dur: endMicros - startMicros, + args: args, + )); + } + + /// Reads a trace JSON file written by a subprocess and appends its events. + void mergeEventsFromFile(File file) { + if (!file.existsSync()) { + return; + } + final String contents = file.readAsStringSync(); + final jsonList = json.decode(contents) as List; + for (final item in jsonList) { + if (item is Map) { + _events.add(BuildTraceEvent.fromJson(item)); + } + } + } + + /// Writes all collected events as a JSON array to [file]. + void writeToFile(File file) { + if (!file.parent.existsSync()) { + file.parent.createSync(recursive: true); + } + final jsonList = >[ + for (final BuildTraceEvent event in _events) event.toJson(), + ]; + file.writeAsStringSync(json.encode(jsonList)); + } +} diff --git a/packages/flutter_tools/lib/src/commands/assemble.dart b/packages/flutter_tools/lib/src/commands/assemble.dart index 564aa971a1a20..90282345088f0 100644 --- a/packages/flutter_tools/lib/src/commands/assemble.dart +++ b/packages/flutter_tools/lib/src/commands/assemble.dart @@ -112,6 +112,10 @@ class AssembleCommand extends FlutterCommand { 'performance-measurement-file', help: 'Output individual target performance to a JSON file.', ); + argParser.addOption( + 'trace-file', + help: 'Output build trace in Chrome Trace Event Format JSON.', + ); argParser.addMultiOption( 'input', abbr: 'i', @@ -394,6 +398,10 @@ class AssembleCommand extends FlutterCommand { final File outFile = globals.fs.file(argumentResults['performance-measurement-file']); writePerformanceData(result.performance.values, outFile); } + if (argumentResults.wasParsed('trace-file')) { + final File outFile = globals.fs.file(argumentResults['trace-file']); + writeTraceData(result.performance.values, outFile); + } if (argumentResults.wasParsed('depfile')) { final File depfileFile = globals.fs.file(stringArg('depfile')); final depfile = Depfile(result.inputFiles, result.outputFiles); @@ -440,3 +448,29 @@ void writePerformanceData(Iterable measurements, File ou } outFile.writeAsStringSync(json.encode(jsonData)); } + +/// Output build trace data in Chrome Trace Event Format in [outFile]. +@visibleForTesting +void writeTraceData(Iterable measurements, File outFile) { + final events = >[ + for (final PerformanceMeasurement measurement in measurements) + { + 'ph': 'X', + 'name': measurement.analyticsName, + 'cat': 'assemble', + 'ts': measurement.startTimeMicroseconds, + 'dur': measurement.elapsedMilliseconds * 1000, + 'pid': 1, + 'tid': 3, + 'args': { + 'target': measurement.target, + 'skipped': measurement.skipped, + 'succeeded': measurement.succeeded, + }, + }, + ]; + if (!outFile.parent.existsSync()) { + outFile.parent.createSync(recursive: true); + } + outFile.writeAsStringSync(json.encode(events)); +} diff --git a/packages/flutter_tools/lib/src/commands/build_apk.dart b/packages/flutter_tools/lib/src/commands/build_apk.dart index d3f50300804ef..fd397ef109adb 100644 --- a/packages/flutter_tools/lib/src/commands/build_apk.dart +++ b/packages/flutter_tools/lib/src/commands/build_apk.dart @@ -31,6 +31,7 @@ class BuildApkCommand extends BuildSubCommand { usesExtraDartFlagOptions(verboseHelp: verboseHelp); addEnableExperimentation(hide: !verboseHelp); addBuildPerformanceFile(hide: !verboseHelp); + usesBuildTraceOption(hide: !verboseHelp); usesAnalyzeSizeFlag(); addAndroidSpecificBuildOptions(hide: !verboseHelp); addIgnoreDeprecationOption(); diff --git a/packages/flutter_tools/lib/src/runner/flutter_command.dart b/packages/flutter_tools/lib/src/runner/flutter_command.dart index 452ab9d2a4e42..b4fb994ecc270 100644 --- a/packages/flutter_tools/lib/src/runner/flutter_command.dart +++ b/packages/flutter_tools/lib/src/runner/flutter_command.dart @@ -131,6 +131,7 @@ abstract final class FlutterOptions { static const kDartDefineFromFileOption = 'dart-define-from-file'; static const kWebDefinesOption = 'web-define'; static const kPerformanceMeasurementFile = 'performance-measurement-file'; + static const kBuildTrace = 'trace'; static const kDeviceUser = 'device-user'; static const kDeviceTimeout = 'device-timeout'; static const kDeviceConnection = 'device-connection'; @@ -1048,6 +1049,17 @@ abstract class FlutterCommand extends Command { ); } + void usesBuildTraceOption({bool hide = false}) { + argParser.addOption( + FlutterOptions.kBuildTrace, + help: + 'Output a Chrome Trace Event Format JSON file for build profiling. ' + 'The resulting file can be viewed at https://ui.perfetto.dev.', + hide: hide, + valueHelp: 'path/to/trace.json', + ); + } + void addAndroidSpecificBuildOptions({bool hide = false}) { argParser.addFlag( FlutterOptions.kAndroidGradleDaemon, @@ -1426,6 +1438,11 @@ abstract class FlutterCommand extends Command { ? stringArg(FlutterOptions.kPerformanceMeasurementFile) : null; + final String? traceFilePath = + argParser.options.containsKey(FlutterOptions.kBuildTrace) + ? stringArg(FlutterOptions.kBuildTrace) + : null; + final Map defineConfigJsonMap = extractDartDefineConfigJsonMap(); final List dartDefines = extractDartDefines(defineConfigJsonMap: defineConfigJsonMap); @@ -1477,6 +1494,7 @@ abstract class FlutterCommand extends Command { dartDefines: dartDefines, dartExperiments: experiments, performanceMeasurementFile: performanceMeasurementFile, + traceFilePath: traceFilePath, packageConfigPath: packagesPath ?? packageConfigFile.path, codeSizeDirectory: codeSizeDirectory, androidGradleDaemon: androidGradleDaemon, diff --git a/packages/flutter_tools/test/commands.shard/hermetic/assemble_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/assemble_test.dart index e394c0c217381..51baea33c1218 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/assemble_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/assemble_test.dart @@ -364,6 +364,7 @@ void main() { elapsedMilliseconds: 123, skipped: false, succeeded: true, + startTimeMicroseconds: 0, ), }, ), @@ -502,6 +503,7 @@ void main() { skipped: false, succeeded: true, elapsedMilliseconds: 123, + startTimeMicroseconds: 0, ), ]; final FileSystem fileSystem = MemoryFileSystem.test(); @@ -545,6 +547,41 @@ void main() { await commandRunner.run(['--help' /* -- verbose omitted (verboseHelp: true) is set above */]); expect(testLogger.statusText, contains('assemble')); }); + + testWithoutContext('writeTraceData outputs Chrome Trace Event Format', () { + final measurements = [ + PerformanceMeasurement( + analyticsName: 'KernelSnapshot', + target: 'kernel_snapshot', + skipped: false, + succeeded: true, + elapsedMilliseconds: 500, + startTimeMicroseconds: 1000000, + ), + ]; + final FileSystem fileSystem = MemoryFileSystem.test(); + final outFile = fileSystem.currentDirectory.childDirectory('foo').childFile('trace.json'); + + writeTraceData(measurements, outFile); + + expect(outFile, exists); + final events = json.decode(outFile.readAsStringSync()) as List; + expect(events, hasLength(1)); + + final event = events.first! as Map; + expect(event['ph'], 'X'); + expect(event['name'], 'KernelSnapshot'); + expect(event['cat'], 'assemble'); + expect(event['ts'], 1000000); + expect(event['dur'], 500000); + expect(event['pid'], 1); + expect(event['tid'], 3); + expect(event['args'], { + 'target': 'kernel_snapshot', + 'skipped': false, + 'succeeded': true, + }); + }); } final class _StubCommand extends FlutterCommand { diff --git a/packages/flutter_tools/test/general.shard/build_system/build_trace_test.dart b/packages/flutter_tools/test/general.shard/build_system/build_trace_test.dart new file mode 100644 index 0000000000000..2916c7562f14e --- /dev/null +++ b/packages/flutter_tools/test/general.shard/build_system/build_trace_test.dart @@ -0,0 +1,229 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:file/memory.dart'; +import 'package:flutter_tools/src/base/file_system.dart'; +import 'package:flutter_tools/src/build_system/build_trace.dart'; +import 'package:flutter_tools/src/convert.dart'; + +import '../../src/common.dart'; + +void main() { + group('BuildTraceEvent', () { + testWithoutContext('toJson produces Chrome Trace Event Format', () { + final event = BuildTraceEvent( + name: 'test_target', + cat: 'assemble', + ts: 1000000, + dur: 500000, + pid: 1, + tid: 3, + args: {'key': 'value'}, + ); + + final result = event.toJson(); + + expect(result['ph'], 'X'); + expect(result['name'], 'test_target'); + expect(result['cat'], 'assemble'); + expect(result['ts'], 1000000); + expect(result['dur'], 500000); + expect(result['pid'], 1); + expect(result['tid'], 3); + expect(result['args'], {'key': 'value'}); + }); + + testWithoutContext('toJson omits args when null', () { + final event = BuildTraceEvent( + name: 'test', + cat: 'flutter', + ts: 0, + dur: 100, + tid: 1, + ); + + final result = event.toJson(); + + expect(result.containsKey('args'), isFalse); + }); + + testWithoutContext('fromJson round-trips correctly', () { + final original = { + 'ph': 'X', + 'name': 'test', + 'cat': 'flutter', + 'ts': 1000, + 'dur': 500, + 'pid': 1, + 'tid': 2, + 'args': {'foo': 'bar'}, + }; + + final event = BuildTraceEvent.fromJson(original); + final result = event.toJson(); + + expect(result['name'], 'test'); + expect(result['cat'], 'flutter'); + expect(result['ts'], 1000); + expect(result['dur'], 500); + expect(result['pid'], 1); + expect(result['tid'], 2); + }); + }); + + group('BuildTracer', () { + late FileSystem fileSystem; + + setUp(() { + fileSystem = MemoryFileSystem.test(); + }); + + testWithoutContext('addCompleteEvent adds event with correct duration', () { + final tracer = BuildTracer(); + + tracer.addCompleteEvent( + name: 'gradle build', + cat: 'gradle', + tid: 2, + startMicros: 1000, + endMicros: 5000, + ); + + final outFile = fileSystem.file('trace.json'); + tracer.writeToFile(outFile); + + final events = json.decode(outFile.readAsStringSync()) as List; + expect(events, hasLength(1)); + + final event = events.first! as Map; + expect(event['name'], 'gradle build'); + expect(event['cat'], 'gradle'); + expect(event['tid'], 2); + expect(event['ts'], 1000); + expect(event['dur'], 4000); + expect(event['ph'], 'X'); + expect(event['pid'], 1); + }); + + testWithoutContext('mergeEventsFromFile reads and appends events', () { + final tracer = BuildTracer(); + + // Write a trace file to merge. + final sourceFile = fileSystem.file('source_trace.json'); + sourceFile.writeAsStringSync(json.encode(>[ + { + 'ph': 'X', + 'name': 'KernelSnapshot', + 'cat': 'assemble', + 'ts': 2000, + 'dur': 1000, + 'pid': 1, + 'tid': 3, + }, + ])); + + tracer.addCompleteEvent( + name: 'gradle build', + cat: 'gradle', + tid: 2, + startMicros: 1000, + endMicros: 5000, + ); + + tracer.mergeEventsFromFile(sourceFile); + + final outFile = fileSystem.file('trace.json'); + tracer.writeToFile(outFile); + + final events = json.decode(outFile.readAsStringSync()) as List; + expect(events, hasLength(2)); + expect((events[0]! as Map)['name'], 'gradle build'); + expect((events[1]! as Map)['name'], 'KernelSnapshot'); + }); + + testWithoutContext('mergeEventsFromFile does nothing for non-existent file', () { + final tracer = BuildTracer(); + + tracer.addCompleteEvent( + name: 'test', + cat: 'flutter', + tid: 1, + startMicros: 0, + endMicros: 100, + ); + + // Should not throw. + tracer.mergeEventsFromFile(fileSystem.file('does_not_exist.json')); + + final outFile = fileSystem.file('trace.json'); + tracer.writeToFile(outFile); + + final events = json.decode(outFile.readAsStringSync()) as List; + expect(events, hasLength(1)); + }); + + testWithoutContext('writeToFile creates parent directories', () { + final tracer = BuildTracer(); + tracer.addCompleteEvent( + name: 'test', + cat: 'flutter', + tid: 1, + startMicros: 0, + endMicros: 100, + ); + + final outFile = fileSystem.file('/a/b/c/trace.json'); + tracer.writeToFile(outFile); + + expect(outFile.existsSync(), isTrue); + }); + + testWithoutContext('output is valid Chrome Trace Event Format', () { + final tracer = BuildTracer(); + + tracer.addCompleteEvent( + name: 'flutter build apk', + cat: 'flutter', + tid: 1, + startMicros: 0, + endMicros: 15000000, + ); + tracer.addCompleteEvent( + name: 'gradle assembleRelease', + cat: 'gradle', + tid: 2, + startMicros: 500000, + endMicros: 12500000, + ); + tracer.addCompleteEvent( + name: 'KernelSnapshot', + cat: 'assemble', + tid: 3, + startMicros: 2000000, + endMicros: 5000000, + args: {'skipped': false}, + ); + + final outFile = fileSystem.file('trace.json'); + tracer.writeToFile(outFile); + + // Verify it's a valid JSON array. + final events = json.decode(outFile.readAsStringSync()) as List; + expect(events, hasLength(3)); + + // Verify all required fields are present in each event. + for (final item in events) { + final event = item! as Map; + expect(event.containsKey('ph'), isTrue); + expect(event.containsKey('name'), isTrue); + expect(event.containsKey('cat'), isTrue); + expect(event.containsKey('ts'), isTrue); + expect(event.containsKey('dur'), isTrue); + expect(event.containsKey('pid'), isTrue); + expect(event.containsKey('tid'), isTrue); + expect(event['ph'], 'X'); + } + }); + }); +} From 4288514fe31fb4bc078e53022d6f06a0ed43444e Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Sat, 14 Mar 2026 00:16:24 -0700 Subject: [PATCH 50/51] Add comment about multi-variant trace file collision risk The intermediate trace file path is shared across all Gradle variants. This is safe today since flutter build apk only runs one variant per invocation, but would need per-variant paths if that changes. --- packages/flutter_tools/lib/src/android/gradle.dart | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/flutter_tools/lib/src/android/gradle.dart b/packages/flutter_tools/lib/src/android/gradle.dart index 7938e9edc82d5..8f4592c52582d 100644 --- a/packages/flutter_tools/lib/src/android/gradle.dart +++ b/packages/flutter_tools/lib/src/android/gradle.dart @@ -564,6 +564,12 @@ class AndroidGradleBuilder implements AndroidBuilder { options.addAll(androidBuildInfo.buildInfo.toGradleConfig()); // Pass trace file path as a Gradle property for flutter assemble. // We use an intermediate file that gradle.dart will merge into the final trace. + // NOTE: This path is passed as a project-level Gradle property, so it's + // shared across all variants. This is safe because `flutter build apk` + // only runs a single variant's FlutterTask per invocation. If we ever + // support tracing builds that run multiple variants simultaneously (e.g. + // multiple flavors in one Gradle invocation), this path would need to + // include the variant name to avoid collisions. final String? assembleTraceFilePath; if (buildInfo.traceFilePath != null) { assembleTraceFilePath = _fileSystem.path.join( From 8ea86fb5252b0cba56871e6881772f9a62df7a38 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Sat, 14 Mar 2026 00:23:13 -0700 Subject: [PATCH 51/51] Add iOS build tracing support - Add --trace option to flutter build ios / flutter build ipa - Pass TRACE_FILE through Xcode build settings to xcode_backend.dart - Instrument buildXcodeProject() in mac.dart with pre-xcode, xcode, and post-xcode spans - Merge flutter assemble trace events from intermediate file - Remove TRACE_FILE from toEnvironmentConfig() since both Android and iOS orchestrators compute intermediate paths directly --- packages/flutter_tools/bin/xcode_backend.dart | 7 ++ .../flutter_tools/lib/src/build_info.dart | 2 - .../lib/src/commands/build_ios.dart | 1 + packages/flutter_tools/lib/src/ios/mac.dart | 68 +++++++++++++++++++ 4 files changed, 76 insertions(+), 2 deletions(-) diff --git a/packages/flutter_tools/bin/xcode_backend.dart b/packages/flutter_tools/bin/xcode_backend.dart index e32980a357a4d..021f070b763c6 100644 --- a/packages/flutter_tools/bin/xcode_backend.dart +++ b/packages/flutter_tools/bin/xcode_backend.dart @@ -722,6 +722,13 @@ class Context { ); } + if (environment['TRACE_FILE'] != null && + environment['TRACE_FILE']!.isNotEmpty) { + flutterArgs.add( + '--trace-file=${environment['TRACE_FILE']}', + ); + } + if (environment['CODE_SIZE_DIRECTORY'] != null && environment['CODE_SIZE_DIRECTORY']!.isNotEmpty) { flutterArgs.add('-dCodeSizeDirectory=${environment['CODE_SIZE_DIRECTORY']}'); diff --git a/packages/flutter_tools/lib/src/build_info.dart b/packages/flutter_tools/lib/src/build_info.dart index 79dda712a19e0..05221cf95dfb7 100644 --- a/packages/flutter_tools/lib/src/build_info.dart +++ b/packages/flutter_tools/lib/src/build_info.dart @@ -359,8 +359,6 @@ class BuildInfo { 'TREE_SHAKE_ICONS': treeShakeIcons.toString(), if (performanceMeasurementFile != null) 'PERFORMANCE_MEASUREMENT_FILE': performanceMeasurementFile!, - if (traceFilePath != null) - 'TRACE_FILE': traceFilePath!, 'PACKAGE_CONFIG': packageConfigPath, 'CODE_SIZE_DIRECTORY': ?codeSizeDirectory, 'FLAVOR': ?flavor, diff --git a/packages/flutter_tools/lib/src/commands/build_ios.dart b/packages/flutter_tools/lib/src/commands/build_ios.dart index e783f793b9e7a..7c2700d77b417 100644 --- a/packages/flutter_tools/lib/src/commands/build_ios.dart +++ b/packages/flutter_tools/lib/src/commands/build_ios.dart @@ -907,6 +907,7 @@ abstract class _BuildIOSSubCommand extends BuildSubCommand { usesExtraDartFlagOptions(verboseHelp: verboseHelp); addEnableExperimentation(hide: !verboseHelp); addBuildPerformanceFile(hide: !verboseHelp); + usesBuildTraceOption(hide: !verboseHelp); usesAnalyzeSizeFlag(); argParser.addFlag( 'codesign', diff --git a/packages/flutter_tools/lib/src/ios/mac.dart b/packages/flutter_tools/lib/src/ios/mac.dart index 82e94b215eee1..f7390ca74abc0 100644 --- a/packages/flutter_tools/lib/src/ios/mac.dart +++ b/packages/flutter_tools/lib/src/ios/mac.dart @@ -18,6 +18,7 @@ import '../base/project_migrator.dart'; import '../base/utils.dart'; import '../base/version.dart'; import '../build_info.dart'; +import '../build_system/build_trace.dart'; import '../cache.dart'; import '../darwin/darwin.dart'; import '../device.dart'; @@ -353,6 +354,26 @@ Future buildXcodeProject({ return XcodeBuildResult(success: true); } + final int buildStartMicros = DateTime.now().microsecondsSinceEpoch; + final BuildTracer? tracer = buildInfo.traceFilePath != null + ? BuildTracer() + : null; + + // Pass trace file path as an intermediate location for flutter assemble. + // mac.dart will merge this into the final trace file. + final String? assembleTraceFilePath; + if (buildInfo.traceFilePath != null) { + assembleTraceFilePath = globals.fs.path.join( + globals.fs.currentDirectory.path, + getBuildDirectory(), + 'ios', + 'flutter_assemble_trace.json', + ); + buildCommands.add('TRACE_FILE=$assembleTraceFilePath'); + } else { + assembleTraceFilePath = null; + } + if (globals.logger.isVerbose) { // An environment variable to be passed to xcode_backend.sh determining // whether to echo back executed commands. @@ -531,6 +552,16 @@ Future buildXcodeProject({ ]); } + final int preXcodeEndMicros = DateTime.now().microsecondsSinceEpoch; + tracer?.addCompleteEvent( + name: 'pre-xcode setup', + cat: 'flutter', + tid: 1, + startMicros: buildStartMicros, + endMicros: preXcodeEndMicros, + ); + + final int xcodeStartMicros = DateTime.now().microsecondsSinceEpoch; final sw = Stopwatch()..start(); initialBuildStatus = globals.logger.startProgress('Running Xcode build...'); @@ -555,6 +586,43 @@ Future buildXcodeProject({ ), ); + // Record Xcode span and merge assemble trace if tracing is enabled. + if (tracer != null) { + final int xcodeEndMicros = DateTime.now().microsecondsSinceEpoch; + tracer.addCompleteEvent( + name: 'xcode ${xcodeBuildActionToString(buildAction)}', + cat: 'xcode', + tid: 2, + startMicros: xcodeStartMicros, + endMicros: xcodeEndMicros, + ); + // Merge the assemble trace file that flutter assemble wrote. + if (assembleTraceFilePath != null) { + final File assembleTraceFile = globals.fs.file(assembleTraceFilePath); + tracer.mergeEventsFromFile(assembleTraceFile); + } + tracer.addCompleteEvent( + name: 'post-xcode processing', + cat: 'flutter', + tid: 1, + startMicros: xcodeEndMicros, + endMicros: DateTime.now().microsecondsSinceEpoch, + ); + tracer.addCompleteEvent( + name: 'flutter build ios', + cat: 'flutter', + tid: 1, + startMicros: buildStartMicros, + endMicros: DateTime.now().microsecondsSinceEpoch, + ); + final File traceFile = globals.fs.file(buildInfo.traceFilePath); + tracer.writeToFile(traceFile); + globals.printStatus( + 'Build trace written to ${buildInfo.traceFilePath}. ' + 'View at https://ui.perfetto.dev', + ); + } + if (tempDir.existsSync()) { // Display additional warning and error message from xcresult bundle. final Directory resultBundle = tempDir.childDirectory(_kResultBundlePath);