diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000000..ade159ddabcb9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,43 @@ +name: ci + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + push: + branches: + - main + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + runs-on: ${{ matrix.os }} + + name: ๐Ÿงช Test + + env: + FLUTTER_STORAGE_BASE_URL: https://download.shorebird.dev + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + with: + # Fetch all branches and tags to ensure that Flutter can determine its version + fetch-depth: 0 + + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - name: ๐Ÿ“ฆ Install Dependencies + run: | + dart pub get -C ./dev/bots + dart pub get -C ./dev/tools + + - name: ๐Ÿงช Run Tests + run: dart ./dev/bots/test.dart diff --git a/.github/workflows/shorebird_ci.yml b/.github/workflows/shorebird_ci.yml new file mode 100644 index 0000000000000..83e1511d102de --- /dev/null +++ b/.github/workflows/shorebird_ci.yml @@ -0,0 +1,140 @@ +name: shorebird_ci + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + push: + branches: + - shorebird/dev + +jobs: + # NOTE: Windows is not included because shorebird_tests depends on + # flutter_flavorizr which requires Xcode. Additionally, flutter_tools + # tests on Windows would need additional setup work. + + flutter-tools-tests: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + + runs-on: ${{ matrix.os }} + + name: ๐Ÿ› ๏ธ Flutter Tools Tests (${{ matrix.os }}) + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - name: ๐Ÿฆ Run Flutter Tools Tests + run: ../../bin/flutter test test/general.shard + working-directory: packages/flutter_tools + + shard-runner-tests: + runs-on: ubuntu-latest + + name: ๐Ÿงฉ Shard Runner Tests + + defaults: + run: + working-directory: shorebird/ci/shard_runner + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - name: ๐Ÿ“ฆ Install Dependencies + run: dart pub get + + - name: ๐Ÿ” Check Formatting + run: dart format --output=none --set-exit-if-changed . + + - name: ๐Ÿ”Ž Analyze + run: dart analyze --fatal-infos + + - name: ๐Ÿงช Run Tests + run: dart test + + shorebird-android-tests: + # Android tests run on Ubuntu (faster runners, no macOS needed) + runs-on: ubuntu-latest + + name: ๐Ÿค– Shorebird Android Tests + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - uses: actions/setup-java@v4 + with: + distribution: "zulu" + java-version: "17" + + - name: ๐Ÿ“ฆ Cache Gradle + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: ๐Ÿค– Run Android Tests + run: dart test test/base_test.dart test/android_test.dart + working-directory: packages/shorebird_tests + + shorebird-ios-tests: + # iOS tests require macOS for Xcode + runs-on: macos-latest + + name: ๐ŸŽ Shorebird iOS + Android Smoke Tests + + steps: + - name: ๐Ÿ“š Git Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ๐ŸŽฏ Setup Dart + uses: dart-lang/setup-dart@v1 + + - uses: actions/setup-java@v4 + with: + distribution: "zulu" + java-version: "17" + + - name: ๐Ÿ“ฆ Cache Gradle + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: ๐ŸŽ Run iOS Tests + run: dart test test/base_test.dart test/ios_test.dart + working-directory: packages/shorebird_tests + + - name: ๐Ÿค– Run Android Smoke Test (macOS) + # Quick sanity check that Android builds work on macOS too + run: dart test test/android_test.dart --name "can build an apk" + working-directory: packages/shorebird_tests diff --git a/DEPS b/DEPS index 0139ff7ebda07..794c24a84375c 100644 --- a/DEPS +++ b/DEPS @@ -16,6 +16,10 @@ vars = { 'skia_git': 'https://skia.googlesource.com', 'llvm_git': 'https://llvm.googlesource.com', 'skia_revision': '837be28dd21818d2169e1e6789e89e771a55f01d', + "dart_sdk_revision": "912ee5b7a1b5c7a41079ea55888031ebf9381ddf", + "dart_sdk_git": "git@github.com:shorebirdtech/dart-sdk.git", + "updater_git": "https://github.com/shorebirdtech/updater.git", + "updater_rev": "8691c8f60e69f8eb1f35361f4a22e8c9a7fdf93c", # WARNING: DO NOT EDIT canvaskit_cipd_instance MANUALLY # See `lib/web_ui/README.md` for how to roll CanvasKit to a new version. @@ -294,7 +298,7 @@ deps = { # Var('flutter_git') + '/third_party/protobuf-gn' + '@' + Var('dart_protobuf_gn_rev'), 'engine/src/flutter/third_party/dart': - Var('dart_git') + '/sdk.git' + '@' + Var('dart_revision'), + Var('dart_sdk_git') + '@' + Var('dart_sdk_revision'), # WARNING: Unused Dart dependencies in the list below till "WARNING:" marker are removed automatically - see create_updated_flutter_deps.py. @@ -491,6 +495,9 @@ deps = { 'engine/src/flutter/third_party/ocmock': Var('flutter_git') + '/third_party/ocmock' + '@' + Var('ocmock_rev'), + 'engine/src/flutter/third_party/updater': + Var('updater_git') + '@' + Var('updater_rev'), + 'engine/src/flutter/third_party/libjpeg-turbo/src': Var('flutter_git') + '/third_party/libjpeg-turbo' + '@' + '0fb821f3b2e570b2783a94ccd9a2fb1f4916ae9f', diff --git a/bin/internal/engine.version b/bin/internal/engine.version index fcaec1b3622e3..3ce4086c4a81c 100644 --- a/bin/internal/engine.version +++ b/bin/internal/engine.version @@ -1 +1 @@ -e4b8dca3f1b4ede4c30371002441c88c12187ed6 +9899da7fccd15f22e047bd2a87e9ed6d6f09daeb diff --git a/bin/internal/update_dart_sdk.ps1 b/bin/internal/update_dart_sdk.ps1 index 318ef62add853..a588137834eb6 100644 --- a/bin/internal/update_dart_sdk.ps1 +++ b/bin/internal/update_dart_sdk.ps1 @@ -41,7 +41,7 @@ if ((Test-Path $engineStamp) -and ($engineVersion -eq (Get-Content $engineStamp) $dartSdkBaseUrl = $Env:FLUTTER_STORAGE_BASE_URL if (-not $dartSdkBaseUrl) { - $dartSdkBaseUrl = "https://storage.googleapis.com" + $dartSdkBaseUrl = "https://download.shorebird.dev" } if ($engineRealm) { $dartSdkBaseUrl = "$dartSdkBaseUrl/$engineRealm" diff --git a/bin/internal/update_dart_sdk.sh b/bin/internal/update_dart_sdk.sh index 9ab6151021a8b..6e3617a328342 100755 --- a/bin/internal/update_dart_sdk.sh +++ b/bin/internal/update_dart_sdk.sh @@ -123,7 +123,7 @@ if [ ! -f "$ENGINE_STAMP" ] || [ "$ENGINE_VERSION" != `cat "$ENGINE_STAMP"` ]; t FIND=find fi - DART_SDK_BASE_URL="${FLUTTER_STORAGE_BASE_URL:-https://storage.googleapis.com}${ENGINE_REALM:+/$ENGINE_REALM}" + DART_SDK_BASE_URL="${FLUTTER_STORAGE_BASE_URL:-https://download.shorebird.dev}${ENGINE_REALM:+/$ENGINE_REALM}" DART_SDK_URL="$DART_SDK_BASE_URL/flutter_infra_release/flutter/$ENGINE_VERSION/$DART_ZIP_NAME" # if the sdk path exists, copy it to a temporary location diff --git a/dev/bots/post_process_docs.dart b/dev/bots/post_process_docs.dart index aaf08e82e4913..5c40a60cd4f3b 100644 --- a/dev/bots/post_process_docs.dart +++ b/dev/bots/post_process_docs.dart @@ -38,7 +38,7 @@ Future postProcess() async { await runProcessWithValidations([ 'curl', '-L', - 'https://storage.googleapis.com/flutter_infra_release/flutter/$revision/api_docs.zip', + 'https://download.shorebird.dev/flutter_infra_release/flutter/$revision/api_docs.zip', '--output', zipDestination, '--fail', diff --git a/dev/bots/unpublish_package.dart b/dev/bots/unpublish_package.dart index 022111bd16ed9..03cc3a24d2fab 100644 --- a/dev/bots/unpublish_package.dart +++ b/dev/bots/unpublish_package.dart @@ -23,7 +23,7 @@ import 'package:process/process.dart'; const String gsBase = 'gs://flutter_infra_release'; const String releaseFolder = '/releases'; const String gsReleaseFolder = '$gsBase$releaseFolder'; -const String baseUrl = 'https://storage.googleapis.com/flutter_infra_release'; +const String baseUrl = 'https://download.shorebird.dev/flutter_infra_release'; /// Exception class for when a process fails to run, so we can catch /// it and provide something more readable than a stack trace. diff --git a/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle b/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle index 5368feb7ecde7..66364cdf4a190 100644 --- a/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle +++ b/dev/integration_tests/pure_android_host_apps/android_host_app_v2_embedding/settings.gradle @@ -7,7 +7,7 @@ dependencyResolutionManagement { repositories { google() mavenCentral() - def flutterStorageUrl = System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "https://storage.googleapis.com" + def flutterStorageUrl = System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "https://download.shorebird.dev" maven { url = uri("$flutterStorageUrl/download.flutter.io") } diff --git a/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts b/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts index f6d75bce11757..da25b49a46f7f 100644 --- a/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts +++ b/dev/integration_tests/pure_android_host_apps/host_app_kotlin_gradle_dsl/settings.gradle.kts @@ -16,7 +16,7 @@ dependencyResolutionManagement { repositories { google() mavenCentral() - val flutterStorageUrl = System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "https://storage.googleapis.com" + val flutterStorageUrl = System.getenv("FLUTTER_STORAGE_BASE_URL") ?: "https://download.shorebird.dev" maven("$flutterStorageUrl/download.flutter.io") } } diff --git a/dev/tools/create_api_docs.dart b/dev/tools/create_api_docs.dart index a98f16398b21f..4ece883e09df7 100644 --- a/dev/tools/create_api_docs.dart +++ b/dev/tools/create_api_docs.dart @@ -923,8 +923,8 @@ class PlatformDocGenerator { for (final String platform in kPlatformDocs.keys) { final String zipFile = kPlatformDocs[platform]!.zipName; - final url = - 'https://storage.googleapis.com/${realm}flutter_infra_release/flutter/$engineRevision/$zipFile'; + final String url = + 'https://download.shorebird.dev/${realm}flutter_infra_release/flutter/$engineRevision/$zipFile'; await _extractDocs(url, platform, kPlatformDocs[platform]!, outputDir); } } diff --git a/engine/src/build/config/compiler/BUILD.gn b/engine/src/build/config/compiler/BUILD.gn index 2d5e0e0ccbcef..7df82c918b172 100644 --- a/engine/src/build/config/compiler/BUILD.gn +++ b/engine/src/build/config/compiler/BUILD.gn @@ -443,7 +443,10 @@ config("compiler") { # Example PR: https://github.com/dart-lang/native/pull/1615 ldflags += [ "-Wl,--no-undefined", - "-Wl,--exclude-libs,ALL", + + # TODO: Terrible hack, but otherwise libupdater.a symbols can't + # be exported from libflutter.so, even when added to android_exports.lst. + # "-Wl,--exclude-libs,ALL", # Enable identical code folding to reduce size. "-Wl,--icf=all", @@ -646,6 +649,8 @@ config("runtime_library") { ldflags += [ "-Wl,--warn-shared-textrel" ] libs += [ + # Rust requires libunwind. + "unwind", "c", "dl", "m", @@ -660,6 +665,9 @@ config("runtime_library") { } else if (current_cpu == "x86") { current_android_cpu = "i686" } + # libunwind.a is located in the respective android cpu subdirectories. + # The clang version needs to match the version in the lib_dirs line above. + lib_dirs += [ "${android_toolchain_root}/lib/clang/19/lib/linux/${current_android_cpu}/" ] libs += [ "clang_rt.builtins-${current_android_cpu}-android" ] } diff --git a/engine/src/build/toolchain/win/BUILD.gn b/engine/src/build/toolchain/win/BUILD.gn index 45a98b1ecd64b..b5073fb22277b 100644 --- a/engine/src/build/toolchain/win/BUILD.gn +++ b/engine/src/build/toolchain/win/BUILD.gn @@ -204,8 +204,11 @@ template("msvc_toolchain") { expname = "${dllname}.exp" pdbname = "${dllname}.pdb" rspfile = "${dllname}.rsp" + # .def files are used to export symbols from the DLL. This arg will be + # removed by the python tool wrapper if the .def file doesn't exist. + deffile = "${dllname}.def" - link_command = "\"$python_path\" $tool_wrapper_path link-wrapper $env False link.exe /nologo /IMPLIB:$libname /DLL /OUT:$dllname /PDB:${dllname}.pdb @$rspfile" + link_command = "\"$python_path\" $tool_wrapper_path link-wrapper $env False link.exe /nologo /IMPLIB:$libname /DLL /OUT:$dllname /PDB:${dllname}.pdb /DEF:$deffile @$rspfile" # TODO(brettw) support manifests #manifest_command = "\"$python_path\" $tool_wrapper_path manifest-wrapper $env mt.exe -nologo -manifest $manifests -out:${dllname}.manifest" diff --git a/engine/src/build/toolchain/win/tool_wrapper.py b/engine/src/build/toolchain/win/tool_wrapper.py index b4fc8485ffa17..7866c6122d832 100644 --- a/engine/src/build/toolchain/win/tool_wrapper.py +++ b/engine/src/build/toolchain/win/tool_wrapper.py @@ -121,6 +121,17 @@ def ExecLinkWrapper(self, arch, use_separate_mspdbsrv, *args): self._UseSeparateMspdbsrv(env, args) if sys.platform == 'win32': args = list(args) # *args is a tuple by default, which is read-only. + + # Remove the /DEF arg if not provided. We would ideally be able to do this + # in build\toolchain\win\BUILD.gn, but there doesn't seem to be a way to + # conditionally add args to the command line based on whether a file exists + # or not, so we do it here instead. + def_arg_prefix = "/DEF:" + for arg in args: + if arg.startswith(def_arg_prefix): + def_file = arg[len(def_arg_prefix):] + if not os.path.exists(def_file): + args.remove(arg) args[0] = args[0].replace('/', '\\') # https://docs.python.org/2/library/subprocess.html: # "On Unix with shell=True [...] if args is a sequence, the first item diff --git a/engine/src/flutter/BUILD.gn b/engine/src/flutter/BUILD.gn index ec7b55d24f152..348f732c329ed 100644 --- a/engine/src/flutter/BUILD.gn +++ b/engine/src/flutter/BUILD.gn @@ -118,6 +118,9 @@ group("flutter") { # path_ops "//flutter/tools/path_ops", + + # Built alongside gen_snapshot arm64 targets. + "$dart_src/runtime/bin:analyze_snapshot", ] if (host_os == "linux" || host_os == "mac") { @@ -127,13 +130,6 @@ group("flutter") { ] } - if (host_os == "linux") { - public_deps += [ - # Built alongside gen_snapshot for 64 bit targets - "$dart_src/runtime/bin:analyze_snapshot", - ] - } - if (full_dart_sdk) { public_deps += [ "//flutter/web_sdk" ] } @@ -213,6 +209,7 @@ group("unittests") { "//flutter/runtime:no_dart_plugin_registrant_unittests", "//flutter/runtime:runtime_unittests", "//flutter/shell/common:shell_unittests", + "//flutter/shell/common/shorebird:shorebird_unittests", "//flutter/shell/geometry:geometry_unittests", "//flutter/shell/platform/embedder:embedder_a11y_unittests", "//flutter/shell/platform/embedder:embedder_proctable_unittests", @@ -344,3 +341,19 @@ if (host_os == "win") { outputs = [ "$root_build_dir/gen_snapshot/gen_snapshot.exe" ] } } + +# A top-level target for analyze_snapshot, modeled after the gen_snapshot +# target above. +if (host_os == "win") { + _analyze_snapshot_target = + "$dart_src/runtime/bin:analyze_snapshot($host_toolchain)" + + copy("analyze_snapshot") { + deps = [ _analyze_snapshot_target ] + + analyze_snapshot_out_dir = + get_label_info(_analyze_snapshot_target, "root_out_dir") + sources = [ "$analyze_snapshot_out_dir/analyze_snapshot.exe" ] + outputs = [ "$root_build_dir/analyze_snapshot/analyze_snapshot.exe" ] + } +} diff --git a/engine/src/flutter/build/archives/BUILD.gn b/engine/src/flutter/build/archives/BUILD.gn index 4f902010d21dc..55700433541db 100644 --- a/engine/src/flutter/build/archives/BUILD.gn +++ b/engine/src/flutter/build/archives/BUILD.gn @@ -45,6 +45,7 @@ generated_file("artifacts_entitlement_config") { if (build_engine_artifacts) { zip_bundle("artifacts") { deps = [ + "$dart_src/runtime/bin:analyze_snapshot", "$dart_src/runtime/bin:gen_snapshot", "//flutter/flutter_frontend_server:frontend_server", "//flutter/impeller/compiler:impellerc", @@ -142,6 +143,10 @@ if (build_engine_artifacts) { if (host_os == "mac") { deps += [ ":artifacts_entitlement_config" ] files += [ + { + source = "$root_out_dir/analyze_snapshot$exe" + destination = "analyze_snapshot$exe" + }, { source = "$target_gen_dir/entitlements.txt" destination = "entitlements.txt" @@ -320,14 +325,25 @@ if (is_mac) { } if (host_os == "win") { + # This rule archives both gen_snapshot *and* analyze_snapshot. The name is + # misleading. We (shorebird) have updated this rule to include + # analyze_snapshot but did not update the name because it is referenced + # elsewhere in the tooling. zip_bundle("archive_win_gen_snapshot") { - deps = [ "//flutter:gen_snapshot" ] + deps = [ + "//flutter:analyze_snapshot", + "//flutter:gen_snapshot", + ] output = "$full_target_platform_name-$flutter_runtime_mode/windows-x64.zip" files = [ { source = "$root_out_dir/gen_snapshot/gen_snapshot.exe" destination = "gen_snapshot.exe" }, + { + source = "$root_out_dir/analyze_snapshot/analyze_snapshot.exe" + destination = "analyze_snapshot.exe" + }, ] } } diff --git a/engine/src/flutter/build/dart/tools/dart_pkg.py b/engine/src/flutter/build/dart/tools/dart_pkg.py index c60e3e02431e4..5b3e0a8b5f3ee 100755 --- a/engine/src/flutter/build/dart/tools/dart_pkg.py +++ b/engine/src/flutter/build/dart/tools/dart_pkg.py @@ -163,7 +163,10 @@ def main(): for source in args.package_sources: relative_source = os.path.relpath(source, common_source_prefix) target = os.path.join(target_dir, relative_source) - copy(source, target) + try: + copy(source, target) + except shutil.SameFileError: + pass # Copy sdk-ext sources into pkg directory sdk_ext_dir = os.path.join(target_dir, 'sdk_ext') @@ -179,7 +182,10 @@ def main(): for source in args.sdk_ext_files: relative_source = os.path.relpath(source, common_source_prefix) target = os.path.join(sdk_ext_dir, relative_source) - copy(source, target) + try: + copy(source, target) + except shutil.SameFileError: + pass # Write stamp file. with open(args.stamp_file, 'w'): diff --git a/engine/src/flutter/build/zip_bundle.gni b/engine/src/flutter/build/zip_bundle.gni index 51e72df0ad854..707ab046d3637 100644 --- a/engine/src/flutter/build/zip_bundle.gni +++ b/engine/src/flutter/build/zip_bundle.gni @@ -55,7 +55,7 @@ template("zip_bundle") { license_path = rebase_path("//flutter/sky/packages/sky_engine/LICENSE", "//flutter") git_url = "https://github.com/flutter/engine/tree/$engine_version" - sky_engine_url = "https://storage.googleapis.com/flutter_infra_release/flutter/$engine_version/sky_engine.zip" + sky_engine_url = "https://download.shorebird.dev/flutter_infra_release/flutter/$engine_version/sky_engine.zip" outputs = [ license_readme ] contents = [ "# $target_name", diff --git a/engine/src/flutter/common/config.gni b/engine/src/flutter/common/config.gni index 318d305bd37fc..35ec1c7e058dc 100644 --- a/engine/src/flutter/common/config.gni +++ b/engine/src/flutter/common/config.gni @@ -67,6 +67,14 @@ if (slimpeller) { feature_defines_list += [ "SLIMPELLER=1" ] } +if (is_android || is_ios || is_linux || is_mac || is_win) { + feature_defines_list += [ "SHOREBIRD_PLATFORM_SUPPORTED=1" ] +} + +if (is_ios) { + feature_defines_list += [ "SHOREBIRD_USE_INTERPRETER=1" ] +} + if (is_ios || is_mac) { flutter_cflags_objc = [ "-Werror=overriding-method-mismatch", diff --git a/engine/src/flutter/impeller/toolkit/interop/README.md b/engine/src/flutter/impeller/toolkit/interop/README.md index 6e75a53a11775..279379217e143 100644 --- a/engine/src/flutter/impeller/toolkit/interop/README.md +++ b/engine/src/flutter/impeller/toolkit/interop/README.md @@ -27,7 +27,7 @@ A single-header C API for 2D graphics and text rendering. [Impeller](../../READM Users may plug in a custom toolchain into the Flutter Engine build system to build the `libimpeller.so` dynamic library. However, for the common platforms, the CI bots upload a tarball containing the library and headers. This URL for the SDK tarball for a particular platform can be constructed as follows: ```sh -https://storage.googleapis.com/flutter_infra_release/flutter/$FLUTTER_SHA/$PLATFORM_ARCH/impeller_sdk.zip +https://download.shorebird.dev/flutter_infra_release/flutter/$FLUTTER_SHA/$PLATFORM_ARCH/impeller_sdk.zip ``` The `$FLUTTER_SHA` is the Git hash in the [Flutter repository](https://github.com/flutter/flutter). The `$PLATFORM_ARCH` can be determined from the table below. diff --git a/engine/src/flutter/lib/snapshot/BUILD.gn b/engine/src/flutter/lib/snapshot/BUILD.gn index e4b52cac21985..e495383162a89 100644 --- a/engine/src/flutter/lib/snapshot/BUILD.gn +++ b/engine/src/flutter/lib/snapshot/BUILD.gn @@ -35,7 +35,10 @@ group("generate_snapshot_bins") { if (host_os == "mac" && (target_os == "mac" || target_os == "ios" || target_os == "android")) { # For macOS target builds: needed for both target CPUs (arm64, x64). - public_deps += [ ":create_macos_gen_snapshots" ] + public_deps += [ + ":create_macos_analyze_snapshots", + ":create_macos_gen_snapshots", + ] } else if (host_os == "mac" && (target_cpu == "arm" || target_cpu == "arm64")) { # For iOS, Android target builds: all AOT target CPUs are arm/arm64. @@ -46,9 +49,11 @@ group("generate_snapshot_bins") { public_deps = [ "$dart_src/runtime/bin:gen_snapshot($host_toolchain)" ] } - # Build analyze_snapshot for 64-bit target CPUs. - if (host_os == "linux" && (target_cpu == "x64" || target_cpu == "arm64" || - target_cpu == "riscv64")) { + # Build analyze_snapshot for 64-bit target CPUs on linux. + # Or always targeting arm64 for Shorebird builds. + if ((host_os == "linux" && + (target_cpu == "x64" || target_cpu == "riscv64")) || + target_cpu == "arm64") { public_deps += [ "$dart_src/runtime/bin:analyze_snapshot($host_toolchain)" ] } } @@ -257,6 +262,69 @@ if (host_os == "mac" && ":create_macos_gen_snapshot_x64${gen_snapshot_suffix}", ] } + + # Added by shorebird. + # analyze_snapshot targets below were copied from the gen_snapshot targets + # above to allow us to include analyze_snapshot in the artifacts generated + # for create_ios_framework.py. + template("build_mac_analyze_snapshot") { + assert(defined(invoker.host_arch)) + host_cpu = invoker.host_arch + + build_toolchain = "//build/toolchain/mac:clang_$host_cpu" + analyze_snapshot_target_name = "analyze_snapshot" + + # At this point, the gen_snapshot equivalent changes + # gen_ snapshot_target_name to "gen_snapshot_host_targeting_host". There is + # no equivalent for analyze_snapshot, so we don't do that here. + # + # It's unclear whether we need to do so now, but we didn't previously, so + # we're not doing it now until we have a reason to. + + analyze_snapshot_target = + "$dart_src/runtime/bin:$analyze_snapshot_target_name($build_toolchain)" + + copy(target_name) { + # The toolchain-specific output directory. For cross-compiles, this is a + # clang-x64 or clang-arm64 subdirectory of the top-level build directory. + output_dir = get_label_info(analyze_snapshot_target, "root_out_dir") + + sources = [ "${output_dir}/${analyze_snapshot_target_name}" ] + outputs = [ + "${root_out_dir}/artifacts_$host_cpu/analyze_snapshot_${target_cpu}", + ] + deps = [ analyze_snapshot_target ] + } + } + + build_mac_analyze_snapshot( + "create_macos_analyze_snapshot_arm64_${target_cpu}") { + host_arch = "arm64" + } + + build_mac_analyze_snapshot( + "create_macos_analyze_snapshot_x64_${target_cpu}") { + host_arch = "x64" + } + + action("create_macos_analyze_snapshots") { + script = "//flutter/sky/tools/create_macos_binary.py" + outputs = [ "${root_out_dir}/analyze_snapshot_${target_cpu}" ] + args = [ + "--in-arm64", + rebase_path( + "${root_out_dir}/artifacts_arm64/analyze_snapshot_${target_cpu}"), + "--in-x64", + rebase_path( + "${root_out_dir}/artifacts_x64/analyze_snapshot_${target_cpu}"), + "--out", + rebase_path("${root_out_dir}/analyze_snapshot_${target_cpu}"), + ] + deps = [ + ":create_macos_analyze_snapshot_arm64_${target_cpu}", + ":create_macos_analyze_snapshot_x64_${target_cpu}", + ] + } } source_set("snapshot") { diff --git a/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart b/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart index e95e0b3f0a4b3..5d94008deed48 100644 --- a/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart +++ b/engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart @@ -55,8 +55,8 @@ class CopyArtifactsStep implements PipelineStep { 'Could not generate artifact bucket url for unknown realm.', ), }; - final url = Uri.https( - 'storage.googleapis.com', + final Uri url = Uri.https( + 'download.shorebird.dev', '${realmComponent}flutter_infra_release/flutter/${realm == LuciRealm.Try ? gitRevision : contentHash}/flutter-web-sdk.zip', ); final http.Response response = await http.Client().get(url); diff --git a/engine/src/flutter/runtime/BUILD.gn b/engine/src/flutter/runtime/BUILD.gn index 2f7aea3d0d107..1d35652477aca 100644 --- a/engine/src/flutter/runtime/BUILD.gn +++ b/engine/src/flutter/runtime/BUILD.gn @@ -118,6 +118,10 @@ source_set("runtime") { "//flutter/third_party/tonic", "//flutter/txt", ] + + if (is_ios) { + deps += [ "//flutter/runtime/shorebird:patch_cache" ] + } } if (enable_unittests) { diff --git a/engine/src/flutter/runtime/dart_isolate.cc b/engine/src/flutter/runtime/dart_isolate.cc index 521c284f1c905..c81ab7c7bd44d 100644 --- a/engine/src/flutter/runtime/dart_isolate.cc +++ b/engine/src/flutter/runtime/dart_isolate.cc @@ -1108,7 +1108,9 @@ Dart_Isolate DartIsolate::DartIsolateGroupCreateCallback( advisory_script_entrypoint, parent_group_data.GetChildIsolatePreparer(), parent_group_data.GetIsolateCreateCallback(), - parent_group_data.GetIsolateShutdownCallback()))); + parent_group_data.GetIsolateShutdownCallback(), + nullptr // native_assets_manager + ))); TaskRunners null_task_runners(advisory_script_uri, /* platform= */ nullptr, diff --git a/engine/src/flutter/runtime/dart_snapshot.cc b/engine/src/flutter/runtime/dart_snapshot.cc index 198a2e75a7edc..fc70f63a346f2 100644 --- a/engine/src/flutter/runtime/dart_snapshot.cc +++ b/engine/src/flutter/runtime/dart_snapshot.cc @@ -6,6 +6,7 @@ #include +#include #include "flutter/fml/native_library.h" #include "flutter/fml/paths.h" #include "flutter/fml/trace_event.h" @@ -13,6 +14,11 @@ #include "flutter/runtime/dart_vm.h" #include "third_party/dart/runtime/include/dart_api.h" +#if SHOREBIRD_USE_INTERPRETER +#include "flutter/runtime/shorebird/patch_cache.h" // nogncheck +#endif +#include "flutter/shell/common/shorebird/updater.h" // nogncheck + namespace flutter { const char* DartSnapshot::kVMDataSymbol = "kDartVmSnapshotData"; @@ -145,7 +151,20 @@ static std::shared_ptr ResolveIsolateData( nullptr, // release_func true // dontneed_safe ); -#else // DART_SNAPSHOT_STATIC_LINK +#else // DART_SNAPSHOT_STATIC_LINK + // Tell the Rust updater we're booting from whatever patch it selected. + // This copies next_boot โ†’ current_boot in the Rust state. The call is + // guarded inside Updater to execute at most once per process โ€” see the + // Updater class comment for why this matters in add-to-app and + // FlutterEngineGroup scenarios. + shorebird::Updater::Instance().ReportLaunchStart(); +#if SHOREBIRD_USE_INTERPRETER + // Try loading from a Shorebird patch first. + if (auto mapping = TryLoadFromPatch(settings.application_library_paths, + DartSnapshot::kIsolateDataSymbol)) { + return mapping; + } +#endif // SHOREBIRD_USE_INTERPRETER return SearchMapping( settings.isolate_snapshot_data, // embedder_mapping_callback settings.isolate_snapshot_data_path, // file_path @@ -165,7 +184,15 @@ static std::shared_ptr ResolveIsolateInstructions( nullptr, // release_func true // dontneed_safe ); -#else // DART_SNAPSHOT_STATIC_LINK +#else // DART_SNAPSHOT_STATIC_LINK +#if SHOREBIRD_USE_INTERPRETER + // Try loading from a Shorebird patch first. + if (auto mapping = + TryLoadFromPatch(settings.application_library_paths, + DartSnapshot::kIsolateInstructionsSymbol)) { + return mapping; + } +#endif // SHOREBIRD_USE_INTERPRETER return SearchMapping( settings.isolate_snapshot_instr, // embedder_mapping_callback settings.isolate_snapshot_instr_path, // file_path diff --git a/engine/src/flutter/runtime/shorebird/BUILD.gn b/engine/src/flutter/runtime/shorebird/BUILD.gn new file mode 100644 index 0000000000000..1f856e36d3f48 --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/BUILD.gn @@ -0,0 +1,16 @@ +import("//flutter/common/config.gni") + +source_set("patch_cache") { + sources = [ + "patch_cache.cc", + "patch_cache.h", + "patch_mapping.cc", + "patch_mapping.h", + ] + + deps = [ + "//flutter/fml", + "//flutter/runtime:libdart", + "//flutter/shell/common/shorebird:updater", + ] +} diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.cc b/engine/src/flutter/runtime/shorebird/patch_cache.cc new file mode 100644 index 0000000000000..8c5cc3a83f182 --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/patch_cache.cc @@ -0,0 +1,165 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/runtime/shorebird/patch_cache.h" + +#include + +#include "flutter/fml/logging.h" +#include "flutter/fml/mapping.h" +#include "flutter/runtime/shorebird/patch_mapping.h" +#include "third_party/dart/runtime/include/dart_api.h" + +namespace flutter { + +namespace { + +// These symbol names match the constants in dart_snapshot.cc. +// We duplicate them here rather than extracting them into a header. +// They are actually defined down in Dart and will never change. +constexpr const char* kIsolateDataSymbol = "kDartIsolateSnapshotData"; +constexpr const char* kIsolateInstructionsSymbol = + "kDartIsolateSnapshotInstructions"; + +} // namespace + +// PatchCacheEntry implementation + +std::shared_ptr PatchCacheEntry::Create( + const std::string& path) { + // vmcode files currently use ELF internally after a prefix of a Shorebird + // linker header. + auto elf_mapping = fml::FileMapping::CreateReadOnly(path); + if (!elf_mapping) { + FML_LOG(ERROR) << "Failed to map file: " << path; + return nullptr; + } + + int elf_file_offset = Shorebird_ReadLinkHeader(elf_mapping->GetMapping(), + elf_mapping->GetSize()); + + const char* error = nullptr; + // The VM Snapshot is identical for all binaries produced by a given version + // of Dart. Our linker checks this and will fail to link if ever the VM + // snapshot changes. We ignore the VM data/instrs here. + const uint8_t* ignored_vm_data = nullptr; + const uint8_t* ignored_vm_instrs = nullptr; + const uint8_t* isolate_data = nullptr; + const uint8_t* isolate_instrs = nullptr; + + Dart_LoadedElf* elf = Dart_LoadELF( + path.c_str(), elf_file_offset, &error, &ignored_vm_data, + &ignored_vm_instrs, &isolate_data, &isolate_instrs, dart::bin::kReadOnly); + + if (elf == nullptr) { + FML_LOG(ERROR) << "Failed to load patch at " << path << " error: " << error; + return nullptr; + } + + FML_LOG(INFO) << "Loaded patch from " << path; + + return std::shared_ptr( + new PatchCacheEntry(path, elf, isolate_data, isolate_instrs)); +} + +PatchCacheEntry::PatchCacheEntry(const std::string& path, + Dart_LoadedElf* elf, + const uint8_t* isolate_data, + const uint8_t* isolate_instrs) + : path_(path), + elf_(elf), + isolate_data_(isolate_data), + isolate_instrs_(isolate_instrs) {} + +PatchCacheEntry::~PatchCacheEntry() { + if (elf_ != nullptr) { + FML_LOG(INFO) << "Unloading patch from " << path_; + Dart_UnloadELF(elf_); + elf_ = nullptr; + } +} + +PatchCache& PatchCache::Instance() { + static PatchCache instance; + return instance; +} + +std::shared_ptr PatchCache::GetOrLoad( + const std::string& path) { + std::lock_guard lock(mutex_); + + // Check if we have a cached entry that's still alive + auto it = cache_.find(path); + if (it != cache_.end()) { + if (auto entry = it->second.lock()) { + FML_LOG(INFO) << "PatchCache hit for " << path; + return entry; + } + // Entry expired, remove it + cache_.erase(it); + } + + // Load a new entry + auto entry = PatchCacheEntry::Create(path); + if (entry) { + cache_[path] = entry; // Store weak_ptr + } + + return entry; +} + +void PatchCache::PruneExpired() { + std::lock_guard lock(mutex_); + + for (auto it = cache_.begin(); it != cache_.end();) { + if (it->second.expired()) { + it = cache_.erase(it); + } else { + ++it; + } + } +} + +std::shared_ptr TryLoadFromPatch( + const std::vector& native_library_paths, + const char* symbol_name) { + if (native_library_paths.empty()) { + return nullptr; + } + + // Check if the first path is a Shorebird patch (.vmcode file) + const auto& patch_path = native_library_paths.front(); + bool is_patch = patch_path.find(".vmcode") != std::string::npos; + if (!is_patch) { + return nullptr; + } + + // Patches only contain isolate data/instructions, not VM data/instructions. + // Return nullptr for VM symbols to allow fallback to the base app. + std::string symbol(symbol_name); + if (symbol != kIsolateDataSymbol && symbol != kIsolateInstructionsSymbol) { + return nullptr; + } + + // Load the patch using the cache. + auto cache_entry = PatchCache::Instance().GetOrLoad(patch_path); + if (!cache_entry) { + FML_LOG(FATAL) << "Failed to load symbol from patch at " << patch_path; + return nullptr; + } + + FML_LOG(INFO) << "Loading symbol from patch: " << symbol_name; + + // ReportLaunchStart is now called from ResolveIsolateData in + // dart_snapshot.cc, which runs before TryLoadFromPatch on all platforms. + + if (symbol == kIsolateDataSymbol) { + return PatchMapping::CreateIsolateData(cache_entry); + } else { + FML_CHECK(symbol == kIsolateInstructionsSymbol); + return PatchMapping::CreateIsolateInstructions(cache_entry); + } +} + +} // namespace flutter diff --git a/engine/src/flutter/runtime/shorebird/patch_cache.h b/engine/src/flutter/runtime/shorebird/patch_cache.h new file mode 100644 index 0000000000000..1f2e14fab711d --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/patch_cache.h @@ -0,0 +1,104 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_RUNTIME_SHOREBIRD_PATCH_CACHE_H_ +#define FLUTTER_RUNTIME_SHOREBIRD_PATCH_CACHE_H_ + +#include +#include +#include +#include +#include + +#include + +#include "flutter/fml/macros.h" +#include "flutter/fml/mapping.h" + +namespace flutter { + +/// A cache entry that holds a loaded patch file and its extracted snapshot +/// pointers. The patch is automatically unloaded when the last reference to +/// this entry is released. +class PatchCacheEntry { + public: + /// Creates a new cache entry by loading the patch file at the given path. + /// Returns nullptr if loading fails. + static std::shared_ptr Create(const std::string& path); + + ~PatchCacheEntry(); + + /// Returns the isolate snapshot data pointer. + const uint8_t* isolate_data() const { return isolate_data_; } + + /// Returns the isolate snapshot instructions pointer. + const uint8_t* isolate_instructions() const { return isolate_instrs_; } + + /// Returns the path this entry was loaded from. + const std::string& path() const { return path_; } + + private: + PatchCacheEntry(const std::string& path, + Dart_LoadedElf* elf, + const uint8_t* isolate_data, + const uint8_t* isolate_instrs); + + std::string path_; + Dart_LoadedElf* elf_; + const uint8_t* isolate_data_; + const uint8_t* isolate_instrs_; + + FML_DISALLOW_COPY_AND_ASSIGN(PatchCacheEntry); +}; + +/// A thread-safe cache for loaded patch files. Cache entries are automatically +/// removed when all references to them are released. +class PatchCache { + public: + /// Returns the singleton instance of the cache. + static PatchCache& Instance(); + + /// Gets or loads a patch file at the given path. If the file is already + /// cached and the entry is still alive, returns the existing entry. + /// Otherwise, loads the file and creates a new cache entry. + /// Returns nullptr if loading fails. + std::shared_ptr GetOrLoad(const std::string& path); + + /// Removes expired entries from the cache. This is called automatically + /// by GetOrLoad, but can also be called explicitly. + void PruneExpired(); + + private: + PatchCache() = default; + ~PatchCache() = default; + + std::mutex mutex_; + // We store weak references so entries are automatically cleaned up when + // all ElfMapping instances release their references. + std::map> cache_; + + FML_DISALLOW_COPY_AND_ASSIGN(PatchCache); +}; + +/// Checks if the first path in native_library_paths is a Shorebird patch +/// (.vmcode file) and if so, attempts to load the requested symbol from +/// the patch. +/// +/// @param native_library_paths The list of library paths to check. The first +/// path is checked for the .vmcode extension. +/// @param symbol_name The symbol to load (kIsolateDataSymbol or +/// kIsolateInstructionsSymbol). +/// @return A mapping for the requested symbol if this is a patch and the +/// symbol is available in the patch, nullptr otherwise. +/// +/// Note: Patches only contain isolate data/instructions, not VM +/// data/instructions. For VM symbols, this will always return nullptr, +/// allowing the caller to fall back to loading from the base app. +std::shared_ptr TryLoadFromPatch( + const std::vector& native_library_paths, + const char* symbol_name); + +} // namespace flutter + +#endif // FLUTTER_RUNTIME_SHOREBIRD_PATCH_CACHE_H_ diff --git a/engine/src/flutter/runtime/shorebird/patch_mapping.cc b/engine/src/flutter/runtime/shorebird/patch_mapping.cc new file mode 100644 index 0000000000000..d444cda817c34 --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/patch_mapping.cc @@ -0,0 +1,51 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/runtime/shorebird/patch_mapping.h" + +#include "third_party/dart/runtime/include/dart_native_api.h" + +namespace flutter { + +std::shared_ptr PatchMapping::CreateIsolateData( + std::shared_ptr entry) { + if (!entry) { + return nullptr; + } + const uint8_t* data = entry->isolate_data(); + size_t size = Dart_SnapshotDataSize(data); + return std::shared_ptr(new PatchMapping(entry, data, size)); +} + +std::shared_ptr PatchMapping::CreateIsolateInstructions( + std::shared_ptr entry) { + if (!entry) { + return nullptr; + } + const uint8_t* data = entry->isolate_instructions(); + size_t size = Dart_SnapshotInstrSize(data); + return std::shared_ptr(new PatchMapping(entry, data, size)); +} + +PatchMapping::PatchMapping(std::shared_ptr entry, + const uint8_t* data, + size_t size) + : cache_entry_(std::move(entry)), data_(data), size_(size) {} + +PatchMapping::~PatchMapping() = default; + +size_t PatchMapping::GetSize() const { + return size_; +} + +const uint8_t* PatchMapping::GetMapping() const { + return data_; +} + +bool PatchMapping::IsDontNeedSafe() const { + // Patch mappings are file-backed and safe for madvise(DONTNEED). + return true; +} + +} // namespace flutter diff --git a/engine/src/flutter/runtime/shorebird/patch_mapping.h b/engine/src/flutter/runtime/shorebird/patch_mapping.h new file mode 100644 index 0000000000000..8c2e494a9004c --- /dev/null +++ b/engine/src/flutter/runtime/shorebird/patch_mapping.h @@ -0,0 +1,55 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_RUNTIME_SHOREBIRD_PATCH_MAPPING_H_ +#define FLUTTER_RUNTIME_SHOREBIRD_PATCH_MAPPING_H_ + +#include + +#include "flutter/fml/macros.h" +#include "flutter/fml/mapping.h" +#include "flutter/runtime/shorebird/patch_cache.h" + +namespace flutter { + +/// A Mapping implementation that references data from a cached patch file. +/// Holding a reference to this mapping keeps the underlying patch loaded. +class PatchMapping final : public fml::Mapping { + public: + /// Creates a mapping for the isolate snapshot data from the given cache + /// entry. + static std::shared_ptr CreateIsolateData( + std::shared_ptr entry); + + /// Creates a mapping for the isolate snapshot instructions from the given + /// cache entry. + static std::shared_ptr CreateIsolateInstructions( + std::shared_ptr entry); + + ~PatchMapping() override; + + // |fml::Mapping| + size_t GetSize() const override; + + // |fml::Mapping| + const uint8_t* GetMapping() const override; + + // |fml::Mapping| + bool IsDontNeedSafe() const override; + + private: + PatchMapping(std::shared_ptr entry, + const uint8_t* data, + size_t size); + + std::shared_ptr cache_entry_; + const uint8_t* data_; + size_t size_; + + FML_DISALLOW_COPY_AND_ASSIGN(PatchMapping); +}; + +} // namespace flutter + +#endif // FLUTTER_RUNTIME_SHOREBIRD_PATCH_MAPPING_H_ diff --git a/engine/src/flutter/shell/common/BUILD.gn b/engine/src/flutter/shell/common/BUILD.gn index e1aba738f17be..5eb7b979fe2f0 100644 --- a/engine/src/flutter/shell/common/BUILD.gn +++ b/engine/src/flutter/shell/common/BUILD.gn @@ -153,6 +153,7 @@ source_set("common") { "//flutter/lib/ui", "//flutter/runtime", "//flutter/shell/common:base64", + "//flutter/shell/common/shorebird:updater", "//flutter/shell/geometry", "//flutter/shell/profiling", "//flutter/skia", @@ -342,6 +343,7 @@ if (enable_unittests) { "//flutter/common/graphics", "//flutter/display_list/testing:display_list_testing", "//flutter/shell/common:base64", + "//flutter/shell/common/shorebird:updater", "//flutter/shell/profiling:profiling_unittests", "//flutter/shell/version", "//flutter/testing:fixture_test", diff --git a/engine/src/flutter/shell/common/shell.cc b/engine/src/flutter/shell/common/shell.cc index 762663b061d1c..87b5d9094f76b 100644 --- a/engine/src/flutter/shell/common/shell.cc +++ b/engine/src/flutter/shell/common/shell.cc @@ -47,6 +47,8 @@ #include "third_party/skia/include/core/SkGraphics.h" #include "third_party/tonic/common/log.h" +#include "flutter/shell/common/shorebird/updater.h" + namespace flutter { constexpr char kSkiaChannel[] = "flutter/skia"; @@ -522,6 +524,18 @@ Shell::Shell(DartVMRef vm, is_gpu_disabled_sync_switch_(new fml::SyncSwitch(is_gpu_disabled)), weak_factory_gpu_(nullptr), weak_factory_(this) { + // Report launch outcome to the Shorebird updater for crash recovery. + // If the VM failed to start, we report failure so the updater can roll + // back the patch. These calls are guarded inside Updater to execute at + // most once per process โ€” only the first Shell's outcome is reported. + // In add-to-app, subsequent engines are silently ignored since they + // boot from the same snapshot that was already reported on. + // On unsupported platforms, NoOpUpdater handles these calls gracefully. + if (!vm_) { + shorebird::Updater::Instance().ReportLaunchFailure(); + } else { + shorebird::Updater::Instance().ReportLaunchSuccess(); + } FML_CHECK(!settings.enable_software_rendering || !settings.enable_impeller) << "Software rendering is incompatible with Impeller."; if (!settings.enable_impeller && settings.warn_on_impeller_opt_out) { diff --git a/engine/src/flutter/shell/common/shell_unittests.cc b/engine/src/flutter/shell/common/shell_unittests.cc index 2ce1a9040d5a4..5ed39de31b6be 100644 --- a/engine/src/flutter/shell/common/shell_unittests.cc +++ b/engine/src/flutter/shell/common/shell_unittests.cc @@ -52,6 +52,8 @@ #include "third_party/skia/include/codec/SkCodecAnimation.h" #include "third_party/tonic/converter/dart_converter.h" +#include "flutter/shell/common/shorebird/updater.h" + #ifdef SHELL_ENABLE_VULKAN #include "flutter/vulkan/vulkan_application.h" // nogncheck #endif @@ -5108,6 +5110,70 @@ TEST_F(ShellTest, ShoulDiscardLayerTreeIfFrameIsSizedIncorrectly) { DestroyShell(std::move(shell), task_runners); } +// Test the full boot flow: ReportLaunchStart is called from +// ResolveIsolateData, then ReportLaunchSuccess from the Shell constructor. +// Both are guarded to run at most once per process. +TEST_F(ShellTest, ShorebirdBootFlowCallsLaunchStartThenSuccess) { + auto mock = std::make_unique(); + auto* mock_ptr = mock.get(); + shorebird::Updater::SetInstanceForTesting(std::move(mock)); + shorebird::Updater::ResetLaunchStateForTesting(); + + auto settings = CreateSettingsForFixture(); + auto task_runners = GetTaskRunnersForFixture(); + auto shell = CreateShell(settings, task_runners); + ASSERT_TRUE(shell); + + const auto& log = mock_ptr->call_log(); + ASSERT_EQ(log.size(), 2u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchSuccess"); + + DestroyShell(std::move(shell), task_runners); + shorebird::Updater::ResetLaunchStateForTesting(); + shorebird::Updater::ResetInstanceForTesting(); +} + +// In add-to-app, multiple engines may be created within a single process. +// Only the first engine should report launch start/success to the Rust +// updater. This prevents the updater from promoting a newly-downloaded patch +// to "current_boot" when subsequent engines are still running the original +// snapshot that was selected at process init time. +TEST_F(ShellTest, ShorebirdUpdaterReportsOnlyOnceForMultipleShells) { + auto mock = std::make_unique(); + auto* mock_ptr = mock.get(); + shorebird::Updater::SetInstanceForTesting(std::move(mock)); + shorebird::Updater::ResetLaunchStateForTesting(); + + auto settings = CreateSettingsForFixture(); + + // Create first shell โ€” gets Start + Success + auto task_runners1 = GetTaskRunnersForFixture(); + auto shell1 = CreateShell(settings, task_runners1); + ASSERT_TRUE(shell1); + EXPECT_EQ(mock_ptr->launch_start_count(), 1); + EXPECT_EQ(mock_ptr->launch_success_count(), 1); + + // Create second shell โ€” guarded, no additional Start or Success calls. + auto task_runners2 = GetTaskRunnersForFixture(); + auto shell2 = CreateShell(settings, task_runners2); + ASSERT_TRUE(shell2); + EXPECT_EQ(mock_ptr->launch_start_count(), 1); + EXPECT_EQ(mock_ptr->launch_success_count(), 1); + + // Only one Start+Success pair in the call log. + const auto& log = mock_ptr->call_log(); + ASSERT_EQ(log.size(), 2u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchSuccess"); + + DestroyShell(std::move(shell1), task_runners1); + DestroyShell(std::move(shell2), task_runners2); + + shorebird::Updater::ResetLaunchStateForTesting(); + shorebird::Updater::ResetInstanceForTesting(); +} + } // namespace testing } // namespace flutter diff --git a/engine/src/flutter/shell/common/shorebird/BUILD.gn b/engine/src/flutter/shell/common/shorebird/BUILD.gn new file mode 100644 index 0000000000000..2b1e29c03512d --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/BUILD.gn @@ -0,0 +1,111 @@ +import("//flutter/common/config.gni") +import("//flutter/testing/testing.gni") + +source_set("snapshots_data_handle") { + sources = [ + "snapshots_data_handle.cc", + "snapshots_data_handle.h", + ] + + deps = [ + "//flutter/fml", + "//flutter/runtime", + "//flutter/runtime:libdart", + "//flutter/shell/common", + ] +} + +# C++ wrapper around the Rust updater C API. +# This provides a testable abstraction layer that can be mocked for testing. +source_set("updater") { + sources = [ + "updater.cc", + "updater.h", + ] + + deps = [ "//flutter/fml" ] + + # For the Rust updater C API (shorebird_report_launch_start, etc.) + include_dirs = [ "//flutter" ] + + # Link the Rust updater static library based on target platform. + if (is_android) { + if (target_cpu == "arm") { + libs = [ "//flutter/third_party/updater/target/armv7-linux-androideabi/release/libupdater.a" ] + } else if (target_cpu == "arm64") { + libs = [ "//flutter/third_party/updater/target/aarch64-linux-android/release/libupdater.a" ] + } else if (target_cpu == "x64") { + libs = [ "//flutter/third_party/updater/target/x86_64-linux-android/release/libupdater.a" ] + } else if (target_cpu == "x86") { + libs = [ "//flutter/third_party/updater/target/i686-linux-android/release/libupdater.a" ] + } + } else if (is_ios) { + if (target_cpu == "arm64") { + libs = [ "//flutter/third_party/updater/target/aarch64-apple-ios/release/libupdater.a" ] + } else if (target_cpu == "x64") { + libs = [ "//flutter/third_party/updater/target/x86_64-apple-ios/release/libupdater.a" ] + } + } else if (is_mac) { + if (target_cpu == "arm64") { + libs = [ "//flutter/third_party/updater/target/aarch64-apple-darwin/release/libupdater.a" ] + } else if (target_cpu == "x64") { + libs = [ "//flutter/third_party/updater/target/x86_64-apple-darwin/release/libupdater.a" ] + } + } else if (is_win) { + if (target_cpu == "x64") { + libs = [ + "userenv.lib", + "//flutter/third_party/updater/target/x86_64-pc-windows-msvc/release/updater.lib", + ] + } + } else if (is_linux) { + if (target_cpu == "x64") { + libs = [ "//flutter/third_party/updater/target/x86_64-unknown-linux-gnu/release/libupdater.a" ] + } + } +} + +source_set("shorebird") { + sources = [ + "shorebird.cc", + "shorebird.h", + ] + + deps = [ + ":snapshots_data_handle", + ":updater", + "//flutter/fml", + "//flutter/runtime", + "//flutter/runtime:libdart", + "//flutter/shell/common", + "//flutter/shell/platform/embedder:embedder_headers", + ] +} + +if (enable_unittests) { + test_fixtures("shorebird_fixtures") { + fixtures = [] + } + + executable("shorebird_unittests") { + testonly = true + + sources = [ + "patch_cache_unittests.cc", + "shorebird_unittests.cc", + "snapshots_data_handle_unittests.cc", + "updater_unittests.cc", + ] + + deps = [ + ":shorebird", + ":shorebird_fixtures", + ":snapshots_data_handle", + ":updater", + "//flutter/runtime", + "//flutter/runtime/shorebird:patch_cache", + "//flutter/testing", + "//flutter/testing:fixture_test", + ] + } +} diff --git a/engine/src/flutter/shell/common/shorebird/patch_cache_unittests.cc b/engine/src/flutter/shell/common/shorebird/patch_cache_unittests.cc new file mode 100644 index 0000000000000..51fc20f9ca562 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/patch_cache_unittests.cc @@ -0,0 +1,70 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/runtime/shorebird/patch_cache.h" + +#include +#include + +#include "gtest/gtest.h" + +namespace flutter { +namespace testing { + +TEST(PatchCache, InstanceReturnsSameInstance) { + PatchCache& instance1 = PatchCache::Instance(); + PatchCache& instance2 = PatchCache::Instance(); + EXPECT_EQ(&instance1, &instance2); +} + +TEST(TryLoadFromPatch, ReturnsNullptrForEmptyPaths) { + std::vector empty_paths; + auto result = TryLoadFromPatch(empty_paths, "kDartIsolateSnapshotData"); + EXPECT_EQ(result, nullptr); +} + +TEST(TryLoadFromPatch, ReturnsNullptrForNonVmcodePath) { + std::vector paths = {"/path/to/some/file.so"}; + auto result = TryLoadFromPatch(paths, "kDartIsolateSnapshotData"); + EXPECT_EQ(result, nullptr); +} + +TEST(TryLoadFromPatch, ReturnsNullptrForVmSymbol) { + // Even with a .vmcode path, VM symbols should return nullptr + // (we can't actually load the file, but we can verify the symbol check) + std::vector paths = {"/path/to/patch.vmcode"}; + + // VM data symbol should return nullptr (patches don't contain VM snapshots) + auto result_vm_data = TryLoadFromPatch(paths, "kDartVmSnapshotData"); + EXPECT_EQ(result_vm_data, nullptr); + + // VM instructions symbol should return nullptr + auto result_vm_instrs = + TryLoadFromPatch(paths, "kDartVmSnapshotInstructions"); + EXPECT_EQ(result_vm_instrs, nullptr); +} + +TEST(TryLoadFromPatch, ReturnsNullptrForUnknownSymbol) { + std::vector paths = {"/path/to/patch.vmcode"}; + auto result = TryLoadFromPatch(paths, "kSomeUnknownSymbol"); + EXPECT_EQ(result, nullptr); +} + +TEST(TryLoadFromPatch, ChecksOnlyFirstPath) { + // Only the first path should be checked for .vmcode extension + std::vector paths = {"/path/to/regular.so", + "/path/to/patch.vmcode"}; + auto result = TryLoadFromPatch(paths, "kDartIsolateSnapshotData"); + // Should return nullptr because first path is not .vmcode + EXPECT_EQ(result, nullptr); +} + +TEST(PatchCache, GetOrLoadReturnsNullptrForNonexistentFile) { + auto result = + PatchCache::Instance().GetOrLoad("/nonexistent/path/to/file.vmcode"); + EXPECT_EQ(result, nullptr); +} + +} // namespace testing +} // namespace flutter diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.cc b/engine/src/flutter/shell/common/shorebird/shorebird.cc new file mode 100644 index 0000000000000..71d336e90f8b0 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/shorebird.cc @@ -0,0 +1,304 @@ + +#include "flutter/shell/common/shorebird/shorebird.h" + +#include +#include +#include +#include +#include + +#include "flutter/fml/command_line.h" +#include "flutter/fml/file.h" +#include "flutter/fml/macros.h" +#include "flutter/fml/mapping.h" +#include "flutter/fml/message_loop.h" +#include "flutter/fml/native_library.h" +#include "flutter/fml/paths.h" +#include "flutter/lib/ui/plugins/callback_cache.h" +#include "flutter/runtime/dart_snapshot.h" +#include "flutter/runtime/dart_vm.h" +#include "flutter/shell/common/shell.h" +#include "flutter/shell/common/shorebird/snapshots_data_handle.h" +#include "flutter/shell/common/shorebird/updater.h" +#include "flutter/shell/common/switches.h" +#include "fml/logging.h" +#include "shell/platform/embedder/embedder.h" +#include "third_party/dart/runtime/include/dart_tools_api.h" + +// Namespaced to avoid Google style warnings. +namespace flutter { + +// Old Android versions (e.g. the v16 ndk Flutter uses) don't always include a +// getauxval symbol, but the Rust ring crate assumes it exists: +// https://github.com/briansmith/ring/blob/fa25bf3a7403c9fe6458cb87bd8427be41225ca2/src/cpu/arm.rs#L22 +// It uses it to determine if the CPU supports AES instructions. +// Making this a weak symbol allows the linker to use a real version instead +// if it can find one. +// BoringSSL just reads from procfs instead, which is what we would do if +// we needed to implement this ourselves. Implementation looks straightforward: +// https://lwn.net/Articles/519085/ +// https://github.com/google/boringssl/blob/6ab4f0ae7f2db96d240eb61a5a8b4724e5a09b2f/crypto/cpu_arm_linux.c +#if defined(__ANDROID__) && defined(__arm__) +extern "C" __attribute__((weak)) unsigned long getauxval(unsigned long type) { + return 0; +} +#endif + +#if SHOREBIRD_USE_INTERPRETER +// Global references to the base (unpatched) snapshots from the App.framework. +// These are process-global because: +// 1. The Shorebird updater library is a process-global singleton with its own +// internal state. FileCallbacksImpl provides it access to the base snapshot +// data for patch generation/validation. +// 2. The base snapshots are immutable (baked into the IPA) so sharing them +// across isolate groups is safe. +// +// Note: This design doesn't support multiple engines with different base +// snapshots, but I'm not aware of any use cases for that on iOS. +static fml::RefPtr vm_snapshot; +static fml::RefPtr isolate_snapshot; + +void SetBaseSnapshot(Settings& settings) { + // These mappings happen to be to static data in the App.framework, but + // we still need to seem to hold onto the DartSnapshot objects to keep + // the mappings alive. + vm_snapshot = DartSnapshot::VMSnapshotFromSettings(settings); + isolate_snapshot = DartSnapshot::IsolateSnapshotFromSettings(settings); + Shorebird_SetBaseSnapshots(isolate_snapshot->GetDataMapping(), + isolate_snapshot->GetInstructionsMapping(), + vm_snapshot->GetDataMapping(), + vm_snapshot->GetInstructionsMapping()); +} +#endif // SHOREBIRD_USE_INTERPRETER + +class FileCallbacksImpl { + public: + static void* Open(); + static uintptr_t Read(void* file, uint8_t* buffer, uintptr_t length); + static int64_t Seek(void* file, int64_t offset, int32_t whence); + static void Close(void* file); +}; + +shorebird::FileCallbacks ShorebirdFileCallbacks() { + return { + .open = FileCallbacksImpl::Open, + .read = FileCallbacksImpl::Read, + .seek = FileCallbacksImpl::Seek, + .close = FileCallbacksImpl::Close, + }; +} + +// Given the contents of a yaml file, return the given value if it exists, +// otherwise return an empty string. +// Does not support nested keys. +std::string GetValueFromYaml(const std::string& yaml, const std::string& key) { + std::stringstream ss(yaml); + std::string line; + std::string prefix = key + ":"; + while (std::getline(ss, line, '\n')) { + if (line.find(prefix) != std::string::npos) { + auto ret = line.substr(line.find(prefix) + prefix.size()); + + // Remove leading and trailing spaces + while (!ret.empty() && std::isspace(ret.front())) { + ret.erase(0, 1); + } + while (!ret.empty() && std::isspace(ret.back())) { + ret.pop_back(); + } + return ret; + } + } + return ""; +} + +/// Newer api, used by Desktop implementations. +/// Does not directly manipulate Settings. +// TODO(eseidel): Consolidate this with the other ConfigureShorebird() API. +bool ConfigureShorebird(const ShorebirdConfigArgs& args, + std::string& patch_path) { + patch_path = args.release_app_library_path; + auto shorebird_updater_dir_name = "shorebird_updater"; + + // Parse app id from shorebird.yaml + std::string app_id = GetValueFromYaml(args.shorebird_yaml, "app_id"); + if (app_id.empty()) { + FML_LOG(ERROR) << "Shorebird updater: appid not found in shorebird.yaml"; + return false; + } + + auto code_cache_dir = fml::paths::JoinPaths( + {std::move(args.code_cache_path), shorebird_updater_dir_name, app_id}); + auto app_storage_dir = fml::paths::JoinPaths( + {std::move(args.app_storage_path), shorebird_updater_dir_name, app_id}); + + fml::CreateDirectory(fml::paths::GetCachesDirectory(), + {shorebird_updater_dir_name}, + fml::FilePermission::kReadWrite); + + // Combine version and version_code into a single string. + // We could also pass these separately through to the updater if needed. + auto release_version = args.release_version.version; + if (!args.release_version.build_number.empty()) { + release_version += "+" + args.release_version.build_number; + } + + shorebird::AppConfig config; + config.release_version = release_version; + config.original_libapp_paths = {args.release_app_library_path}; + config.app_storage_dir = app_storage_dir; + config.code_cache_dir = code_cache_dir; + config.file_callbacks = ShorebirdFileCallbacks(); + config.yaml_config = args.shorebird_yaml; + + bool init_result = shorebird::Updater::Instance().Init(config); + + // We do not support synchronous updates on launch, it's a terrible UX. + // Users can implement custom check-for-updates using + // package:shorebird_code_push. + // https://github.com/shorebirdtech/shorebird/issues/950 + + FML_LOG(INFO) << "Checking for active patch"; + shorebird::Updater::Instance().ValidateNextBootPatch(); + std::string active_path = shorebird::Updater::Instance().NextBootPatchPath(); + if (!active_path.empty()) { + patch_path = active_path; + FML_LOG(INFO) << "Shorebird updater: patch path: " << patch_path; + } else { + FML_LOG(INFO) << "Shorebird updater: no active patch."; + } + + // Note: shorebird_report_launch_start() is now called from TryLoadFromPatch() + // in runtime/shorebird/patch_cache.cc, right before the patched snapshot is + // actually loaded. This fixes issues with FlutterEngineGroup and other cases + // where ConfigureShorebird() is called but no Shell is created. + if (!init_result) { + return false; + } + + if (shorebird::Updater::Instance().ShouldAutoUpdate()) { + FML_LOG(INFO) << "Starting Shorebird update"; + shorebird::Updater::Instance().StartUpdateThread(); + } else { + FML_LOG(INFO) + << "Shorebird auto_update disabled, not checking for updates."; + } + + return true; +} + +/// Older api used by iOS and Android, directly manipulates Settings. +// TODO(eseidel): Consolidate this with the other ConfigureShorebird() API. +void ConfigureShorebird(std::string code_cache_path, + std::string app_storage_path, + Settings& settings, + const std::string& shorebird_yaml, + const std::string& version, + const std::string& version_code) { + // If you are crashing here, you probably are running Shorebird in a Debug + // config, where the AOT snapshot won't be linked into the process, and thus + // lookups will fail. Change your Scheme to Release to fix: + // https://github.com/flutter/flutter/wiki/Debugging-the-engine#debugging-ios-builds-with-xcode + FML_CHECK(DartSnapshot::VMSnapshotFromSettings(settings)) + << "XCode Scheme must be set to Release to use Shorebird"; + + auto shorebird_updater_dir_name = "shorebird_updater"; + + auto code_cache_dir = fml::paths::JoinPaths( + {std::move(code_cache_path), shorebird_updater_dir_name}); + auto app_storage_dir = fml::paths::JoinPaths( + {std::move(app_storage_path), shorebird_updater_dir_name}); + + fml::CreateDirectory(fml::paths::GetCachesDirectory(), + {shorebird_updater_dir_name}, + fml::FilePermission::kReadWrite); + + // Combine version and version_code into a single string. + // We could also pass these separately through to the updater if needed. + shorebird::AppConfig config; + config.release_version = version + "+" + version_code; + config.original_libapp_paths = settings.application_library_paths; + config.app_storage_dir = app_storage_dir; + config.code_cache_dir = code_cache_dir; + config.file_callbacks = ShorebirdFileCallbacks(); + config.yaml_config = shorebird_yaml; + + bool init_result = shorebird::Updater::Instance().Init(config); + + // We do not support synchronous updates on launch, it's a terrible UX. + // Users can implement custom check-for-updates using + // package:shorebird_code_push. + // https://github.com/shorebirdtech/shorebird/issues/950 + + // We only set the base snapshot on iOS for now. +#if SHOREBIRD_USE_INTERPRETER + SetBaseSnapshot(settings); +#endif + + shorebird::Updater::Instance().ValidateNextBootPatch(); + std::string active_path = shorebird::Updater::Instance().NextBootPatchPath(); + if (!active_path.empty()) { + FML_LOG(INFO) << "Shorebird updater: active path: " << active_path; + +#if SHOREBIRD_USE_INTERPRETER + // On iOS we add the patch to the front of the list instead of clearing + // the list, to allow dart_snapshot.cc to still find the base snapshot + // for the vm isolate. + settings.application_library_paths.insert( + settings.application_library_paths.begin(), active_path); +#else + settings.application_library_paths.clear(); + settings.application_library_paths.emplace_back(active_path); +#endif + } else { + FML_LOG(INFO) << "Shorebird updater: no active patch."; + } + + // Note: shorebird_report_launch_start() is now called from TryLoadFromPatch() + // in runtime/shorebird/patch_cache.cc, right before the patched snapshot is + // actually loaded. This fixes issues with FlutterEngineGroup and other cases + // where ConfigureShorebird() is called but no Shell is created. + + if (!init_result) { + return; + } + + if (shorebird::Updater::Instance().ShouldAutoUpdate()) { + FML_LOG(INFO) << "Starting Shorebird update"; + shorebird::Updater::Instance().StartUpdateThread(); + } else { + FML_LOG(INFO) + << "Shorebird auto_update disabled, not checking for updates."; + } +} + +void* FileCallbacksImpl::Open() { +#if SHOREBIRD_USE_INTERPRETER + return SnapshotsDataHandle::createForSnapshots(*vm_snapshot, + *isolate_snapshot) + .release(); +#else + // SnapshotsDataHandle exists on all platforms (for testing) but is only used + // on iOS. iOS patches are generated from just the Dart parts of the snapshot, + // excluding the Mach-O specific headers which contain dates and paths that + // make them change on every build. + return nullptr; +#endif // SHOREBIRD_USE_INTERPRETER +} + +uintptr_t FileCallbacksImpl::Read(void* file, + uint8_t* buffer, + uintptr_t length) { + return reinterpret_cast(file)->Read(buffer, length); +} + +int64_t FileCallbacksImpl::Seek(void* file, int64_t offset, int32_t whence) { + // Currently we only support blob handles. + return reinterpret_cast(file)->Seek(offset, whence); +} + +void FileCallbacksImpl::Close(void* file) { + delete reinterpret_cast(file); +} + +} // namespace flutter \ No newline at end of file diff --git a/engine/src/flutter/shell/common/shorebird/shorebird.h b/engine/src/flutter/shell/common/shorebird/shorebird.h new file mode 100644 index 0000000000000..ab5c9162ea0f2 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/shorebird.h @@ -0,0 +1,59 @@ +#ifndef FLUTTER_SHELL_COMMON_SHOREBIRD_SHOREBIRD_H_ +#define FLUTTER_SHELL_COMMON_SHOREBIRD_SHOREBIRD_H_ + +#include "flutter/common/settings.h" +#include "flutter/fml/memory/ref_ptr.h" +#include "shell/platform/embedder/embedder.h" + +namespace flutter { + +class DartSnapshot; + +/// Version and build number of the release. +/// Used by ShorebirdConfigArgs. +struct ReleaseVersion { + std::string version; + std::string build_number; +}; + +/// Arguments for ConfigureShorebird. +/// Used by Desktop implementations. +struct ShorebirdConfigArgs { + std::string code_cache_path; + std::string app_storage_path; + std::string release_app_library_path; + std::string shorebird_yaml; + ReleaseVersion release_version; + + ShorebirdConfigArgs(std::string code_cache_path, + std::string app_storage_path, + std::string release_app_library_path, + std::string shorebird_yaml, + ReleaseVersion release_version) + : code_cache_path(code_cache_path), + app_storage_path(app_storage_path), + release_app_library_path(release_app_library_path), + shorebird_yaml(shorebird_yaml), + release_version(release_version) {} +}; + +/// Newer api, used by Desktop implementations. +/// Does not directly manipulate Settings. +bool ConfigureShorebird(const ShorebirdConfigArgs& args, + std::string& patch_path); + +/// Older api used by iOS and Android, directly manipulates Settings. +void ConfigureShorebird(std::string code_cache_path, + std::string app_storage_path, + Settings& settings, + const std::string& shorebird_yaml, + const std::string& version, + const std::string& version_code); + +/// Used for reading app_id from shorebird.yaml. +/// Exposed for testing. +std::string GetValueFromYaml(const std::string& yaml, const std::string& key); + +} // namespace flutter + +#endif // FLUTTER_SHELL_COMMON_SHOREBIRD_SHOREBIRD_H_ diff --git a/engine/src/flutter/shell/common/shorebird/shorebird_unittests.cc b/engine/src/flutter/shell/common/shorebird/shorebird_unittests.cc new file mode 100644 index 0000000000000..7c108ac1e6231 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/shorebird_unittests.cc @@ -0,0 +1,21 @@ +#include "flutter/shell/common/shorebird/shorebird.h" + +#include "gtest/gtest.h" + +namespace flutter { +namespace testing { +TEST(Shorebird, GetValueFromYamlValueExists) { + std::string yaml = "appid: com.example.app\nversion: 1.0.0\n"; + std::string key = "appid"; + std::string value = GetValueFromYaml(yaml, key); + EXPECT_EQ(value, "com.example.app"); +} + +TEST(Shorebird, GetValueFromYamlValueDoesNotExist) { + std::string yaml = "appid: com.example.app\nversion: 1.0.0\n"; + std::string key = "appid2"; + std::string value = GetValueFromYaml(yaml, key); + EXPECT_EQ(value, ""); +} +} // namespace testing +} // namespace flutter \ No newline at end of file diff --git a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc new file mode 100644 index 0000000000000..0c6c5a45450a1 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.cc @@ -0,0 +1,144 @@ +#include "flutter/shell/common/shorebird/snapshots_data_handle.h" + +#include "third_party/dart/runtime/include/dart_native_api.h" + +namespace flutter { + +static std::unique_ptr DataMapping(const DartSnapshot& snapshot) { + auto ptr = snapshot.GetDataMapping(); + return std::make_unique(ptr, + Dart_SnapshotDataSize(ptr)); +} + +static std::unique_ptr InstructionsMapping( + const DartSnapshot& snapshot) { + auto ptr = snapshot.GetInstructionsMapping(); + return std::make_unique(ptr, + Dart_SnapshotInstrSize(ptr)); +} + +// The size of the snapshot data is the sum of the sizes of the blobs. +size_t SnapshotsDataHandle::FullSize() const { + size_t size = 0; + for (const auto& blob : blobs_) { + size += blob->GetSize(); + } + return size; +} + +// The offset into the snapshots data blobs as though they were a single +// contiguous buffer. +size_t SnapshotsDataHandle::AbsoluteOffsetForIndex(BlobsIndex index) { + if (index.blob >= blobs_.size()) { + FML_LOG(WARNING) << "Blob index " << index.blob + << " is larger than the number of blobs (" << blobs_.size() + << "). Returning full size (" << FullSize() << ")"; + return FullSize(); + } + if (index.offset > blobs_[index.blob]->GetSize()) { + FML_LOG(WARNING) << "Offset for blob " << index.blob << " (" << index.offset + << ") is larger than the blob size (" + << blobs_[index.blob]->GetSize() + << "). Returning index start of next blob"; + return AbsoluteOffsetForIndex({index.blob + 1, 0}); + } + size_t offset = 0; + for (size_t i = 0; i < index.blob; i++) { + offset += blobs_[i]->GetSize(); + } + offset += index.offset; + return offset; +} + +BlobsIndex SnapshotsDataHandle::IndexForAbsoluteOffset(int64_t offset, + BlobsIndex start_index) { + size_t start_offset = AbsoluteOffsetForIndex(start_index); + if (offset < 0) { + if ((size_t)abs(offset) > start_offset) { + FML_LOG(WARNING) + << "Offset is before the beginning of SnapshotsData. Returning 0, 0"; + return {0, 0}; + } + } else if (offset + start_offset >= FullSize()) { + FML_LOG(WARNING) << "Target offset is past the end of SnapshotsData (" + << offset + start_offset << ", blobs size:" << FullSize() + << "). Returning last blob index and offset"; + return {blobs_.size(), blobs_.back()->GetSize()}; + } + + size_t dest_offset = start_offset + offset; + BlobsIndex index = {0, 0}; + for (const auto& blob : blobs_) { + if (dest_offset < blob->GetSize()) { + // The remaining offset is within this blob. + index.offset = dest_offset; + break; + } + + index.blob++; + dest_offset -= blob->GetSize(); + } + return index; +} + +std::unique_ptr SnapshotsDataHandle::createForSnapshots( + const DartSnapshot& vm_snapshot, + const DartSnapshot& isolate_snapshot) { + // This needs to match the order in which the blobs are written out in + // analyze_snapshot --dump_blobs + std::vector> blobs; + blobs.push_back(DataMapping(vm_snapshot)); + blobs.push_back(DataMapping(isolate_snapshot)); + blobs.push_back(InstructionsMapping(vm_snapshot)); + blobs.push_back(InstructionsMapping(isolate_snapshot)); + return std::make_unique(std::move(blobs)); +} + +uintptr_t SnapshotsDataHandle::Read(uint8_t* buffer, uintptr_t length) { + uintptr_t bytes_read = 0; + // Copy current blob from current offset and possibly into the next blob + // until we have read length bytes. + while (bytes_read < length) { + if (current_index_.blob >= blobs_.size()) { + // We have read all blobs. + break; + } + intptr_t remaining_blob_length = + blobs_[current_index_.blob]->GetSize() - current_index_.offset; + if (remaining_blob_length <= 0) { + // We have read all bytes in this blob. + current_index_.blob++; + current_index_.offset = 0; + continue; + } + intptr_t bytes_to_read = fmin(length - bytes_read, remaining_blob_length); + memcpy(buffer + bytes_read, + blobs_[current_index_.blob]->GetMapping() + current_index_.offset, + bytes_to_read); + bytes_read += bytes_to_read; + current_index_.offset += bytes_to_read; + } + + return bytes_read; +} + +int64_t SnapshotsDataHandle::Seek(int64_t offset, int32_t whence) { + BlobsIndex start_index; + switch (whence) { + case SEEK_CUR: + start_index = current_index_; + break; + case SEEK_SET: + start_index = {0, 0}; + break; + case SEEK_END: + start_index = {blobs_.size(), blobs_.back()->GetSize()}; + break; + default: + FML_CHECK(false) << "Unrecognized whence value in Seek: " << whence; + } + current_index_ = IndexForAbsoluteOffset(offset, start_index); + return current_index_.offset; +} + +} // namespace flutter \ No newline at end of file diff --git a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.h b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.h new file mode 100644 index 0000000000000..50c4b8179b412 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle.h @@ -0,0 +1,48 @@ +#ifndef FLUTTER_SHELL_COMMON_SHOREBIRD_SNAPSHOTS_DATA_HANDLE_H_ +#define FLUTTER_SHELL_COMMON_SHOREBIRD_SNAPSHOTS_DATA_HANDLE_H_ + +#include +#include "flutter/fml/file.h" +#include "flutter/runtime/dart_snapshot.h" +#include "third_party/dart/runtime/include/dart_tools_api.h" + +namespace flutter { + +// An offset into an indexed collection of buffers. blob is the index of the +// buffer, and offset is the offset into that buffer. +struct BlobsIndex { + size_t blob; + size_t offset; +}; + +// Implements a POSIX file I/O interface which allows us to provide the four +// data blobs of a Dart snapshot (vm_data, vm_instructions, isolate_data, +// isolate_instructions) to Rust as though it were a single piece of memory. +class SnapshotsDataHandle { + public: + // This would ideally be private, but we need to be able to call it from the + // static createForSnapshots method. + explicit SnapshotsDataHandle(std::vector> blobs) + : blobs_(std::move(blobs)) {} + + static std::unique_ptr createForSnapshots( + const DartSnapshot& vm_snapshot, + const DartSnapshot& isolate_snapshot); + + uintptr_t Read(uint8_t* buffer, uintptr_t length); + int64_t Seek(int64_t offset, int32_t whence); + + // The sum of all the blobs' sizes. + size_t FullSize() const; + + private: + size_t AbsoluteOffsetForIndex(BlobsIndex index); + BlobsIndex IndexForAbsoluteOffset(int64_t offset, BlobsIndex startIndex); + + BlobsIndex current_index_ = {0, 0}; + std::vector> blobs_; +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_COMMON_SHOREBIRD_SNAPSHOTS_DATA_HANDLE_H_ diff --git a/engine/src/flutter/shell/common/shorebird/snapshots_data_handle_unittests.cc b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle_unittests.cc new file mode 100644 index 0000000000000..2acf44a7b8973 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/snapshots_data_handle_unittests.cc @@ -0,0 +1,177 @@ +#include +#include +#include + +#include "flutter/shell/common/shorebird/snapshots_data_handle.h" + +#include "flutter/fml/mapping.h" +#include "flutter/runtime/dart_snapshot.h" +#include "flutter/testing/testing.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "testing/fixture_test.h" + +namespace flutter { +namespace testing { + +std::unique_ptr MakeHandle( + std::vector& blobs) { + // Map the strings into non-owned mappings: + std::vector> mappings = {}; + for (auto& blob : blobs) { + std::unique_ptr mapping = + std::make_unique( + reinterpret_cast(blob.data()), blob.size()); + mappings.push_back(std::move(mapping)); + } + auto handle = + std::make_unique(std::move(mappings)); + return handle; +} + +TEST(SnapshotsDataHandle, Read) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 12; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + blobs_handle->Read(buffer, 6); + + EXPECT_EQ(buffer[0], 'a'); + EXPECT_EQ(buffer[1], 'b'); + EXPECT_EQ(buffer[2], 'c'); + EXPECT_EQ(buffer[3], 'd'); + EXPECT_EQ(buffer[4], 'e'); + EXPECT_EQ(buffer[5], 'f'); + + // Only the first 6 bytes should have been read. + EXPECT_EQ(buffer[6], 0); +} + +TEST(SnapshotsDataHandle, ReadAfterSeekWithPositiveOffset) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + blobs_handle->Seek(4, SEEK_CUR); + blobs_handle->Read(buffer, 6); + + EXPECT_EQ(buffer[0], 'e'); + EXPECT_EQ(buffer[1], 'f'); + EXPECT_EQ(buffer[2], 'g'); + EXPECT_EQ(buffer[3], 'h'); + EXPECT_EQ(buffer[4], 'i'); + EXPECT_EQ(buffer[5], 'j'); + + // Only the first 6 bytes should have been read. + EXPECT_EQ(buffer[6], 0); +} + +TEST(SnapshotsDataHandle, ReadAfterSeekWithNegativeOffset) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + blobs_handle->Read(buffer, 5); + EXPECT_EQ(buffer[0], 'a'); + EXPECT_EQ(buffer[1], 'b'); + EXPECT_EQ(buffer[2], 'c'); + EXPECT_EQ(buffer[3], 'd'); + EXPECT_EQ(buffer[4], 'e'); + EXPECT_EQ(buffer[5], 0); + + // Reset buffer + std::fill(buffer, buffer + buffer_size, 0); + + // Read 5, seeked back 4, should start reading at offset 1 ('b') + blobs_handle->Seek(-4, SEEK_CUR); + blobs_handle->Read(buffer, 6); + + EXPECT_EQ(buffer[0], 'b'); + EXPECT_EQ(buffer[1], 'c'); + EXPECT_EQ(buffer[2], 'd'); + EXPECT_EQ(buffer[3], 'e'); + EXPECT_EQ(buffer[4], 'f'); + EXPECT_EQ(buffer[5], 'g'); + EXPECT_EQ(buffer[6], 0); +} + +TEST(SnapshotsDataHandle, SeekPastEnd) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + // Seek 1 past the end + blobs_handle->Seek(blobs_handle->FullSize() + 1, SEEK_CUR); + + // Seek back 2 bytes and read 2 bytes + blobs_handle->Seek(-2, SEEK_CUR); + blobs_handle->Read(buffer, 2); + + EXPECT_EQ(buffer[0], 'k'); + EXPECT_EQ(buffer[1], 'l'); +} + +TEST(SnapshotsDataHandle, SeekBeforeBeginning) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + // Seek before the start of the blobs and read the first 2 bytes. + blobs_handle->Seek(-2, SEEK_CUR); + blobs_handle->Read(buffer, 2); + + EXPECT_EQ(buffer[0], 'a'); + EXPECT_EQ(buffer[1], 'b'); +} + +TEST(SnapshotsDataHandle, SeekFromBeginning) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + // Seek 10 bytes from current (the beginning) + blobs_handle->Seek(10, SEEK_CUR); + + // Seek 2 bytes from the beginning and read 2 bytes + blobs_handle->Seek(2, SEEK_SET); + blobs_handle->Read(buffer, 2); + + EXPECT_EQ(buffer[0], 'c'); + EXPECT_EQ(buffer[1], 'd'); +} + +TEST(SnapshotsDataHandle, SeekFromEnd) { + std::vector blobs = {"abc", "def", "ghi", "jkl"}; + std::unique_ptr blobs_handle = MakeHandle(blobs); + + const size_t buffer_size = 20; + uint8_t buffer[buffer_size]; + std::fill(buffer, buffer + buffer_size, 0); + + // Seek 2 bytes from the end and read 2 bytes + blobs_handle->Seek(-2, SEEK_END); + blobs_handle->Read(buffer, 2); + + EXPECT_EQ(buffer[0], 'k'); + EXPECT_EQ(buffer[1], 'l'); +} + +} // namespace testing +} // namespace flutter diff --git a/engine/src/flutter/shell/common/shorebird/updater.cc b/engine/src/flutter/shell/common/shorebird/updater.cc new file mode 100644 index 0000000000000..63d993613c3f9 --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/updater.cc @@ -0,0 +1,202 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/common/shorebird/updater.h" + +#include "flutter/fml/logging.h" + +#if SHOREBIRD_PLATFORM_SUPPORTED +#include "third_party/updater/library/include/updater.h" +#endif + +namespace flutter { +namespace shorebird { + +// Static member definitions +std::unique_ptr Updater::instance_; +std::mutex Updater::instance_mutex_; +std::atomic Updater::launch_started_{false}; +std::atomic Updater::launch_completed_{false}; + +Updater& Updater::Instance() { + std::lock_guard lock(instance_mutex_); + if (!instance_) { +#if SHOREBIRD_PLATFORM_SUPPORTED + instance_ = std::make_unique(); +#else + instance_ = std::make_unique(); +#endif + } + return *instance_; +} + +void Updater::SetInstanceForTesting(std::unique_ptr instance) { + std::lock_guard lock(instance_mutex_); + instance_ = std::move(instance); +} + +void Updater::ResetInstanceForTesting() { + std::lock_guard lock(instance_mutex_); + instance_.reset(); +} + +void Updater::ResetLaunchStateForTesting() { + launch_started_.store(false); + launch_completed_.store(false); +} + +void Updater::ReportLaunchStart() { + // Guard: only the first engine in a process should promote next_boot โ†’ + // current_boot in the Rust updater. See class-level comment for rationale. + bool expected = false; + if (!launch_started_.compare_exchange_strong(expected, true)) { + return; + } + DoReportLaunchStart(); +} + +void Updater::ReportLaunchSuccess() { + // Guard: only report success once per process. Subsequent engines reuse + // the same patch and don't need to re-confirm the boot. + bool expected = false; + if (!launch_completed_.compare_exchange_strong(expected, true)) { + return; + } + DoReportLaunchSuccess(); +} + +void Updater::ReportLaunchFailure() { + // Guard: only report failure once per process. + bool expected = false; + if (!launch_completed_.compare_exchange_strong(expected, true)) { + return; + } + DoReportLaunchFailure(); +} + +#if SHOREBIRD_PLATFORM_SUPPORTED +// RealUpdater implementation - wraps the Rust C API + +bool RealUpdater::Init(const AppConfig& config) { + // Convert paths to C strings + std::vector c_paths; + c_paths.reserve(config.original_libapp_paths.size()); + for (const auto& path : config.original_libapp_paths) { + c_paths.push_back(path.c_str()); + } + + AppParameters params; + params.release_version = config.release_version.c_str(); + params.original_libapp_paths = c_paths.data(); + params.original_libapp_paths_size = static_cast(c_paths.size()); + params.app_storage_dir = config.app_storage_dir.c_str(); + params.code_cache_dir = config.code_cache_dir.c_str(); + + // Convert our FileCallbacks to the Rust struct + ::FileCallbacks rust_callbacks; + rust_callbacks.open = config.file_callbacks.open; + rust_callbacks.read = config.file_callbacks.read; + rust_callbacks.seek = config.file_callbacks.seek; + rust_callbacks.close = config.file_callbacks.close; + + return shorebird_init(¶ms, rust_callbacks, config.yaml_config.c_str()); +} + +void RealUpdater::ValidateNextBootPatch() { + shorebird_validate_next_boot_patch(); +} + +std::string RealUpdater::NextBootPatchPath() { + char* c_path = shorebird_next_boot_patch_path(); + if (c_path == nullptr) { + return ""; + } + std::string path(c_path); + shorebird_free_string(c_path); + return path; +} + +void RealUpdater::DoReportLaunchStart() { + shorebird_report_launch_start(); +} + +void RealUpdater::DoReportLaunchSuccess() { + shorebird_report_launch_success(); +} + +void RealUpdater::DoReportLaunchFailure() { + shorebird_report_launch_failure(); +} + +bool RealUpdater::ShouldAutoUpdate() { + return shorebird_should_auto_update(); +} + +void RealUpdater::StartUpdateThread() { + shorebird_start_update_thread(); +} +#endif // SHOREBIRD_PLATFORM_SUPPORTED + +// MockUpdater implementation - for testing + +bool MockUpdater::Init(const AppConfig& config) { + init_count_++; + last_release_version_ = config.release_version; + last_yaml_config_ = config.yaml_config; + call_log_.push_back("Init"); + return init_result_; +} + +void MockUpdater::ValidateNextBootPatch() { + validate_count_++; + call_log_.push_back("ValidateNextBootPatch"); +} + +std::string MockUpdater::NextBootPatchPath() { + call_log_.push_back("NextBootPatchPath"); + return next_boot_patch_path_; +} + +void MockUpdater::DoReportLaunchStart() { + launch_start_count_++; + call_log_.push_back("ReportLaunchStart"); +} + +void MockUpdater::DoReportLaunchSuccess() { + launch_success_count_++; + call_log_.push_back("ReportLaunchSuccess"); +} + +void MockUpdater::DoReportLaunchFailure() { + launch_failure_count_++; + call_log_.push_back("ReportLaunchFailure"); +} + +bool MockUpdater::ShouldAutoUpdate() { + call_log_.push_back("ShouldAutoUpdate"); + return should_auto_update_; +} + +void MockUpdater::StartUpdateThread() { + start_update_thread_count_++; + call_log_.push_back("StartUpdateThread"); +} + +void MockUpdater::Reset() { + init_count_ = 0; + validate_count_ = 0; + launch_start_count_ = 0; + launch_success_count_ = 0; + launch_failure_count_ = 0; + start_update_thread_count_ = 0; + init_result_ = true; + should_auto_update_ = false; + next_boot_patch_path_.clear(); + last_release_version_.clear(); + last_yaml_config_.clear(); + call_log_.clear(); +} + +} // namespace shorebird +} // namespace flutter diff --git a/engine/src/flutter/shell/common/shorebird/updater.h b/engine/src/flutter/shell/common/shorebird/updater.h new file mode 100644 index 0000000000000..aeb2d1f2d90cc --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/updater.h @@ -0,0 +1,230 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_COMMON_SHOREBIRD_UPDATER_H_ +#define FLUTTER_SHELL_COMMON_SHOREBIRD_UPDATER_H_ + +#include +#include +#include +#include +#include +#include +#include + +namespace flutter { +namespace shorebird { + +/// File callbacks for iOS patch loading. +/// Mirrors the FileCallbacks struct from the Rust updater. +struct FileCallbacks { + void* (*open)(void); + uintptr_t (*read)(void* file_handle, uint8_t* buffer, uintptr_t count); + int64_t (*seek)(void* file_handle, int64_t offset, int32_t whence); + void (*close)(void* file_handle); +}; + +/// Configuration for initializing the Shorebird updater. +struct AppConfig { + /// Version string for this release (e.g., "1.0.0+1"). + std::string release_version; + + /// Paths to the original AOT libraries (libapp.so on Android, App.framework + /// on iOS). + std::vector original_libapp_paths; + + /// Directory for persistent updater state (survives app updates). + std::string app_storage_dir; + + /// Directory for cached artifacts (cleared on app updates). + std::string code_cache_dir; + + /// Callbacks for iOS patch file access (can be null callbacks on Android). + FileCallbacks file_callbacks; + + /// YAML configuration from shorebird.yaml. + std::string yaml_config; +}; + +/// Abstract interface for the Shorebird updater. +/// +/// This abstraction allows for: +/// 1. Mocking in tests without requiring the real Rust library +/// 2. Future migration from Rust to C++ implementation +/// 3. Test instrumentation (call counting, logging) +/// +/// ## Launch lifecycle (start/success/failure) +/// +/// The Rust updater uses a start/success/failure protocol to detect crashes: +/// - `ReportLaunchStart` copies `next_boot` โ†’ `current_boot` in the Rust +/// state. If the app crashes before `ReportLaunchSuccess`, the updater +/// assumes the patch caused the crash and rolls back on the next launch. +/// +/// These calls are guarded to execute at most once per process because: +/// 1. The Rust updater is a process-global singleton โ€” calling +/// `report_launch_start` multiple times would repeatedly copy `next_boot` +/// โ†’ `current_boot`, which could promote a newly-downloaded (but not yet +/// booted) patch to "current" even though the running engine loaded the +/// old snapshot. +/// 2. In add-to-app, multiple FlutterEngines may be created and destroyed +/// within a single process. Each engine creation resolves snapshots and +/// constructs a Shell, but we must only report launch start/success once +/// โ€” for the first engine that actually boots. Without this guard, a +/// background update that completes between engine creations would get +/// promoted to "current" by the second engine's `ReportLaunchStart`, +/// even though that engine is still running the old snapshot. +/// +/// Tests can call `ResetLaunchStateForTesting()` to re-enable the guards. +class Updater { + public: + virtual ~Updater() = default; + + /// Initialize the updater with configuration. + /// @param config Configuration containing release version, paths, and + /// callbacks + /// @return true if initialization succeeded + virtual bool Init(const AppConfig& config) = 0; + + /// Validate the next boot patch. If invalid, falls back to last good state. + virtual void ValidateNextBootPatch() = 0; + + /// Get the path to the patch that will boot on next run. + /// @return Path to patch, or empty string if no patch available + virtual std::string NextBootPatchPath() = 0; + + // Boot lifecycle methods โ€” guarded to run at most once per process. + // Callers may call these freely; subsequent calls after the first are + // silently ignored. + void ReportLaunchStart(); + void ReportLaunchSuccess(); + void ReportLaunchFailure(); + + // Update checking + virtual bool ShouldAutoUpdate() = 0; + virtual void StartUpdateThread() = 0; + + // Singleton access + static Updater& Instance(); + + // Test support - allows injecting a mock implementation + static void SetInstanceForTesting(std::unique_ptr instance); + static void ResetInstanceForTesting(); + + /// Resets the once-per-process launch guards so tests can verify + /// start/success/failure calls on fresh Updater instances. + static void ResetLaunchStateForTesting(); + + protected: + Updater() = default; + + // Subclass hooks โ€” called by the public guarded methods above. + virtual void DoReportLaunchStart() = 0; + virtual void DoReportLaunchSuccess() = 0; + virtual void DoReportLaunchFailure() = 0; + + private: + static std::unique_ptr instance_; + static std::mutex instance_mutex_; + + // Once-per-process guards for launch lifecycle. + static std::atomic launch_started_; + static std::atomic launch_completed_; +}; + +/// No-op implementation for unsupported platforms. +/// All methods are safe to call but do nothing. +class NoOpUpdater : public Updater { + public: + NoOpUpdater() = default; + ~NoOpUpdater() override = default; + + bool Init(const AppConfig& config) override { return true; } + void ValidateNextBootPatch() override {} + std::string NextBootPatchPath() override { return ""; } + void DoReportLaunchStart() override {} + void DoReportLaunchSuccess() override {} + void DoReportLaunchFailure() override {} + bool ShouldAutoUpdate() override { return false; } + void StartUpdateThread() override {} +}; + +#if SHOREBIRD_PLATFORM_SUPPORTED +/// Production implementation that wraps the Rust updater C API. +/// Only available on supported platforms (Android, iOS, macOS, Windows, Linux). +class RealUpdater : public Updater { + public: + RealUpdater() = default; + ~RealUpdater() override = default; + + bool Init(const AppConfig& config) override; + void ValidateNextBootPatch() override; + std::string NextBootPatchPath() override; + void DoReportLaunchStart() override; + void DoReportLaunchSuccess() override; + void DoReportLaunchFailure() override; + bool ShouldAutoUpdate() override; + void StartUpdateThread() override; +}; +#endif // SHOREBIRD_PLATFORM_SUPPORTED + +/// Mock implementation for testing. +/// Tracks call counts and can be queried to verify behavior. +class MockUpdater : public Updater { + public: + MockUpdater() = default; + ~MockUpdater() override = default; + + bool Init(const AppConfig& config) override; + void ValidateNextBootPatch() override; + std::string NextBootPatchPath() override; + void DoReportLaunchStart() override; + void DoReportLaunchSuccess() override; + void DoReportLaunchFailure() override; + bool ShouldAutoUpdate() override; + void StartUpdateThread() override; + + // Test accessors + int init_count() const { return init_count_; } + int validate_count() const { return validate_count_; } + int launch_start_count() const { return launch_start_count_; } + int launch_success_count() const { return launch_success_count_; } + int launch_failure_count() const { return launch_failure_count_; } + int start_update_thread_count() const { return start_update_thread_count_; } + const std::vector& call_log() const { return call_log_; } + + // Last init parameters (for verification) + const std::string& last_release_version() const { + return last_release_version_; + } + const std::string& last_yaml_config() const { return last_yaml_config_; } + + // Test configuration + void set_init_result(bool value) { init_result_ = value; } + void set_should_auto_update(bool value) { should_auto_update_ = value; } + void set_next_boot_patch_path(const std::string& path) { + next_boot_patch_path_ = path; + } + + // Reset all counters and logs + void Reset(); + + private: + int init_count_ = 0; + int validate_count_ = 0; + int launch_start_count_ = 0; + int launch_success_count_ = 0; + int launch_failure_count_ = 0; + int start_update_thread_count_ = 0; + bool init_result_ = true; + bool should_auto_update_ = false; + std::string next_boot_patch_path_; + std::string last_release_version_; + std::string last_yaml_config_; + std::vector call_log_; +}; + +} // namespace shorebird +} // namespace flutter + +#endif // FLUTTER_SHELL_COMMON_SHOREBIRD_UPDATER_H_ diff --git a/engine/src/flutter/shell/common/shorebird/updater_unittests.cc b/engine/src/flutter/shell/common/shorebird/updater_unittests.cc new file mode 100644 index 0000000000000..def93050e885d --- /dev/null +++ b/engine/src/flutter/shell/common/shorebird/updater_unittests.cc @@ -0,0 +1,183 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/common/shorebird/updater.h" + +#include "gtest/gtest.h" + +namespace flutter { +namespace shorebird { +namespace testing { + +class UpdaterTest : public ::testing::Test { + protected: + void SetUp() override { + // Install a mock for each test and reset the once-per-process guards + // so each test starts with a clean slate. + auto mock = std::make_unique(); + mock_ = mock.get(); + Updater::SetInstanceForTesting(std::move(mock)); + Updater::ResetLaunchStateForTesting(); + } + + void TearDown() override { + mock_ = nullptr; + Updater::ResetInstanceForTesting(); + Updater::ResetLaunchStateForTesting(); + } + + MockUpdater* mock_ = nullptr; +}; + +// ReportLaunchStart is guarded to run at most once per process. +// The second call should be silently ignored. +TEST_F(UpdaterTest, ReportLaunchStartOnlyCallsOnce) { + EXPECT_EQ(mock_->launch_start_count(), 0); + + Updater::Instance().ReportLaunchStart(); + EXPECT_EQ(mock_->launch_start_count(), 1); + + // Second call is a no-op due to the once-per-process guard. + Updater::Instance().ReportLaunchStart(); + EXPECT_EQ(mock_->launch_start_count(), 1); +} + +TEST_F(UpdaterTest, ReportLaunchSuccessOnlyCallsOnce) { + EXPECT_EQ(mock_->launch_success_count(), 0); + + Updater::Instance().ReportLaunchSuccess(); + EXPECT_EQ(mock_->launch_success_count(), 1); + + // Second call is a no-op. + Updater::Instance().ReportLaunchSuccess(); + EXPECT_EQ(mock_->launch_success_count(), 1); +} + +TEST_F(UpdaterTest, ReportLaunchFailureOnlyCallsOnce) { + EXPECT_EQ(mock_->launch_failure_count(), 0); + + Updater::Instance().ReportLaunchFailure(); + EXPECT_EQ(mock_->launch_failure_count(), 1); + + // Second call is a no-op. + Updater::Instance().ReportLaunchFailure(); + EXPECT_EQ(mock_->launch_failure_count(), 1); +} + +TEST_F(UpdaterTest, MockUpdaterTracksShouldAutoUpdate) { + mock_->set_should_auto_update(false); + EXPECT_FALSE(Updater::Instance().ShouldAutoUpdate()); + + mock_->set_should_auto_update(true); + EXPECT_TRUE(Updater::Instance().ShouldAutoUpdate()); +} + +TEST_F(UpdaterTest, MockUpdaterTracksStartUpdateThreadCalls) { + EXPECT_EQ(mock_->start_update_thread_count(), 0); + + Updater::Instance().StartUpdateThread(); + EXPECT_EQ(mock_->start_update_thread_count(), 1); +} + +TEST_F(UpdaterTest, MockUpdaterCallLogRecordsSequence) { + EXPECT_TRUE(mock_->call_log().empty()); + + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ShouldAutoUpdate(); + Updater::Instance().ReportLaunchSuccess(); + + const auto& log = mock_->call_log(); + ASSERT_EQ(log.size(), 3u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ShouldAutoUpdate"); + EXPECT_EQ(log[2], "ReportLaunchSuccess"); +} + +TEST_F(UpdaterTest, MockUpdaterResetClearsState) { + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + mock_->set_should_auto_update(true); + + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_success_count(), 1); + EXPECT_TRUE(mock_->ShouldAutoUpdate()); + + mock_->Reset(); + + EXPECT_EQ(mock_->launch_start_count(), 0); + EXPECT_EQ(mock_->launch_success_count(), 0); + // Check call_log before ShouldAutoUpdate() since the method adds to call_log + EXPECT_TRUE(mock_->call_log().empty()); + EXPECT_FALSE(mock_->ShouldAutoUpdate()); +} + +// ReportLaunchStart and ReportLaunchSuccess are paired once per process. +// The Rust updater no-ops both when no patch is booting. +TEST_F(UpdaterTest, LaunchStartAndSuccessArePairedOncePerProcess) { + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_success_count(), 1); + const auto& log = mock_->call_log(); + ASSERT_EQ(log.size(), 2u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchSuccess"); +} + +// ReportLaunchStart and ReportLaunchFailure are paired once per process. +TEST_F(UpdaterTest, LaunchStartAndFailureArePairedOncePerProcess) { + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchFailure(); + + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_failure_count(), 1); + const auto& log = mock_->call_log(); + ASSERT_EQ(log.size(), 2u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchFailure"); +} + +// Simulates the add-to-app scenario: multiple engines call ReportLaunchStart +// and ReportLaunchSuccess, but only the first should actually reach the +// updater. This prevents the Rust updater from promoting a newly-downloaded +// patch to "current_boot" when subsequent engines are still running the +// original snapshot. +TEST_F(UpdaterTest, MultipleEnginesOnlyReportOnce) { + // First engine boots. + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + + // Second engine boots โ€” these should be no-ops. + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_success_count(), 1); + + const auto& log = mock_->call_log(); + ASSERT_EQ(log.size(), 2u); + EXPECT_EQ(log[0], "ReportLaunchStart"); + EXPECT_EQ(log[1], "ReportLaunchSuccess"); +} + +// ResetLaunchStateForTesting re-enables the guards, allowing tests to +// verify launch calls on a fresh state. +TEST_F(UpdaterTest, ResetLaunchStateReenablesGuards) { + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + EXPECT_EQ(mock_->launch_start_count(), 1); + EXPECT_EQ(mock_->launch_success_count(), 1); + + Updater::ResetLaunchStateForTesting(); + + Updater::Instance().ReportLaunchStart(); + Updater::Instance().ReportLaunchSuccess(); + EXPECT_EQ(mock_->launch_start_count(), 2); + EXPECT_EQ(mock_->launch_success_count(), 2); +} + +} // namespace testing +} // namespace shorebird +} // namespace flutter diff --git a/engine/src/flutter/shell/platform/android/BUILD.gn b/engine/src/flutter/shell/platform/android/BUILD.gn index 90af9ceb17ac2..c3b50e380bacf 100644 --- a/engine/src/flutter/shell/platform/android/BUILD.gn +++ b/engine/src/flutter/shell/platform/android/BUILD.gn @@ -171,6 +171,7 @@ source_set("flutter_shell_native_src") { "//flutter/runtime", "//flutter/runtime:libdart", "//flutter/shell/common", + "//flutter/shell/common/shorebird", "//flutter/shell/platform/android/context", "//flutter/shell/platform/android/external_view_embedder", "//flutter/shell/platform/android/jni", @@ -809,8 +810,10 @@ if (target_cpu != "x86") { } } -if (host_os == "linux" && - (target_cpu == "x64" || target_cpu == "arm64" || target_cpu == "riscv64")) { +# Build analyze_snapshot for 64-bit target CPUs on linux. +# Or always targeting arm64 for Shorebird builds. +if ((host_os == "linux" && (target_cpu == "x64" || target_cpu == "riscv64")) || + target_cpu == "arm64") { zip_bundle("analyze_snapshot") { deps = [ "$dart_src/runtime/bin:analyze_snapshot($host_toolchain)" ] diff --git a/engine/src/flutter/shell/platform/android/android_exports.lst b/engine/src/flutter/shell/platform/android/android_exports.lst index 198bff773dd74..9b924e7738cd4 100644 --- a/engine/src/flutter/shell/platform/android/android_exports.lst +++ b/engine/src/flutter/shell/platform/android/android_exports.lst @@ -11,6 +11,18 @@ _binary_icudtl_dat_size; InternalFlutterGpu*; kInternalFlutterGpu*; + shorebird_init; + shorebird_active_path; + shorebird_active_patch_number; + shorebird_free_string; + shorebird_free_update_result; + shorebird_check_for_downloadable_update; + shorebird_check_for_update; + shorebird_update; + shorebird_update_with_result; + shorebird_next_boot_patch_number; + shorebird_current_boot_patch_number; + shorebird_validate_next_boot_patch; local: *; }; diff --git a/engine/src/flutter/shell/platform/android/flutter_main.cc b/engine/src/flutter/shell/platform/android/flutter_main.cc index d14ac40029645..45e369f5b3a9d 100644 --- a/engine/src/flutter/shell/platform/android/flutter_main.cc +++ b/engine/src/flutter/shell/platform/android/flutter_main.cc @@ -20,6 +20,7 @@ #include "flutter/fml/platform/android/paths_android.h" #include "flutter/lib/ui/plugins/callback_cache.h" #include "flutter/runtime/dart_vm.h" +#include "flutter/shell/common/shorebird/shorebird.h" #include "flutter/shell/common/switches.h" #include "flutter/shell/platform/android/android_context_vk_impeller.h" #include "flutter/shell/platform/android/android_rendering_selector.h" @@ -93,6 +94,9 @@ void FlutterMain::Init(JNIEnv* env, jstring kernelPath, jstring appStoragePath, jstring engineCachesPath, + jstring shorebirdYaml, + jstring version, + jstring versionCode, jlong initTimeMillis, jint api_level) { std::vector args; @@ -151,8 +155,18 @@ void FlutterMain::Init(JNIEnv* env, flutter::DartCallbackCache::SetCachePath( fml::jni::JavaStringToString(env, appStoragePath)); - fml::paths::InitializeAndroidCachesPath( - fml::jni::JavaStringToString(env, engineCachesPath)); + auto code_cache_path = fml::jni::JavaStringToString(env, engineCachesPath); + auto app_storage_path = fml::jni::JavaStringToString(env, appStoragePath); + fml::paths::InitializeAndroidCachesPath(code_cache_path); + +#if FLUTTER_RELEASE + std::string shorebird_yaml = fml::jni::JavaStringToString(env, shorebirdYaml); + std::string version_string = fml::jni::JavaStringToString(env, version); + std::string version_code_string = + fml::jni::JavaStringToString(env, versionCode); + ConfigureShorebird(code_cache_path, app_storage_path, settings, + shorebird_yaml, version_string, version_code_string); +#endif flutter::DartCallbackCache::LoadCacheFromDisk(); @@ -245,6 +259,7 @@ bool FlutterMain::Register(JNIEnv* env) { { .name = "nativeInit", .signature = "(Landroid/content/Context;[Ljava/lang/String;Ljava/" + "lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/" "lang/String;Ljava/lang/String;Ljava/lang/String;JI)V", .fnPtr = reinterpret_cast(&Init), }, diff --git a/engine/src/flutter/shell/platform/android/flutter_main.h b/engine/src/flutter/shell/platform/android/flutter_main.h index dda959801bc32..cf03f889ec30e 100644 --- a/engine/src/flutter/shell/platform/android/flutter_main.h +++ b/engine/src/flutter/shell/platform/android/flutter_main.h @@ -44,6 +44,9 @@ class FlutterMain { jstring kernelPath, jstring appStoragePath, jstring engineCachesPath, + jstring shorebirdYaml, + jstring version, + jstring versionCode, jlong initTimeMillis, jint api_level); diff --git a/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java b/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java index 14ffc8d541abf..ad7428e74f557 100644 --- a/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java +++ b/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java @@ -8,6 +8,8 @@ import android.annotation.SuppressLint; import android.content.Context; +import android.content.pm.PackageInfo; +import android.content.pm.PackageManager; import android.content.res.AssetManager; import android.graphics.Bitmap; import android.graphics.SurfaceTexture; @@ -42,6 +44,10 @@ import io.flutter.view.AccessibilityBridge; import io.flutter.view.FlutterCallbackInformation; import io.flutter.view.TextureRegistry; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.lang.ref.WeakReference; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -178,6 +184,9 @@ private static native void nativeInit( @Nullable String bundlePath, @NonNull String appStoragePath, @NonNull String engineCachesPath, + @Nullable String shorebirdYaml, + @Nullable String version, + @Nullable String versionCode, long initTimeMillis, int apiLevel); @@ -206,8 +215,47 @@ public void init( Log.w(TAG, "FlutterJNI.init called more than once"); } + String version = null; + String versionCode = null; + try { + PackageInfo packageInfo = + context.getPackageManager().getPackageInfo(context.getPackageName(), 0); + version = packageInfo.versionName; + if (Build.VERSION.SDK_INT >= API_LEVELS.API_28) { + versionCode = String.valueOf(packageInfo.getLongVersionCode()); + } else { + versionCode = String.valueOf(packageInfo.versionCode); + } + } catch (PackageManager.NameNotFoundException e) { + Log.e(TAG, "Failed to read app version. Shorebird updater can't run.", e); + } + + String shorebirdYaml = null; + try { + InputStream yaml = context.getAssets().open("flutter_assets/shorebird.yaml"); + BufferedReader r = new BufferedReader(new InputStreamReader(yaml)); + StringBuilder total = new StringBuilder(); + for (String line; (line = r.readLine()) != null; ) { + total.append(line).append('\n'); + } + shorebirdYaml = total.toString(); + Log.d(TAG, "shorebird.yaml: " + shorebirdYaml); + } catch (IOException e) { + Log.e(TAG, "Failed to load shorebird.yaml", e); + Log.e(TAG, "Did you remember to include shorebird.yaml in your pubspec.yaml's assets?"); + } + FlutterJNI.nativeInit( - context, args, bundlePath, appStoragePath, engineCachesPath, initTimeMillis, apiLevel); + context, + args, + bundlePath, + appStoragePath, + engineCachesPath, + shorebirdYaml, + version, + versionCode, + initTimeMillis, + apiLevel); FlutterJNI.initCalled = true; } diff --git a/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn b/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn index 911ee431e2516..777b6ae4a64bb 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn +++ b/engine/src/flutter/shell/platform/darwin/ios/BUILD.gn @@ -194,12 +194,14 @@ source_set("flutter_framework_source") { deps = [ ":ios_gpu_configuration", + "$dart_src/runtime/bin:elf_loader", "//flutter/common", "//flutter/common/graphics", "//flutter/fml", "//flutter/lib/ui", "//flutter/runtime", "//flutter/shell/common", + "//flutter/shell/common/shorebird", "//flutter/shell/platform/common:common_cpp_input", "//flutter/shell/platform/darwin/common", "//flutter/shell/platform/darwin/common:framework_common", diff --git a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterDartProject.mm b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterDartProject.mm index 31a49a856a4a5..45660d1cd637d 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterDartProject.mm +++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterDartProject.mm @@ -13,6 +13,8 @@ #include "flutter/common/constants.h" #include "flutter/fml/build_config.h" +#include "flutter/fml/paths.h" +#include "flutter/shell/common/shorebird/shorebird.h" #include "flutter/shell/common/switches.h" #import "flutter/shell/platform/darwin/common/InternalFlutterSwiftCommon/InternalFlutterSwiftCommon.h" #include "flutter/shell/platform/darwin/common/command_line.h" @@ -92,10 +94,12 @@ static BOOL DoesHardwareSupportWideGamut() { } if (flutter::DartVM::IsRunningPrecompiledCode()) { + NSLog(@"SANITY CHECK: Running precompiled code."); if (hasExplicitBundle) { NSString* executablePath = bundle.executablePath; if ([[NSFileManager defaultManager] fileExistsAtPath:executablePath]) { settings.application_library_paths.push_back(executablePath.UTF8String); + NSLog(@"Using precompiled library from %@", executablePath); } } @@ -107,6 +111,7 @@ static BOOL DoesHardwareSupportWideGamut() { NSString* executablePath = [NSBundle bundleWithPath:libraryPath].executablePath; if (executablePath.length > 0) { settings.application_library_paths.push_back(executablePath.UTF8String); + NSLog(@"Using library from %@", libraryPath); } } } @@ -121,6 +126,7 @@ static BOOL DoesHardwareSupportWideGamut() { [NSBundle bundleWithPath:applicationFrameworkPath].executablePath; if (executablePath.length > 0) { settings.application_library_paths.push_back(executablePath.UTF8String); + NSLog(@"Using App.framework from %@", applicationFrameworkPath); } } } @@ -152,6 +158,33 @@ static BOOL DoesHardwareSupportWideGamut() { } } + NSString* assetsPath = [NSString stringWithUTF8String:settings.assets_path.c_str()]; + NSLog(@"ASSET PATH %@", assetsPath); + + // FIXME: This may not be the correct path (e.g., should it include the organization id?) + // See + // https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html#//apple_ref/doc/uid/TP40010672-CH2-SW13 + // /private/var/mobile/Containers/Data/Application/264477BF-6E38-47C9-AAD9-532BB842F197/Library/Application + // Support/shorebird/shorebird_updater + std::string cache_path = + fml::paths::JoinPaths({getenv("HOME"), "Library/Application Support/shorebird"}); + NSURL* shorebirdYamlPath = [NSURL URLWithString:@"shorebird.yaml" + relativeToURL:[NSURL fileURLWithPath:assetsPath]]; + NSString* appVersion = [mainBundle objectForInfoDictionaryKey:@"CFBundleShortVersionString"]; + NSString* appBuildNumber = [mainBundle objectForInfoDictionaryKey:@"CFBundleVersion"]; + NSString* shorebirdYamlContents = [NSString stringWithContentsOfURL:shorebirdYamlPath + encoding:NSUTF8StringEncoding + error:nil]; + if (shorebirdYamlContents != nil) { + // Note: we intentionally pass cache_path twice. We provide two different directories + // to ConfigureShorebird because Android differentiates between data that persists + // between releases and data that does not. iOS does not make this distinction. + flutter::ConfigureShorebird(cache_path, cache_path, settings, shorebirdYamlContents.UTF8String, + appVersion.UTF8String, appBuildNumber.UTF8String); + } else { + NSLog(@"Failed to find shorebird.yaml, not starting updater."); + } + // Domain network configuration // Disabled in https://github.com/flutter/flutter/issues/72723. // Re-enable in https://github.com/flutter/flutter/issues/54448. diff --git a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm index 74fea45026de4..841bb7c60e6fc 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm +++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm @@ -39,6 +39,7 @@ #import "flutter/third_party/spring_animation/spring_animation.h" FLUTTER_ASSERT_ARC +#import static constexpr int kMicrosecondsPerSecond = 1000 * 1000; static constexpr CGFloat kScrollViewContentSize = 2.0; diff --git a/engine/src/flutter/shell/platform/darwin/macos/BUILD.gn b/engine/src/flutter/shell/platform/darwin/macos/BUILD.gn index bc69c85f797a0..6c5e1f4eb928b 100644 --- a/engine/src/flutter/shell/platform/darwin/macos/BUILD.gn +++ b/engine/src/flutter/shell/platform/darwin/macos/BUILD.gn @@ -144,12 +144,14 @@ source_set("flutter_framework_source") { ":macos_gpu_configuration", "//flutter/flow", "//flutter/fml", + "//flutter/shell/common/shorebird", "//flutter/shell/platform/common:common_cpp_accessibility", "//flutter/shell/platform/common:common_cpp_core", "//flutter/shell/platform/common:common_cpp_enums", "//flutter/shell/platform/common:common_cpp_input", "//flutter/shell/platform/common:common_cpp_isolate_scope", "//flutter/shell/platform/common:common_cpp_switches", + "//flutter/shell/platform/darwin/common", "//flutter/shell/platform/darwin/common:availability_version_check", "//flutter/shell/platform/darwin/common:framework_common", "//flutter/shell/platform/darwin/graphics", diff --git a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm index 8ba35a05761e3..a514611719a56 100644 --- a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm +++ b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm @@ -11,6 +11,8 @@ #include #include "flutter/common/constants.h" +#include "flutter/fml/paths.h" +#include "flutter/shell/common/shorebird/shorebird.h" #include "flutter/shell/platform/common/app_lifecycle_state.h" #include "flutter/shell/platform/common/engine_switches.h" #include "flutter/shell/platform/embedder/embedder.h" @@ -656,6 +658,40 @@ - (void)onFocusChangeRequest:(const FlutterViewFocusChangeRequest*)request { } } +- (BOOL)configureShorebird:(NSString**)patchPath { + NSLog(@"[shorebird] setting up non-linker shorebird"); + NSString* bundlePath = + [[NSBundle bundleWithURL:[NSBundle.mainBundle.privateFrameworksURL + URLByAppendingPathComponent:@"App.framework"]] bundlePath]; + bundlePath = [bundlePath stringByAppendingString:@"/App"]; + NSString* assetsPath = _project.assetsPath; + NSURL* shorebirdYamlPath = [NSURL URLWithString:@"shorebird.yaml" + relativeToURL:[NSURL fileURLWithPath:assetsPath]]; + NSString* shorebirdYamlContents = [NSString stringWithContentsOfURL:shorebirdYamlPath + encoding:NSUTF8StringEncoding + error:nil]; + NSString* appVersion = + [NSBundle.mainBundle objectForInfoDictionaryKey:@"CFBundleShortVersionString"]; + NSString* appBuildNumber = [NSBundle.mainBundle objectForInfoDictionaryKey:@"CFBundleVersion"]; + std::string cache_path = + fml::paths::JoinPaths({getenv("HOME"), "Library", "Application Support", "shorebird"}); + flutter::ReleaseVersion release_version = {appVersion.UTF8String, appBuildNumber.UTF8String}; + flutter::ShorebirdConfigArgs shorebird_args(cache_path, cache_path, bundlePath.UTF8String, + shorebirdYamlContents.UTF8String, release_version); + NSLog(@"[shorebird] calling ConfigureShorebird"); + std::string patch_path; + auto res = flutter::ConfigureShorebird(shorebird_args, patch_path); + if (!res) { + NSLog(@"[shorebird] ConfigureShorebird failed"); + return NO; + } + + NSLog(@"[shorebird] ConfigureShorebird success!"); + *patchPath = [NSString stringWithUTF8String:patch_path.c_str()]; + NSLog(@"[shorebird] patchPath: %@", *patchPath); + return YES; +} + - (BOOL)runWithEntrypoint:(NSString*)entrypoint { if (self.running) { return NO; @@ -753,7 +789,19 @@ - (BOOL)runWithEntrypoint:(NSString*)entrypoint { }; flutterArguments.custom_task_runners = &custom_task_runners; - [self loadAOTData:_project.assetsPath]; + NSString* elfPath; + BOOL configureShorebirdRes = [self configureShorebird:&elfPath]; + if (!configureShorebirdRes) { + // No patch exists, or we failed to configure shorebird. This is a fallback. + // Upstream, this code lives in -(void)loadAOTData:. + // + // This is the location where the test fixture places the snapshot file. + // For applications built by Flutter tool, this is in "App.framework". + elfPath = [NSString pathWithComponents:@[ _project.assetsPath, @"app_elf_snapshot.so" ]]; + } + + [self loadAOTData:elfPath]; + if (_aotData) { flutterArguments.aot_data = _aotData; } @@ -777,6 +825,7 @@ - (BOOL)runWithEntrypoint:(NSString*)entrypoint { }; FlutterRendererConfig rendererConfig = [_renderer createRendererConfig]; + FlutterEngineResult result = _embedderAPI.Initialize( FLUTTER_ENGINE_VERSION, &rendererConfig, &flutterArguments, (__bridge void*)(self), &_engine); if (result != kSuccess) { @@ -806,7 +855,7 @@ - (BOOL)runWithEntrypoint:(NSString*)entrypoint { return YES; } -- (void)loadAOTData:(NSString*)assetsDir { +- (void)loadAOTData:(NSString*)elfPath { if (!_embedderAPI.RunsAOTCompiledDartCode()) { return; } @@ -814,11 +863,8 @@ - (void)loadAOTData:(NSString*)assetsDir { BOOL isDirOut = false; // required for NSFileManager fileExistsAtPath. NSFileManager* fileManager = [NSFileManager defaultManager]; - // This is the location where the test fixture places the snapshot file. - // For applications built by Flutter tool, this is in "App.framework". - NSString* elfPath = [NSString pathWithComponents:@[ assetsDir, @"app_elf_snapshot.so" ]]; - if (![fileManager fileExistsAtPath:elfPath isDirectory:&isDirOut]) { + FML_LOG(INFO) << "in loadAOTData, elfPath does not exist: " << elfPath.UTF8String; return; } diff --git a/engine/src/flutter/shell/platform/linux/BUILD.gn b/engine/src/flutter/shell/platform/linux/BUILD.gn index b9b1fb6657b11..2039603cb0330 100644 --- a/engine/src/flutter/shell/platform/linux/BUILD.gn +++ b/engine/src/flutter/shell/platform/linux/BUILD.gn @@ -156,6 +156,7 @@ source_set("flutter_linux_sources") { "fl_settings_channel.cc", "fl_settings_handler.cc", "fl_settings_portal.cc", + "fl_shorebird.cc", "fl_socket_accessible.cc", "fl_standard_message_codec.cc", "fl_standard_method_codec.cc", @@ -184,12 +185,14 @@ source_set("flutter_linux_sources") { deps = [ "$dart_src/runtime:dart_api", "//flutter/fml", + "//flutter/shell/common/shorebird", "//flutter/shell/platform/common:common_cpp_enums", "//flutter/shell/platform/common:common_cpp_input", "//flutter/shell/platform/common:common_cpp_isolate_scope", "//flutter/shell/platform/common:common_cpp_switches", "//flutter/shell/platform/embedder:embedder_headers", "//flutter/third_party/rapidjson", + "//flutter/third_party/tonic", ] } diff --git a/engine/src/flutter/shell/platform/linux/fl_engine.cc b/engine/src/flutter/shell/platform/linux/fl_engine.cc index c1205d9974536..abb1701c3a95d 100644 --- a/engine/src/flutter/shell/platform/linux/fl_engine.cc +++ b/engine/src/flutter/shell/platform/linux/fl_engine.cc @@ -10,6 +10,7 @@ #include #include "flutter/common/constants.h" +#include "flutter/fml/logging.h" #include "flutter/shell/platform/common/engine_switches.h" #include "flutter/shell/platform/embedder/embedder.h" #include "flutter/shell/platform/linux/fl_accessibility_handler.h" @@ -24,6 +25,7 @@ #include "flutter/shell/platform/linux/fl_platform_handler.h" #include "flutter/shell/platform/linux/fl_plugin_registrar_private.h" #include "flutter/shell/platform/linux/fl_settings_handler.h" +#include "flutter/shell/platform/linux/fl_shorebird.h" #include "flutter/shell/platform/linux/fl_texture_gl_private.h" #include "flutter/shell/platform/linux/fl_texture_registrar_private.h" #include "flutter/shell/platform/linux/public/flutter_linux/fl_plugin_registry.h" @@ -781,6 +783,9 @@ gboolean fl_engine_start(FlEngine* self, GError** error) { g_autoptr(GPtrArray) command_line_args = g_ptr_array_new_with_free_func(g_free); + // FlutterProjectArgs expects a full argv, so when processing it for flags + // the first item is treated as the executable and ignored. Add a dummy + // value so that all switches are used. g_ptr_array_insert(command_line_args, 0, g_strdup("flutter")); for (const auto& env_switch : flutter::GetSwitchesFromEnvironment()) { g_ptr_array_add(command_line_args, g_strdup(env_switch.c_str())); @@ -818,9 +823,22 @@ gboolean fl_engine_start(FlEngine* self, GError** error) { args.compositor = &compositor; if (self->embedder_api.RunsAOTCompiledDartCode()) { + // This struct contains raw C strings and needs to have its lifetime scoped + // to this block. FlutterEngineAOTDataSource source = {}; source.type = kFlutterEngineAOTDataSourceTypeElfPath; - source.elf_path = fl_dart_project_get_aot_library_path(self->project); + std::string patch_path; + auto setup_shorebird_result = + flutter::SetUpShorebird(args.assets_path, patch_path); + if (setup_shorebird_result) { + // If we have a patch installed, we replace the default AOT library path + // with the patch path here. + source.elf_path = patch_path.c_str(); + } else { + FML_LOG(ERROR) << "Failed to configure Shorebird."; + source.elf_path = fl_dart_project_get_aot_library_path(self->project); + } + if (self->embedder_api.CreateAOTData(&source, &self->aot_data) != kSuccess) { g_set_error(error, fl_engine_error_quark(), FL_ENGINE_ERROR_FAILED, diff --git a/engine/src/flutter/shell/platform/linux/fl_shorebird.cc b/engine/src/flutter/shell/platform/linux/fl_shorebird.cc new file mode 100644 index 0000000000000..eb913ad720c61 --- /dev/null +++ b/engine/src/flutter/shell/platform/linux/fl_shorebird.cc @@ -0,0 +1,56 @@ +#include "flutter/shell/platform/linux/fl_shorebird.h" + +#include +#include +#include + +#include "flutter/fml/file.h" +#include "flutter/fml/logging.h" +#include "flutter/fml/paths.h" +#include "flutter/shell/common/shorebird/shorebird.h" +#include "rapidjson/document.h" +#include "third_party/tonic/filesystem/filesystem/file.h" + +// Namespaced to avoid Google style warnings. +namespace flutter { + +gboolean SetUpShorebird(const char* assets_path, std::string& patch_path) { + auto shorebird_yaml_path = + fml::paths::JoinPaths({assets_path, "shorebird.yaml"}); + std::string shorebird_yaml_contents(""); + if (!filesystem::ReadFileToString(shorebird_yaml_path, + &shorebird_yaml_contents)) { + FML_LOG(ERROR) << "Failed to read shorebird.yaml."; + return false; + } + + std::string code_cache_path = + fml::paths::JoinPaths({g_get_home_dir(), ".shorebird_cache"}); + auto executable_location = fml::paths::GetExecutableDirectoryPath().second; + auto app_path = + fml::paths::JoinPaths({executable_location, "lib", "libapp.so"}); + auto version_json_path = fml::paths::JoinPaths({assets_path, "version.json"}); + std::ifstream input(version_json_path); + if (!input) { + return false; + } + std::string json_contents{std::istreambuf_iterator(input), + std::istreambuf_iterator()}; + + rapidjson::Document json_doc; + json_doc.Parse(json_contents.c_str()); + if (json_doc.HasParseError()) { + // Could not parse version file, aborting. + return false; + } + + const auto version_map = json_doc.GetObject(); + ReleaseVersion release_version{version_map["version"].GetString(), + version_map["build_number"].GetString()}; + + ShorebirdConfigArgs shorebird_args(code_cache_path, code_cache_path, app_path, + shorebird_yaml_contents, release_version); + return ConfigureShorebird(shorebird_args, patch_path); +} + +} // namespace flutter diff --git a/engine/src/flutter/shell/platform/linux/fl_shorebird.h b/engine/src/flutter/shell/platform/linux/fl_shorebird.h new file mode 100644 index 0000000000000..e38965776ac97 --- /dev/null +++ b/engine/src/flutter/shell/platform/linux/fl_shorebird.h @@ -0,0 +1,13 @@ +#ifndef FLUTTER_SHELL_PLATFORM_LINUX_FL_SHOREBIRD_H_ +#define FLUTTER_SHELL_PLATFORM_LINUX_FL_SHOREBIRD_H_ + +#include +#include + +namespace flutter { + +gboolean SetUpShorebird(const char* assets_path, std::string& patch_path); + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_LINUX_FL_SHOREBIRD_H_ diff --git a/engine/src/flutter/shell/platform/windows/BUILD.gn b/engine/src/flutter/shell/platform/windows/BUILD.gn index 6c547936faa0c..52690485316d2 100644 --- a/engine/src/flutter/shell/platform/windows/BUILD.gn +++ b/engine/src/flutter/shell/platform/windows/BUILD.gn @@ -169,6 +169,7 @@ source_set("flutter_windows_source") { ":flutter_windows_headers", "//flutter/fml", "//flutter/impeller/renderer/backend/gles", + "//flutter/shell/common/shorebird", "//flutter/shell/geometry", "//flutter/shell/platform/common:common_cpp", "//flutter/shell/platform/common:common_cpp_input", @@ -184,6 +185,7 @@ source_set("flutter_windows_source") { "//flutter/third_party/angle:libEGL_static", "//flutter/third_party/angle:libGLESv2_static", "//flutter/third_party/rapidjson", + "//flutter/third_party/tonic", ] } @@ -195,6 +197,11 @@ copy("publish_headers_windows") { deps = [ "//flutter/shell/platform/common:publish_headers" ] } +copy("updater_exports_windows") { + sources = [ "flutter_windows.dll.def" ] + outputs = [ "$root_out_dir/{{source_file_part}}" ] +} + shared_library("flutter_windows") { deps = [ ":flutter_windows_source" ] @@ -310,6 +317,7 @@ group("windows") { deps = [ ":flutter_windows", ":publish_headers_windows", + ":updater_exports_windows", "//flutter/shell/platform/windows/client_wrapper:publish_wrapper_windows", ] diff --git a/engine/src/flutter/shell/platform/windows/flutter_project_bundle.h b/engine/src/flutter/shell/platform/windows/flutter_project_bundle.h index 4291206c920e0..e95f4780d88b7 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_project_bundle.h +++ b/engine/src/flutter/shell/platform/windows/flutter_project_bundle.h @@ -56,6 +56,10 @@ class FlutterProjectBundle { // Sets engine switches. void SetSwitches(const std::vector& switches); + void SetAotLibraryPath(const std::filesystem::path& aot_library_path) { + aot_library_path_ = aot_library_path; + } + // Attempts to load AOT data for this bundle. The returned data must be // retained until any engine instance it is passed to has been shut down. // diff --git a/engine/src/flutter/shell/platform/windows/flutter_windows.dll.def b/engine/src/flutter/shell/platform/windows/flutter_windows.dll.def new file mode 100644 index 0000000000000..bcaa785977259 --- /dev/null +++ b/engine/src/flutter/shell/platform/windows/flutter_windows.dll.def @@ -0,0 +1,17 @@ +EXPORTS + shorebird_check_for_downloadable_update = shorebird_check_for_downloadable_update + shorebird_check_for_update = shorebird_check_for_update + shorebird_current_boot_patch_number = shorebird_current_boot_patch_number + shorebird_free_string = shorebird_free_string + shorebird_free_update_result = shorebird_free_update_result + shorebird_init = shorebird_init + shorebird_next_boot_patch_number = shorebird_next_boot_patch_number + shorebird_next_boot_patch_path = shorebird_next_boot_patch_path + shorebird_validate_next_boot_patch = shorebird_validate_next_boot_patch + shorebird_report_launch_failure = shorebird_report_launch_failure + shorebird_report_launch_start = shorebird_report_launch_start + shorebird_report_launch_success = shorebird_report_launch_success + shorebird_should_auto_update = shorebird_should_auto_update + shorebird_start_update_thread = shorebird_start_update_thread + shorebird_update = shorebird_update + shorebird_update_with_result = shorebird_update_with_result \ No newline at end of file diff --git a/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc b/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc index 70d7364d56e64..22b028beb03f4 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc +++ b/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc @@ -5,15 +5,20 @@ #include "flutter/shell/platform/windows/flutter_windows_engine.h" #include +#include +#include +#include #include #include #include +#include #include "flutter/fml/logging.h" #include "flutter/fml/paths.h" #include "flutter/fml/platform/win/wstring_conversion.h" #include "flutter/fml/synchronization/waitable_event.h" +#include "flutter/shell/common/shorebird/shorebird.h" #include "flutter/shell/platform/common/client_wrapper/binary_messenger_impl.h" #include "flutter/shell/platform/common/client_wrapper/include/flutter/standard_message_codec.h" #include "flutter/shell/platform/common/path_utils.h" @@ -29,6 +34,7 @@ #include "flutter/shell/platform/windows/window_manager.h" #include "flutter/third_party/accessibility/ax/ax_node.h" #include "shell/platform/windows/flutter_project_bundle.h" +#include "third_party/tonic/filesystem/filesystem/file.h" // winbase.h defines GetCurrentTime as a macro. #undef GetCurrentTime @@ -269,6 +275,125 @@ bool FlutterWindowsEngine::Run() { return Run(""); } +int GetReleaseVersionAndBuildNumber(ReleaseVersion* release_version) { + char module_path[MAX_PATH]; + // Get the full path of the currently running executable. The return value is + // the size of the string that was copied to the buffer, with -1 indicating + // failure. + if (GetModuleFileNameA(NULL, module_path, MAX_PATH) == -1) { + return -1; + } + + // Get the size of the version information + DWORD handle = -1; + DWORD version_info_size = GetFileVersionInfoSizeA(module_path, &handle); + if (version_info_size == -1) { + return -1; + } + + // Allocate memory for version info + std::unique_ptr version_data(new char[version_info_size]); + if (!GetFileVersionInfoA(module_path, handle, version_info_size, + version_data.get())) { + return -1; + } + + // Adopted from + // https://learn.microsoft.com/en-us/windows/win32/api/winver/nf-winver-verqueryvaluea + // Get the translation table + struct LANGANDCODEPAGE { + WORD wLanguage; + WORD wCodePage; + }* lpTranslate; + + UINT cbTranslate = 0; + if (!VerQueryValueA(version_data.get(), "\\VarFileInfo\\Translation", + (LPVOID*)&lpTranslate, &cbTranslate)) { + FML_LOG(ERROR) << "Error: Unable to get translation info."; + return -1; + } + + // Construct the query string using the first translation found + char subBlock[64]; + sprintf_s(subBlock, "\\StringFileInfo\\%04x%04x\\ProductVersion", + lpTranslate[0].wLanguage, lpTranslate[0].wCodePage); + + LPSTR versionString = nullptr; + UINT size = 0; + if (!VerQueryValueA(version_data.get(), subBlock, (LPVOID*)&versionString, + &size)) { + return -1; + } + + if (!versionString) { + return -1; + } + + // The version string is in the format of "1.0.0+1", with the label ("+1") + // being optional. + auto version = std::string(versionString); + auto plusPos = version.find("+"); + if (plusPos != std::string::npos) { + auto semVer = version.substr(0, plusPos); + auto patch = version.substr(plusPos + 1, version.length()); + release_version->version = semVer; + release_version->build_number = patch; + } else { + release_version->version = version; + } + + return kSuccess; +} + +bool GetLocalAppDataPath(std::string& outPath) { + PWSTR path = nullptr; + HRESULT result = SHGetKnownFolderPath(FOLDERID_LocalAppData, 0, NULL, &path); + if (!SUCCEEDED(result)) { + return false; + } + + std::wstring widePath(path); + std::string localAppDataPath(widePath.begin(), widePath.end()); + // The calling process is responsible for freeing this resource + // https://learn.microsoft.com/en-us/windows/win32/api/shlobj_core/nf-shlobj_core-shgetknownfolderpath + CoTaskMemFree(path); + outPath = localAppDataPath; + return true; +} + +bool SetUpShorebird(std::string assets_path_string, std::string& patch_path) { + auto shorebird_yaml_path = + fml::paths::JoinPaths({assets_path_string, "shorebird.yaml"}); + std::string shorebird_yaml_contents(""); + if (!filesystem::ReadFileToString(shorebird_yaml_path, + &shorebird_yaml_contents)) { + FML_LOG(ERROR) << "Failed to read shorebird.yaml."; + return false; + } + + std::string code_cache_path; + if (!GetLocalAppDataPath(code_cache_path)) { + FML_LOG(ERROR) << "Failed to retrieve the local AppData directory."; + return false; + } + + auto executable_location = fml::paths::GetExecutableDirectoryPath().second; + auto app_path = + fml::paths::JoinPaths({executable_location, "data", "app.so"}); + ReleaseVersion release_version; + auto release_version_result = + GetReleaseVersionAndBuildNumber(&release_version); + if (release_version_result != kSuccess) { + FML_LOG(ERROR) + << "Failed to retrieve the release version and build number."; + return false; + } + + ShorebirdConfigArgs shorebird_args(code_cache_path, code_cache_path, app_path, + shorebird_yaml_contents, release_version); + return ConfigureShorebird(shorebird_args, patch_path); +} + bool FlutterWindowsEngine::Run(std::string_view entrypoint) { if (!project_->HasValidPaths()) { FML_LOG(ERROR) << "Missing or unresolvable paths to assets."; @@ -276,6 +401,19 @@ bool FlutterWindowsEngine::Run(std::string_view entrypoint) { } std::string assets_path_string = fml::PathToUtf8(project_->assets_path()); std::string icu_path_string = fml::PathToUtf8(project_->icu_path()); + + std::string patch_path; + auto setup_shorebird_result = SetUpShorebird(assets_path_string, patch_path); + if (setup_shorebird_result) { + // If we have a patch installed, we replace the default AOT library path + // with the patch path here. + FML_LOG(INFO) << "Setting project patch path: " << patch_path; + project_->SetAotLibraryPath(patch_path); + } else { + FML_LOG(ERROR) << "Failed to configure Shorebird."; + } + + // This loads AOT data from the project_'s aot_library_path_. if (embedder_api_.RunsAOTCompiledDartCode()) { aot_data_ = project_->LoadAotData(embedder_api_); if (!aot_data_) { @@ -406,6 +544,15 @@ bool FlutterWindowsEngine::Run(std::string_view entrypoint) { host->root_isolate_create_callback_(); } }; + // Copied from shell\platform\darwin\macos\framework\Source\FlutterEngine.mm + // Writes log messages to stdout. + args.log_message_callback = [](const char* tag, const char* message, + void* user_data) { + if (tag && tag[0]) { + std::cout << tag << ": "; + } + std::cout << message << std::endl; + }; args.channel_update_callback = [](const FlutterChannelUpdate* update, void* user_data) { auto host = static_cast(user_data); diff --git a/engine/src/flutter/shell/testing/BUILD.gn b/engine/src/flutter/shell/testing/BUILD.gn index 79dd5084ffee2..0ed88d93ed52a 100644 --- a/engine/src/flutter/shell/testing/BUILD.gn +++ b/engine/src/flutter/shell/testing/BUILD.gn @@ -42,6 +42,7 @@ executable("testing") { deps = [ "$dart_src/runtime:libdart_jit", "$dart_src/runtime/bin:common_embedder_dart_io", + "$dart_src/runtime/bin:elf_loader", "//flutter/assets", "//flutter/common", "//flutter/flow", diff --git a/engine/src/flutter/sky/tools/create_ios_framework.py b/engine/src/flutter/sky/tools/create_ios_framework.py index d87e0f91bff49..0ead267e9609e 100644 --- a/engine/src/flutter/sky/tools/create_ios_framework.py +++ b/engine/src/flutter/sky/tools/create_ios_framework.py @@ -19,7 +19,8 @@ def main(): parser = argparse.ArgumentParser( description=( 'Creates Flutter.framework, Flutter.xcframework and ' - 'copies architecture-dependent gen_snapshot binaries to output dir' + 'copies architecture-dependent analyze_snapshot and gen_snapshot ' + 'binaries to output dir' ) ) @@ -89,13 +90,17 @@ def main(): '%s_extension_safe' % simulator_x64_out_dir, '%s_extension_safe' % simulator_arm64_out_dir ) - # Copy gen_snapshot binary to destination directory. + # Copy analyze_snapshot and gen_snapshot binaries to destination directory. if arm64_out_dir: gen_snapshot = os.path.join(arm64_out_dir, 'universal', 'gen_snapshot_arm64') + analyze_snapshot = os.path.join(arm64_out_dir, 'analyze_snapshot_arm64') sky_utils.copy_binary(gen_snapshot, os.path.join(dst, 'gen_snapshot_arm64')) + sky_utils.copy_binary(analyze_snapshot, os.path.join(dst, 'analyze_snapshot_arm64')) if x64_out_dir: gen_snapshot = os.path.join(x64_out_dir, 'universal', 'gen_snapshot_x64') + analyze_snapshot = os.path.join(x64_out_dir, 'analyze_snapshot_x64') sky_utils.copy_binary(gen_snapshot, os.path.join(dst, 'gen_snapshot_x64')) + sky_utils.copy_binary(analyze_snapshot, os.path.join(dst, 'analyze_snapshot_x64')) zip_archive(dst, args) return 0 @@ -177,7 +182,7 @@ def zip_archive(dst, args): # See: https://github.com/flutter/flutter/blob/62382c7b83a16b3f48dc06c19a47f6b8667005a5/dev/bots/suite_runners/run_verify_binaries_codesigned_tests.dart#L82-L130 # Binaries that must be codesigned and require entitlements for particular APIs. - with_entitlements = ['gen_snapshot_arm64'] + with_entitlements = ['analyze_snapshot_arm64', 'gen_snapshot_arm64'] with_entitlements_file = os.path.join(dst, 'entitlements.txt') sky_utils.write_codesign_config(with_entitlements_file, with_entitlements) @@ -211,6 +216,7 @@ def zip_archive(dst, args): # pylint: enable=line-too-long zip_contents = [ + 'analyze_snapshot_arm64', 'gen_snapshot_arm64', 'Flutter.xcframework', 'entitlements.txt', diff --git a/engine/src/flutter/testing/run_tests.py b/engine/src/flutter/testing/run_tests.py index 3fa7a2c39d0e0..e5972c413462a 100755 --- a/engine/src/flutter/testing/run_tests.py +++ b/engine/src/flutter/testing/run_tests.py @@ -484,6 +484,7 @@ def make_test( make_test('platform_view_android_delegate_unittests'), # https://github.com/flutter/flutter/issues/36295 make_test('shell_unittests'), + make_test('shorebird_unittests'), ] if is_windows(): diff --git a/packages/flutter_tools/bin/xcode_backend.dart b/packages/flutter_tools/bin/xcode_backend.dart index e32980a357a4d..021f070b763c6 100644 --- a/packages/flutter_tools/bin/xcode_backend.dart +++ b/packages/flutter_tools/bin/xcode_backend.dart @@ -722,6 +722,13 @@ class Context { ); } + if (environment['TRACE_FILE'] != null && + environment['TRACE_FILE']!.isNotEmpty) { + flutterArgs.add( + '--trace-file=${environment['TRACE_FILE']}', + ); + } + if (environment['CODE_SIZE_DIRECTORY'] != null && environment['CODE_SIZE_DIRECTORY']!.isNotEmpty) { flutterArgs.add('-dCodeSizeDirectory=${environment['CODE_SIZE_DIRECTORY']}'); diff --git a/packages/flutter_tools/gradle/aar_init_script.gradle b/packages/flutter_tools/gradle/aar_init_script.gradle index 0d43a9b103fd2..0abc801a2fb62 100644 --- a/packages/flutter_tools/gradle/aar_init_script.gradle +++ b/packages/flutter_tools/gradle/aar_init_script.gradle @@ -42,7 +42,7 @@ void configureProject(Project project, String outputDir) { return } - String storageUrl = System.getenv('FLUTTER_STORAGE_BASE_URL') ?: "https://storage.googleapis.com" + String storageUrl = System.getenv('FLUTTER_STORAGE_BASE_URL') ?: "https://download.shorebird.dev" String engineRealm = Paths.get(getFlutterRoot(project), "bin", "cache", "engine.realm") .toFile().text.trim() diff --git a/packages/flutter_tools/gradle/build.gradle.kts b/packages/flutter_tools/gradle/build.gradle.kts index 31fc5cbaedc50..84a0f8a6e2ca8 100644 --- a/packages/flutter_tools/gradle/build.gradle.kts +++ b/packages/flutter_tools/gradle/build.gradle.kts @@ -64,6 +64,7 @@ dependencies { // * ndkVersion in FlutterExtension in packages/flutter_tools/gradle/src/main/kotlin/FlutterExtension.kt compileOnly("com.android.tools.build:gradle:8.11.1") + implementation("org.yaml:snakeyaml:2.0") testImplementation(kotlin("test")) testImplementation("com.android.tools.build:gradle:8.11.1") testImplementation("org.mockito:mockito-core:5.8.0") diff --git a/packages/flutter_tools/gradle/src/main/kotlin/FlutterPlugin.kt b/packages/flutter_tools/gradle/src/main/kotlin/FlutterPlugin.kt index 9dc036df7b154..c640ed6236fc3 100644 --- a/packages/flutter_tools/gradle/src/main/kotlin/FlutterPlugin.kt +++ b/packages/flutter_tools/gradle/src/main/kotlin/FlutterPlugin.kt @@ -597,6 +597,8 @@ class FlutterPlugin : Plugin { val dartDefinesValue: String? = project.findProperty("dart-defines")?.toString() val performanceMeasurementFileValue: String? = project.findProperty("performance-measurement-file")?.toString() + val traceFileValue: String? = + project.findProperty("trace-file")?.toString() val codeSizeDirectoryValue: String? = project.findProperty("code-size-directory")?.toString() val deferredComponentsValue: Boolean = @@ -694,6 +696,7 @@ class FlutterPlugin : Plugin { dartObfuscation = dartObfuscationValue dartDefines = dartDefinesValue performanceMeasurementFile = performanceMeasurementFileValue + traceFile = traceFileValue codeSizeDirectory = codeSizeDirectoryValue deferredComponents = deferredComponentsValue validateDeferredComponents = validateDeferredComponentsValue diff --git a/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt b/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt index f9525ef479e31..4cc13207211cb 100644 --- a/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt +++ b/packages/flutter_tools/gradle/src/main/kotlin/FlutterPluginConstants.kt @@ -19,7 +19,7 @@ object FlutterPluginConstants { const val INTERMEDIATES_DIR = "intermediates" const val FLUTTER_STORAGE_BASE_URL = "FLUTTER_STORAGE_BASE_URL" - const val DEFAULT_MAVEN_HOST = "https://storage.googleapis.com" + const val DEFAULT_MAVEN_HOST = "https://download.shorebird.dev" /** Maps platforms to ABI architectures. */ @JvmStatic val PLATFORM_ARCH_MAP = diff --git a/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTask.kt b/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTask.kt index 1858a139d0e96..dc011b7ecf22d 100644 --- a/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTask.kt +++ b/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTask.kt @@ -111,6 +111,10 @@ open class BaseFlutterTask : DefaultTask() { @Input var performanceMeasurementFile: String? = null + @Optional + @Input + var traceFile: String? = null + @Optional @Input var deferredComponents: Boolean? = null diff --git a/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTaskHelper.kt b/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTaskHelper.kt index 2d16640d324fd..7a5da3346db54 100644 --- a/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTaskHelper.kt +++ b/packages/flutter_tools/gradle/src/main/kotlin/tasks/BaseFlutterTaskHelper.kt @@ -110,6 +110,9 @@ object BaseFlutterTaskHelper { baseFlutterTask.performanceMeasurementFile?.let { args("--performance-measurement-file=$it") } + baseFlutterTask.traceFile?.let { + args("--trace-file=$it") + } args("-dTargetFile=${baseFlutterTask.targetPath}") args("-dTargetPlatform=android") args("-dBuildMode=${baseFlutterTask.buildMode}") diff --git a/packages/flutter_tools/lib/src/android/gradle.dart b/packages/flutter_tools/lib/src/android/gradle.dart index b132b4cbe530a..8f4592c52582d 100644 --- a/packages/flutter_tools/lib/src/android/gradle.dart +++ b/packages/flutter_tools/lib/src/android/gradle.dart @@ -24,6 +24,7 @@ import '../base/project_migrator.dart'; import '../base/terminal.dart'; import '../base/utils.dart'; import '../build_info.dart'; +import '../build_system/build_trace.dart'; import '../cache.dart'; import '../convert.dart'; import '../flutter_manifest.dart'; @@ -480,6 +481,11 @@ class AndroidGradleBuilder implements AndroidBuilder { } // Assembly work starts here. + final int buildStartMicros = DateTime.now().microsecondsSinceEpoch; + final BuildTracer? tracer = androidBuildInfo.buildInfo.traceFilePath != null + ? BuildTracer() + : null; + final BuildInfo buildInfo = androidBuildInfo.buildInfo; final String assembleTask = isBuildingBundle ? getBundleTaskFor(buildInfo) @@ -556,6 +562,26 @@ class AndroidGradleBuilder implements AndroidBuilder { } } options.addAll(androidBuildInfo.buildInfo.toGradleConfig()); + // Pass trace file path as a Gradle property for flutter assemble. + // We use an intermediate file that gradle.dart will merge into the final trace. + // NOTE: This path is passed as a project-level Gradle property, so it's + // shared across all variants. This is safe because `flutter build apk` + // only runs a single variant's FlutterTask per invocation. If we ever + // support tracing builds that run multiple variants simultaneously (e.g. + // multiple flavors in one Gradle invocation), this path would need to + // include the variant name to avoid collisions. + final String? assembleTraceFilePath; + if (buildInfo.traceFilePath != null) { + assembleTraceFilePath = _fileSystem.path.join( + project.android.buildDirectory.path, + 'intermediates', + 'flutter', + 'flutter_assemble_trace.json', + ); + options.add('-Ptrace-file=$assembleTraceFilePath'); + } else { + assembleTraceFilePath = null; + } if (buildInfo.fileSystemRoots.isNotEmpty) { options.add('-Pfilesystem-roots=${buildInfo.fileSystemRoots.join('|')}'); } @@ -565,10 +591,21 @@ class AndroidGradleBuilder implements AndroidBuilder { if (androidBuildInfo.splitPerAbi) { options.add('-Psplit-per-abi=true'); } + final int preGradleEndMicros = DateTime.now().microsecondsSinceEpoch; + tracer?.addCompleteEvent( + name: 'pre-gradle setup', + cat: 'flutter', + tid: 1, + startMicros: buildStartMicros, + endMicros: preGradleEndMicros, + ); + late Stopwatch sw; + late int gradleStartMicros; final int exitCode = await _runGradleTask( assembleTask, preRunTask: () { + gradleStartMicros = DateTime.now().microsecondsSinceEpoch; sw = Stopwatch()..start(); }, postRunTask: () { @@ -588,6 +625,23 @@ class AndroidGradleBuilder implements AndroidBuilder { gradleExecutablePath: gradleExecutablePath, ); + // Record Gradle span and merge assemble trace if tracing is enabled. + if (tracer != null) { + final int gradleEndMicros = DateTime.now().microsecondsSinceEpoch; + tracer.addCompleteEvent( + name: 'gradle $assembleTask', + cat: 'gradle', + tid: 2, + startMicros: gradleStartMicros, + endMicros: gradleEndMicros, + ); + // Merge the assemble trace file that flutter assemble wrote. + if (assembleTraceFilePath != null) { + final File assembleTraceFile = _fileSystem.file(assembleTraceFilePath); + tracer.mergeEventsFromFile(assembleTraceFile); + } + } + if (exitCode != 0) { throwToolExit( 'Gradle task $assembleTask failed with exit code $exitCode', @@ -595,6 +649,31 @@ class AndroidGradleBuilder implements AndroidBuilder { ); } + // Write the build trace if tracing is enabled. + if (tracer != null && buildInfo.traceFilePath != null) { + final int postGradleEndMicros = DateTime.now().microsecondsSinceEpoch; + tracer.addCompleteEvent( + name: 'post-gradle processing', + cat: 'flutter', + tid: 1, + startMicros: gradleStartMicros + sw.elapsedMicroseconds, + endMicros: postGradleEndMicros, + ); + tracer.addCompleteEvent( + name: isBuildingBundle ? 'flutter build appbundle' : 'flutter build apk', + cat: 'flutter', + tid: 1, + startMicros: buildStartMicros, + endMicros: postGradleEndMicros, + ); + final File traceFile = _fileSystem.file(buildInfo.traceFilePath); + tracer.writeToFile(traceFile); + _logger.printStatus( + 'Build trace written to ${buildInfo.traceFilePath}. ' + 'View at https://ui.perfetto.dev', + ); + } + if (isBuildingBundle) { final File bundleFile = findBundleFile(project, buildInfo, _logger, _analytics); diff --git a/packages/flutter_tools/lib/src/base/build.dart b/packages/flutter_tools/lib/src/base/build.dart index bb967d7111be4..008b93da357d2 100644 --- a/packages/flutter_tools/lib/src/base/build.dart +++ b/packages/flutter_tools/lib/src/base/build.dart @@ -8,7 +8,6 @@ import '../artifacts.dart'; import '../build_info.dart'; import '../darwin/darwin.dart'; import '../macos/xcode.dart'; - import 'file_system.dart'; import 'logger.dart'; import 'process.dart'; @@ -130,7 +129,28 @@ class AOTSnapshotter { final Directory outputDir = _fileSystem.directory(outputPath); outputDir.createSync(recursive: true); - final genSnapshotArgs = ['--deterministic']; + // Currently we only use the linker on iOS, but we will eventually split out + // the concept of "optimizes patch snapshot" from "uses linker" and probably + // only uses the linker on iOS, but optimize patch snapshots everywhere. + // TODO(eseidel): TargetPlatform.darwin doesn't use the linker. + bool usesLinker = (platform == TargetPlatform.ios || platform == TargetPlatform.darwin); + final dumpLinkInfoArgs = [ + // Shorebird dumps the class table information during snapshot compilation which is later used during linking. + '--print_class_table_link_debug_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.class_table.json')}', + '--print_class_table_link_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.ct.link')}', + '--print_field_table_link_debug_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.field_table.json')}', + '--print_field_table_link_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.ft.link')}', + '--print_dispatch_table_link_debug_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.dispatch_table.json')}', + '--print_dispatch_table_link_info_to=${_fileSystem.path.join(outputDir.parent.path, 'App.dt.link')}', + ]; + + final genSnapshotArgs = [ + // Shorebird uses --deterministic to improve snapshot stability and increase linking. + '--deterministic', + // Only save LinkInfo if we're using the linker. + if (usesLinker) + ...dumpLinkInfoArgs, + ]; final bool targetingApplePlatform = platform == TargetPlatform.ios || platform == TargetPlatform.darwin; diff --git a/packages/flutter_tools/lib/src/build_info.dart b/packages/flutter_tools/lib/src/build_info.dart index 6867be4c62a33..05221cf95dfb7 100644 --- a/packages/flutter_tools/lib/src/build_info.dart +++ b/packages/flutter_tools/lib/src/build_info.dart @@ -44,6 +44,7 @@ class BuildInfo { List? dartExperiments, required this.treeShakeIcons, this.performanceMeasurementFile, + this.traceFilePath, required this.packageConfigPath, this.codeSizeDirectory, this.androidGradleDaemon = true, @@ -135,6 +136,10 @@ class BuildInfo { /// rerun tasks. final String? performanceMeasurementFile; + /// The path to a file where a Chrome Trace Event Format JSON trace will be + /// written for build profiling. + final String? traceFilePath; + /// If provided, an output directory where one or more v8-style heap snapshots /// will be written for code size profiling. final String? codeSizeDirectory; diff --git a/packages/flutter_tools/lib/src/build_system/build_system.dart b/packages/flutter_tools/lib/src/build_system/build_system.dart index 7e26475c408a8..82f406f82c099 100644 --- a/packages/flutter_tools/lib/src/build_system/build_system.dart +++ b/packages/flutter_tools/lib/src/build_system/build_system.dart @@ -882,6 +882,7 @@ class _BuildInstance { Future _invokeInternal(Node node) async { final PoolResource resource = await resourcePool.request(); + final int startTimeMicroseconds = DateTime.now().microsecondsSinceEpoch; final stopwatch = Stopwatch()..start(); var succeeded = true; var skipped = false; @@ -982,6 +983,7 @@ class _BuildInstance { skipped: skipped, succeeded: succeeded, analyticsName: node.target.analyticsName, + startTimeMicroseconds: startTimeMicroseconds, ); } return succeeded; @@ -1011,6 +1013,7 @@ class PerformanceMeasurement { required this.skipped, required this.succeeded, required this.analyticsName, + required this.startTimeMicroseconds, }); final int elapsedMilliseconds; @@ -1018,6 +1021,9 @@ class PerformanceMeasurement { final bool skipped; final bool succeeded; final String analyticsName; + + /// Wall-clock start time in microseconds since epoch. + final int startTimeMicroseconds; } /// Check if there are any dependency cycles in the target. diff --git a/packages/flutter_tools/lib/src/build_system/build_trace.dart b/packages/flutter_tools/lib/src/build_system/build_trace.dart new file mode 100644 index 0000000000000..131dec4c06c24 --- /dev/null +++ b/packages/flutter_tools/lib/src/build_system/build_trace.dart @@ -0,0 +1,102 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import '../base/file_system.dart'; +import '../convert.dart'; + +/// A single event in a Chrome Trace Event Format trace. +class BuildTraceEvent { + BuildTraceEvent({ + required this.name, + required this.cat, + required this.ts, + required this.dur, + this.pid = 1, + required this.tid, + this.args, + }); + + factory BuildTraceEvent.fromJson(Map json) { + return BuildTraceEvent( + name: json['name']! as String, + cat: json['cat']! as String, + ts: json['ts']! as int, + dur: json['dur']! as int, + pid: json['pid'] as int? ?? 1, + tid: json['tid']! as int, + args: json['args'] as Map?, + ); + } + + final String name; + final String cat; + final int ts; + final int dur; + final int pid; + final int tid; + final Map? args; + + Map toJson() { + return { + 'ph': 'X', + 'name': name, + 'cat': cat, + 'ts': ts, + 'dur': dur, + 'pid': pid, + 'tid': tid, + if (args != null) 'args': args, + }; + } +} + +/// Collects [BuildTraceEvent]s and writes them as a Chrome Trace Event Format +/// JSON array. +class BuildTracer { + final List _events = []; + + /// Adds a complete event (`ph: "X"`) to the trace. + void addCompleteEvent({ + required String name, + required String cat, + required int tid, + required int startMicros, + required int endMicros, + Map? args, + }) { + _events.add(BuildTraceEvent( + name: name, + cat: cat, + tid: tid, + ts: startMicros, + dur: endMicros - startMicros, + args: args, + )); + } + + /// Reads a trace JSON file written by a subprocess and appends its events. + void mergeEventsFromFile(File file) { + if (!file.existsSync()) { + return; + } + final String contents = file.readAsStringSync(); + final jsonList = json.decode(contents) as List; + for (final item in jsonList) { + if (item is Map) { + _events.add(BuildTraceEvent.fromJson(item)); + } + } + } + + /// Writes all collected events as a JSON array to [file]. + void writeToFile(File file) { + if (!file.parent.existsSync()) { + file.parent.createSync(recursive: true); + } + final jsonList = >[ + for (final BuildTraceEvent event in _events) event.toJson(), + ]; + file.writeAsStringSync(json.encode(jsonList)); + } +} diff --git a/packages/flutter_tools/lib/src/build_system/targets/assets.dart b/packages/flutter_tools/lib/src/build_system/targets/assets.dart index f9ebbae2045b5..0b7be7e672975 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/assets.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/assets.dart @@ -13,6 +13,7 @@ import '../../dart/package_map.dart'; import '../../devfs.dart'; import '../../flutter_manifest.dart'; import '../../isolated/native_assets/dart_hook_result.dart'; +import '../../shorebird/shorebird_yaml.dart'; import '../build_system.dart'; import '../depfile.dart'; import '../exceptions.dart'; @@ -162,6 +163,17 @@ Future copyAssets( } if (doCopy) { await (content.file as File).copy(file.path); + if (file.basename == 'shorebird.yaml') { + try { + updateShorebirdYaml( + environment.defines[kFlavor], + file.path, + environment: environment.platform.environment, + ); + } on Exception catch (error) { + throw Exception('Failed to generate shorebird configuration. Error: $error'); + } + } } } else { await file.writeAsBytes(await entry.value.content.contentsAsBytes()); diff --git a/packages/flutter_tools/lib/src/build_system/targets/common.dart b/packages/flutter_tools/lib/src/build_system/targets/common.dart index 63eb1c736e825..e67e646bebf1b 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/common.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/common.dart @@ -504,3 +504,38 @@ abstract final class Lipo { } } } + +/// For managing the supplementary linking files for Shorebird. +abstract final class LinkSupplement { + static Future create( + Environment environment, { + required String inputBuildDir, + required String outputBuildDir, + }) async { + // If the shorebird directory exists, delete it first. + final Directory shorebirdDir = environment.fileSystem.directory( + environment.fileSystem.path.join(outputBuildDir, 'shorebird'), + ); + if (shorebirdDir.existsSync()) { + shorebirdDir.deleteSync(recursive: true); + } + + void maybeCopy(String name) { + final File file = environment.fileSystem.file( + environment.fileSystem.path.join(inputBuildDir, name), + ); + if (file.existsSync()) { + file.copySync(environment.fileSystem.path.join(shorebirdDir.path, name)); + } + } + + // Copy the link information (generated by gen_snapshot) + // into the shorebird directory. + maybeCopy('App.ct.link'); + maybeCopy('App.class_table.json'); + maybeCopy('App.dt.link'); + maybeCopy('App.dispatch_table.json'); + maybeCopy('App.ft.link'); + maybeCopy('App.field_table.json'); + } +} diff --git a/packages/flutter_tools/lib/src/build_system/targets/ios.dart b/packages/flutter_tools/lib/src/build_system/targets/ios.dart index a6116632fd46c..9a91c94beeece 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/ios.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/ios.dart @@ -143,6 +143,12 @@ abstract class AotAssemblyBase extends Target { // Don't fail if the dSYM wasn't created (i.e. during a debug build). skipMissingInputs: true, ); + + await LinkSupplement.create( + environment, + inputBuildDir: buildOutputPath, + outputBuildDir: getIosBuildDirectory(), + ); } } diff --git a/packages/flutter_tools/lib/src/build_system/targets/macos.dart b/packages/flutter_tools/lib/src/build_system/targets/macos.dart index dbffaa465c994..c0c382c934f48 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/macos.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/macos.dart @@ -344,6 +344,12 @@ class CompileMacOSFramework extends Target { // Don't fail if the dSYM wasn't created (i.e. during a debug build). skipMissingInputs: true, ); + + await LinkSupplement.create( + environment, + inputBuildDir: buildOutputPath, + outputBuildDir: getMacOSBuildDirectory(), + ); } @override diff --git a/packages/flutter_tools/lib/src/cache.dart b/packages/flutter_tools/lib/src/cache.dart index 275fb098f4f83..2cac940fccfb1 100644 --- a/packages/flutter_tools/lib/src/cache.dart +++ b/packages/flutter_tools/lib/src/cache.dart @@ -34,6 +34,7 @@ import 'base/user_messages.dart'; import 'convert.dart'; import 'features.dart'; +const kShorebirdStorageUrl = 'https://download.shorebird.dev'; const kFlutterRootEnvironmentVariableName = 'FLUTTER_ROOT'; // should point to //flutter/ (root of flutter/flutter repo) const kFlutterEngineEnvironmentVariableName = @@ -517,7 +518,7 @@ class Cache { /// The base for URLs that store Flutter engine artifacts that are fetched /// during the installation of the Flutter SDK. /// - /// By default the base URL is https://storage.googleapis.com. However, if + /// By default the base URL is https://download.shorebird.dev. However, if /// `FLUTTER_STORAGE_BASE_URL` environment variable ([kFlutterStorageBaseUrl]) /// is provided, the environment variable value is returned instead. /// @@ -529,9 +530,13 @@ class Cache { String? overrideUrl = _platform.environment[kFlutterStorageBaseUrl]; if (overrideUrl == null) { return storageRealm.isEmpty - ? 'https://storage.googleapis.com' + ? 'https://download.shorebird.dev' : 'https://storage.googleapis.com/$storageRealm'; } + // Shorebird's artifact proxy is a trusted source. + if (overrideUrl == kShorebirdStorageUrl) { + return overrideUrl; + } // verify that this is a valid URI. overrideUrl = storageRealm.isEmpty ? overrideUrl : '$overrideUrl/$storageRealm'; try { diff --git a/packages/flutter_tools/lib/src/commands/assemble.dart b/packages/flutter_tools/lib/src/commands/assemble.dart index 564aa971a1a20..90282345088f0 100644 --- a/packages/flutter_tools/lib/src/commands/assemble.dart +++ b/packages/flutter_tools/lib/src/commands/assemble.dart @@ -112,6 +112,10 @@ class AssembleCommand extends FlutterCommand { 'performance-measurement-file', help: 'Output individual target performance to a JSON file.', ); + argParser.addOption( + 'trace-file', + help: 'Output build trace in Chrome Trace Event Format JSON.', + ); argParser.addMultiOption( 'input', abbr: 'i', @@ -394,6 +398,10 @@ class AssembleCommand extends FlutterCommand { final File outFile = globals.fs.file(argumentResults['performance-measurement-file']); writePerformanceData(result.performance.values, outFile); } + if (argumentResults.wasParsed('trace-file')) { + final File outFile = globals.fs.file(argumentResults['trace-file']); + writeTraceData(result.performance.values, outFile); + } if (argumentResults.wasParsed('depfile')) { final File depfileFile = globals.fs.file(stringArg('depfile')); final depfile = Depfile(result.inputFiles, result.outputFiles); @@ -440,3 +448,29 @@ void writePerformanceData(Iterable measurements, File ou } outFile.writeAsStringSync(json.encode(jsonData)); } + +/// Output build trace data in Chrome Trace Event Format in [outFile]. +@visibleForTesting +void writeTraceData(Iterable measurements, File outFile) { + final events = >[ + for (final PerformanceMeasurement measurement in measurements) + { + 'ph': 'X', + 'name': measurement.analyticsName, + 'cat': 'assemble', + 'ts': measurement.startTimeMicroseconds, + 'dur': measurement.elapsedMilliseconds * 1000, + 'pid': 1, + 'tid': 3, + 'args': { + 'target': measurement.target, + 'skipped': measurement.skipped, + 'succeeded': measurement.succeeded, + }, + }, + ]; + if (!outFile.parent.existsSync()) { + outFile.parent.createSync(recursive: true); + } + outFile.writeAsStringSync(json.encode(events)); +} diff --git a/packages/flutter_tools/lib/src/commands/build_apk.dart b/packages/flutter_tools/lib/src/commands/build_apk.dart index d3f50300804ef..fd397ef109adb 100644 --- a/packages/flutter_tools/lib/src/commands/build_apk.dart +++ b/packages/flutter_tools/lib/src/commands/build_apk.dart @@ -31,6 +31,7 @@ class BuildApkCommand extends BuildSubCommand { usesExtraDartFlagOptions(verboseHelp: verboseHelp); addEnableExperimentation(hide: !verboseHelp); addBuildPerformanceFile(hide: !verboseHelp); + usesBuildTraceOption(hide: !verboseHelp); usesAnalyzeSizeFlag(); addAndroidSpecificBuildOptions(hide: !verboseHelp); addIgnoreDeprecationOption(); diff --git a/packages/flutter_tools/lib/src/commands/build_ios.dart b/packages/flutter_tools/lib/src/commands/build_ios.dart index e783f793b9e7a..7c2700d77b417 100644 --- a/packages/flutter_tools/lib/src/commands/build_ios.dart +++ b/packages/flutter_tools/lib/src/commands/build_ios.dart @@ -907,6 +907,7 @@ abstract class _BuildIOSSubCommand extends BuildSubCommand { usesExtraDartFlagOptions(verboseHelp: verboseHelp); addEnableExperimentation(hide: !verboseHelp); addBuildPerformanceFile(hide: !verboseHelp); + usesBuildTraceOption(hide: !verboseHelp); usesAnalyzeSizeFlag(); argParser.addFlag( 'codesign', diff --git a/packages/flutter_tools/lib/src/http_host_validator.dart b/packages/flutter_tools/lib/src/http_host_validator.dart index 9702f83bffd08..8363221166398 100644 --- a/packages/flutter_tools/lib/src/http_host_validator.dart +++ b/packages/flutter_tools/lib/src/http_host_validator.dart @@ -11,7 +11,7 @@ import 'doctor_validator.dart'; import 'features.dart'; /// Common Flutter HTTP hosts. -const kCloudHost = 'https://storage.googleapis.com/'; +const kCloudHost = 'https://download.shorebird.dev/'; const kCocoaPods = 'https://cocoapods.org/'; const kGitHub = 'https://github.com/'; const kMaven = 'https://maven.google.com/'; diff --git a/packages/flutter_tools/lib/src/ios/application_package.dart b/packages/flutter_tools/lib/src/ios/application_package.dart index f1c8f44849548..e8613d4d7a8bd 100644 --- a/packages/flutter_tools/lib/src/ios/application_package.dart +++ b/packages/flutter_tools/lib/src/ios/application_package.dart @@ -130,6 +130,18 @@ class BuildableIOSApp extends IOSApp { @override String? get name => _appProductName; + String get shorebirdYamlPath => + globals.fs.path.join( + archiveBundleOutputPath, + 'Products', + 'Applications', + _appProductName != null ? '$_appProductName.app' : 'Runner.app', + 'Frameworks', + 'App.framework', + 'flutter_assets', + 'shorebird.yaml', + ); + @override String get simulatorBundlePath => _buildAppPath(XcodeSdk.IPhoneSimulator.platformName); diff --git a/packages/flutter_tools/lib/src/ios/mac.dart b/packages/flutter_tools/lib/src/ios/mac.dart index 82e94b215eee1..f7390ca74abc0 100644 --- a/packages/flutter_tools/lib/src/ios/mac.dart +++ b/packages/flutter_tools/lib/src/ios/mac.dart @@ -18,6 +18,7 @@ import '../base/project_migrator.dart'; import '../base/utils.dart'; import '../base/version.dart'; import '../build_info.dart'; +import '../build_system/build_trace.dart'; import '../cache.dart'; import '../darwin/darwin.dart'; import '../device.dart'; @@ -353,6 +354,26 @@ Future buildXcodeProject({ return XcodeBuildResult(success: true); } + final int buildStartMicros = DateTime.now().microsecondsSinceEpoch; + final BuildTracer? tracer = buildInfo.traceFilePath != null + ? BuildTracer() + : null; + + // Pass trace file path as an intermediate location for flutter assemble. + // mac.dart will merge this into the final trace file. + final String? assembleTraceFilePath; + if (buildInfo.traceFilePath != null) { + assembleTraceFilePath = globals.fs.path.join( + globals.fs.currentDirectory.path, + getBuildDirectory(), + 'ios', + 'flutter_assemble_trace.json', + ); + buildCommands.add('TRACE_FILE=$assembleTraceFilePath'); + } else { + assembleTraceFilePath = null; + } + if (globals.logger.isVerbose) { // An environment variable to be passed to xcode_backend.sh determining // whether to echo back executed commands. @@ -531,6 +552,16 @@ Future buildXcodeProject({ ]); } + final int preXcodeEndMicros = DateTime.now().microsecondsSinceEpoch; + tracer?.addCompleteEvent( + name: 'pre-xcode setup', + cat: 'flutter', + tid: 1, + startMicros: buildStartMicros, + endMicros: preXcodeEndMicros, + ); + + final int xcodeStartMicros = DateTime.now().microsecondsSinceEpoch; final sw = Stopwatch()..start(); initialBuildStatus = globals.logger.startProgress('Running Xcode build...'); @@ -555,6 +586,43 @@ Future buildXcodeProject({ ), ); + // Record Xcode span and merge assemble trace if tracing is enabled. + if (tracer != null) { + final int xcodeEndMicros = DateTime.now().microsecondsSinceEpoch; + tracer.addCompleteEvent( + name: 'xcode ${xcodeBuildActionToString(buildAction)}', + cat: 'xcode', + tid: 2, + startMicros: xcodeStartMicros, + endMicros: xcodeEndMicros, + ); + // Merge the assemble trace file that flutter assemble wrote. + if (assembleTraceFilePath != null) { + final File assembleTraceFile = globals.fs.file(assembleTraceFilePath); + tracer.mergeEventsFromFile(assembleTraceFile); + } + tracer.addCompleteEvent( + name: 'post-xcode processing', + cat: 'flutter', + tid: 1, + startMicros: xcodeEndMicros, + endMicros: DateTime.now().microsecondsSinceEpoch, + ); + tracer.addCompleteEvent( + name: 'flutter build ios', + cat: 'flutter', + tid: 1, + startMicros: buildStartMicros, + endMicros: DateTime.now().microsecondsSinceEpoch, + ); + final File traceFile = globals.fs.file(buildInfo.traceFilePath); + tracer.writeToFile(traceFile); + globals.printStatus( + 'Build trace written to ${buildInfo.traceFilePath}. ' + 'View at https://ui.perfetto.dev', + ); + } + if (tempDir.existsSync()) { // Display additional warning and error message from xcresult bundle. final Directory resultBundle = tempDir.childDirectory(_kResultBundlePath); diff --git a/packages/flutter_tools/lib/src/runner/flutter_command.dart b/packages/flutter_tools/lib/src/runner/flutter_command.dart index 452ab9d2a4e42..b4fb994ecc270 100644 --- a/packages/flutter_tools/lib/src/runner/flutter_command.dart +++ b/packages/flutter_tools/lib/src/runner/flutter_command.dart @@ -131,6 +131,7 @@ abstract final class FlutterOptions { static const kDartDefineFromFileOption = 'dart-define-from-file'; static const kWebDefinesOption = 'web-define'; static const kPerformanceMeasurementFile = 'performance-measurement-file'; + static const kBuildTrace = 'trace'; static const kDeviceUser = 'device-user'; static const kDeviceTimeout = 'device-timeout'; static const kDeviceConnection = 'device-connection'; @@ -1048,6 +1049,17 @@ abstract class FlutterCommand extends Command { ); } + void usesBuildTraceOption({bool hide = false}) { + argParser.addOption( + FlutterOptions.kBuildTrace, + help: + 'Output a Chrome Trace Event Format JSON file for build profiling. ' + 'The resulting file can be viewed at https://ui.perfetto.dev.', + hide: hide, + valueHelp: 'path/to/trace.json', + ); + } + void addAndroidSpecificBuildOptions({bool hide = false}) { argParser.addFlag( FlutterOptions.kAndroidGradleDaemon, @@ -1426,6 +1438,11 @@ abstract class FlutterCommand extends Command { ? stringArg(FlutterOptions.kPerformanceMeasurementFile) : null; + final String? traceFilePath = + argParser.options.containsKey(FlutterOptions.kBuildTrace) + ? stringArg(FlutterOptions.kBuildTrace) + : null; + final Map defineConfigJsonMap = extractDartDefineConfigJsonMap(); final List dartDefines = extractDartDefines(defineConfigJsonMap: defineConfigJsonMap); @@ -1477,6 +1494,7 @@ abstract class FlutterCommand extends Command { dartDefines: dartDefines, dartExperiments: experiments, performanceMeasurementFile: performanceMeasurementFile, + traceFilePath: traceFilePath, packageConfigPath: packagesPath ?? packageConfigFile.path, codeSizeDirectory: codeSizeDirectory, androidGradleDaemon: androidGradleDaemon, diff --git a/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart b/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart new file mode 100644 index 0000000000000..a741b2683d765 --- /dev/null +++ b/packages/flutter_tools/lib/src/shorebird/shorebird_yaml.dart @@ -0,0 +1,65 @@ +// Copyright 2024 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:yaml/yaml.dart'; +import 'package:yaml_edit/yaml_edit.dart'; + +import '../base/file_system.dart'; +import '../globals.dart' as globals; + +void updateShorebirdYaml(String? flavor, String shorebirdYamlPath, {required Map environment}) { + final File shorebirdYaml = globals.fs.file(shorebirdYamlPath); + if (!shorebirdYaml.existsSync()) { + throw Exception('shorebird.yaml not found at $shorebirdYamlPath'); + } + final YamlDocument input = loadYamlDocument(shorebirdYaml.readAsStringSync()); + final YamlMap yamlMap = input.contents as YamlMap; + final Map compiled = compileShorebirdYaml(yamlMap, flavor: flavor, environment: environment); + // Currently we write out over the same yaml file, we should fix this to + // write to a new .json file instead and avoid naming confusion between the + // input and compiled files. + final YamlEditor yamlEditor = YamlEditor(''); + yamlEditor.update([], compiled); + shorebirdYaml.writeAsStringSync(yamlEditor.toString(), flush: true); +} + +String appIdForFlavor(YamlMap yamlMap, {required String? flavor}) { + if (flavor == null || flavor.isEmpty) { + final String? defaultAppId = yamlMap['app_id'] as String?; + if (defaultAppId == null || defaultAppId.isEmpty) { + throw Exception('Cannot find "app_id" in shorebird.yaml'); + } + return defaultAppId; + } + + final YamlMap? yamlFlavors = yamlMap['flavors'] as YamlMap?; + if (yamlFlavors == null) { + throw Exception('Cannot find "flavors" in shorebird.yaml.'); + } + final String? flavorAppId = yamlFlavors[flavor] as String?; + if (flavorAppId == null || flavorAppId.isEmpty) { + throw Exception('Cannot find "app_id" for $flavor in shorebird.yaml'); + } + return flavorAppId; +} + +Map compileShorebirdYaml(YamlMap yamlMap, {required String? flavor, required Map environment}) { + final String appId = appIdForFlavor(yamlMap, flavor: flavor); + final Map compiled = { + 'app_id': appId, + }; + void copyIfSet(String key) { + if (yamlMap[key] != null) { + compiled[key] = yamlMap[key]; + } + } + copyIfSet('base_url'); + copyIfSet('auto_update'); + copyIfSet('patch_verification'); + final String? shorebirdPublicKeyEnvVar = environment['SHOREBIRD_PUBLIC_KEY']; + if (shorebirdPublicKeyEnvVar != null) { + compiled['patch_public_key'] = shorebirdPublicKeyEnvVar; + } + return compiled; +} diff --git a/packages/flutter_tools/lib/src/version.dart b/packages/flutter_tools/lib/src/version.dart index 0eb275a7bed11..e7c86a308cf32 100644 --- a/packages/flutter_tools/lib/src/version.dart +++ b/packages/flutter_tools/lib/src/version.dart @@ -1049,6 +1049,25 @@ class GitTagVersion { } } + // Check if running on a Shorebird release branch. + final String shorebirdFlutterReleases = git + .runSync([ + 'for-each-ref', + '--contains', + gitRef, + '--format', + '%(refname:short)', + 'refs/remotes/origin/flutter_release/*', + ], workingDirectory: workingDirectory) + .stdout + .trim(); + final String? shorebirdFlutterVersion = LineSplitter.split( + shorebirdFlutterReleases, + ).map((e) => e.replaceFirst('origin/flutter_release/', '')).toList().firstOrNull; + if (shorebirdFlutterVersion != null) { + return parse(shorebirdFlutterVersion); + } + // If we don't exist in a tag, use git to find the latest tag. return _useNewestTagAndCommitsPastFallback( git: git, diff --git a/packages/flutter_tools/lib/src/windows/build_windows.dart b/packages/flutter_tools/lib/src/windows/build_windows.dart index 44d8992e29e37..61a53a904979e 100644 --- a/packages/flutter_tools/lib/src/windows/build_windows.dart +++ b/packages/flutter_tools/lib/src/windows/build_windows.dart @@ -21,6 +21,7 @@ import '../flutter_plugins.dart'; import '../globals.dart' as globals; import '../migrations/cmake_custom_command_migration.dart'; import '../migrations/cmake_native_assets_migration.dart'; +import '../shorebird/shorebird_yaml.dart'; import 'migrations/build_architecture_migration.dart'; import 'migrations/show_window_migration.dart'; import 'migrations/version_migration.dart'; diff --git a/packages/flutter_tools/test/commands.shard/hermetic/assemble_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/assemble_test.dart index e394c0c217381..51baea33c1218 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/assemble_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/assemble_test.dart @@ -364,6 +364,7 @@ void main() { elapsedMilliseconds: 123, skipped: false, succeeded: true, + startTimeMicroseconds: 0, ), }, ), @@ -502,6 +503,7 @@ void main() { skipped: false, succeeded: true, elapsedMilliseconds: 123, + startTimeMicroseconds: 0, ), ]; final FileSystem fileSystem = MemoryFileSystem.test(); @@ -545,6 +547,41 @@ void main() { await commandRunner.run(['--help' /* -- verbose omitted (verboseHelp: true) is set above */]); expect(testLogger.statusText, contains('assemble')); }); + + testWithoutContext('writeTraceData outputs Chrome Trace Event Format', () { + final measurements = [ + PerformanceMeasurement( + analyticsName: 'KernelSnapshot', + target: 'kernel_snapshot', + skipped: false, + succeeded: true, + elapsedMilliseconds: 500, + startTimeMicroseconds: 1000000, + ), + ]; + final FileSystem fileSystem = MemoryFileSystem.test(); + final outFile = fileSystem.currentDirectory.childDirectory('foo').childFile('trace.json'); + + writeTraceData(measurements, outFile); + + expect(outFile, exists); + final events = json.decode(outFile.readAsStringSync()) as List; + expect(events, hasLength(1)); + + final event = events.first! as Map; + expect(event['ph'], 'X'); + expect(event['name'], 'KernelSnapshot'); + expect(event['cat'], 'assemble'); + expect(event['ts'], 1000000); + expect(event['dur'], 500000); + expect(event['pid'], 1); + expect(event['tid'], 3); + expect(event['args'], { + 'target': 'kernel_snapshot', + 'skipped': false, + 'succeeded': true, + }); + }); } final class _StubCommand extends FlutterCommand { diff --git a/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart index 7c3845044399b..adafb265253c6 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart @@ -73,6 +73,14 @@ final macosPlatformCustomEnv = FakePlatform( environment: {'FLUTTER_ROOT': '/', 'HOME': '/'}, ); +final Platform macosPlatformWithShorebirdPublicKey = FakePlatform( + operatingSystem: 'macos', + environment: { + 'FLUTTER_ROOT': '/', + 'HOME': '/', + 'SHOREBIRD_PUBLIC_KEY': 'my_public_key', + } +); final Platform notMacosPlatform = FakePlatform(environment: {'FLUTTER_ROOT': '/'}); void main() { @@ -1123,4 +1131,37 @@ STDERR STUFF OperatingSystemUtils: () => FakeOperatingSystemUtils(hostPlatform: HostPlatform.darwin_x64), }, ); + + testUsingContext('macOS build outputs path and size when successful', + () async { + final BuildCommand command = BuildCommand( + artifacts: artifacts, + androidSdk: FakeAndroidSdk(), + buildSystem: TestBuildSystem.all(BuildResult(success: true)), + fileSystem: MemoryFileSystem.test(), + processUtils: processUtils, + logger: BufferLogger.test(), + osUtils: FakeOperatingSystemUtils(), + ); + createMinimalMockProjectFiles(); + final File shorebirdYamlFile = fileSystem.file( + 'build/macos/Build/Products/Release/example.app/Contents/Frameworks/App.framework/Resources/flutter_assets/shorebird.yaml', + ) + ..createSync(recursive: true) + ..writeAsStringSync('app_id: my-app-id'); + + await createTestCommandRunner(command) + .run(const ['build', 'macos', '--no-pub']); + + final String updatedYaml = shorebirdYamlFile.readAsStringSync(); + expect(updatedYaml, contains('app_id: my-app-id')); + expect(updatedYaml, contains('patch_public_key: my_public_key')); + }, overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => FakeProcessManager.list([ + setUpFakeXcodeBuildHandler('Release'), + ]), + Platform: () => macosPlatformWithShorebirdPublicKey, + FeatureFlags: () => TestFeatureFlags(isMacOSEnabled: true), + }); } diff --git a/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart index 11c574670713a..25da553b819e3 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart @@ -38,6 +38,13 @@ final Platform windowsPlatform = FakePlatform( 'USERPROFILE': '/', }, ); +final Platform windowsPlatformWithPublicKey = + FakePlatform(operatingSystem: 'windows', environment: { + 'PROGRAMFILES(X86)': r'C:\Program Files (x86)\', + 'FLUTTER_ROOT': flutterRoot, + 'USERPROFILE': '/', + 'SHOREBIRD_PUBLIC_KEY': 'my_public_key', +}); final Platform notWindowsPlatform = FakePlatform( environment: {'FLUTTER_ROOT': flutterRoot}, ); @@ -115,7 +122,9 @@ void main() { ...['--target', 'INSTALL'], if (verbose) '--verbose', ], - environment: {if (verbose) 'VERBOSE_SCRIPT_LOGGING': 'true'}, + environment: { + if (verbose) 'VERBOSE_SCRIPT_LOGGING': 'true' + }, onRun: onRun, stdout: stdout, ); @@ -131,7 +140,8 @@ void main() { setUpMockProjectFilesForBuild(); expect( - createTestCommandRunner(command).run(const ['windows', '--no-pub']), + createTestCommandRunner(command) + .run(const ['windows', '--no-pub']), throwsToolExit(), ); }, @@ -154,10 +164,10 @@ void main() { setUpMockCoreProjectFiles(); expect( - createTestCommandRunner(command).run(const ['windows', '--no-pub']), + createTestCommandRunner(command) + .run(const ['windows', '--no-pub']), throwsToolExit( - message: - 'No Windows desktop project configured. See ' + message: 'No Windows desktop project configured. See ' 'https://flutter.dev/to/add-desktop-support ' 'to learn about adding Windows support to a project.', ), @@ -182,8 +192,10 @@ void main() { setUpMockProjectFilesForBuild(); expect( - createTestCommandRunner(command).run(const ['windows', '--no-pub']), - throwsToolExit(message: '"build windows" only supported on Windows hosts.'), + createTestCommandRunner(command) + .run(const ['windows', '--no-pub']), + throwsToolExit( + message: '"build windows" only supported on Windows hosts.'), ); }, overrides: { @@ -205,7 +217,8 @@ void main() { setUpMockProjectFilesForBuild(); expect( - createTestCommandRunner(command).run(const ['windows', '--no-pub']), + createTestCommandRunner(command) + .run(const ['windows', '--no-pub']), throwsToolExit( message: '"build windows" is not currently supported. To enable, run "flutter config --enable-windows-desktop".', @@ -235,7 +248,8 @@ void main() { buildCommand('Release', stdout: 'STDOUT STUFF'), ]); - await createTestCommandRunner(command).run(const ['windows', '--no-pub']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub']); expect(testLogger.statusText, isNot(contains('STDOUT STUFF'))); expect(testLogger.traceText, contains('STDOUT STUFF')); }, @@ -262,7 +276,8 @@ void main() { buildCommand('Release'), ]); - await createTestCommandRunner(command).run(const ['windows', '--no-pub']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub']); expect( analyticsTimingEventExists( @@ -332,7 +347,8 @@ C:\foo\windows\x64\runner\main.cpp(17,1): error C2065: 'Baz': undeclared identif buildCommand('Release', stdout: stdout), ]); - await createTestCommandRunner(command).run(const ['windows', '--no-pub']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub']); // Just the warnings and errors should be surfaced. expect(testLogger.errorText, r''' C:\foo\windows\x64\runner\main.cpp(18): error C2220: the following warning is treated as an error [C:\foo\build\windows\x64\runner\test.vcxproj] @@ -365,7 +381,8 @@ C:\foo\windows\x64\runner\main.cpp(17,1): error C2065: 'Baz': undeclared identif buildCommand('Release', verbose: true, stdout: 'STDOUT STUFF'), ]); - await createTestCommandRunner(command).run(const ['windows', '--no-pub', '-v']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub', '-v']); expect(testLogger.statusText, contains('STDOUT STUFF')); expect(testLogger.traceText, isNot(contains('STDOUT STUFF'))); }, @@ -485,7 +502,8 @@ if %errorlevel% neq 0 goto :VCEnd assembleProject.createSync(recursive: true); assembleProject.writeAsStringSync(fakeBadProjectContent); - await createTestCommandRunner(command).run(const ['windows', '--no-pub']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub']); final List projectLines = assembleProject.readAsLinesSync(); @@ -645,7 +663,8 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, ).run(const ['windows', '--release', '--no-pub']); - expect(testLogger.statusText, contains(r'โœ“ Built build\windows\x64\runner\Release')); + expect(testLogger.statusText, + contains(r'โœ“ Built build\windows\x64\runner\Release')); }, overrides: { FileSystem: () => fileSystem, @@ -702,7 +721,8 @@ if %errorlevel% neq 0 goto :VCEnd buildCommand('Release'), ]); - await createTestCommandRunner(command).run(const ['windows', '--no-pub']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub']); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -750,7 +770,12 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, - ).run(const ['windows', '--no-pub', '--build-name=1.2.3', '--build-number=4']); + ).run(const [ + 'windows', + '--no-pub', + '--build-name=1.2.3', + '--build-number=4' + ]); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -906,7 +931,12 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, - ).run(const ['windows', '--no-pub', '--build-name=1.2.3', '--build-number=4']); + ).run(const [ + 'windows', + '--no-pub', + '--build-name=1.2.3', + '--build-number=4' + ]); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -954,7 +984,12 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, - ).run(const ['windows', '--no-pub', '--build-name=1.2.3', '--build-number=hello']); + ).run(const [ + 'windows', + '--no-pub', + '--build-name=1.2.3', + '--build-number=hello' + ]); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -1011,7 +1046,12 @@ if %errorlevel% neq 0 goto :VCEnd await createTestCommandRunner( command, - ).run(const ['windows', '--no-pub', '--build-name=1.2.3', '--build-number=4.5']); + ).run(const [ + 'windows', + '--no-pub', + '--build-name=1.2.3', + '--build-number=4.5' + ]); final File cmakeConfig = fileSystem.currentDirectory .childDirectory('windows') @@ -1131,14 +1171,16 @@ if %errorlevel% neq 0 goto :VCEnd contains('A summary of your Windows bundle analysis can be found at'), ); expect(testLogger.statusText, contains('dart devtools --appSizeBase=')); - expect(fakeAnalytics.sentEvents, contains(Event.codeSizeAnalysis(platform: 'windows'))); + expect(fakeAnalytics.sentEvents, + contains(Event.codeSizeAnalysis(platform: 'windows'))); }, overrides: { FeatureFlags: () => TestFeatureFlags(isWindowsEnabled: true), FileSystem: () => fileSystem, ProcessManager: () => processManager, Platform: () => windowsPlatform, - FileSystemUtils: () => FileSystemUtils(fileSystem: fileSystem, platform: windowsPlatform), + FileSystemUtils: () => + FileSystemUtils(fileSystem: fileSystem, platform: windowsPlatform), Analytics: () => fakeAnalytics, }, ); @@ -1155,12 +1197,14 @@ if %errorlevel% neq 0 goto :VCEnd logger: BufferLogger.test(), operatingSystemUtils: FakeOperatingSystemUtils(), )..visualStudioOverride = fakeVisualStudio; - fileSystem.currentDirectory = fileSystem.directory("test_'path")..createSync(); + fileSystem.currentDirectory = fileSystem.directory("test_'path") + ..createSync(); final String absPath = fileSystem.currentDirectory.absolute.path; setUpMockCoreProjectFiles(); expect( - createTestCommandRunner(command).run(const ['windows', '--no-pub']), + createTestCommandRunner(command) + .run(const ['windows', '--no-pub']), throwsToolExit( message: 'Path $absPath contains invalid characters in "\'#!\$^&*=|,;<>?". ' @@ -1199,7 +1243,8 @@ No file or variants found for asset: images/a_dot_burr.jpeg. buildCommand('Release', stdout: stdout), ]); - await createTestCommandRunner(command).run(const ['windows', '--no-pub']); + await createTestCommandRunner(command) + .run(const ['windows', '--no-pub']); // Just the warnings and errors should be surfaced. expect(testLogger.errorText, r''' Error detected in pubspec.yaml: @@ -1213,6 +1258,39 @@ No file or variants found for asset: images/a_dot_burr.jpeg. FeatureFlags: () => TestFeatureFlags(isWindowsEnabled: true), }, ); + + testUsingContext( + 'shorebird.yaml is updated when SHOREBIRD_PUBLIC_KEY env var is set', + () async { + final FakeVisualStudio fakeVisualStudio = FakeVisualStudio(); + final BuildWindowsCommand command = BuildWindowsCommand( + logger: BufferLogger.test(), + operatingSystemUtils: FakeOperatingSystemUtils()) + ..visualStudioOverride = fakeVisualStudio; + setUpMockProjectFilesForBuild(); + final File shorebirdYamlFile = fileSystem.file( + r'build\windows\x64\runner\Release\data\flutter_assets\shorebird.yaml', + ) + ..createSync(recursive: true) + ..writeAsStringSync('app_id: my-app-id'); + + processManager = FakeProcessManager.list([ + cmakeGenerationCommand(), + buildCommand('Release'), + ]); + + await createTestCommandRunner(command) + .run(const ['windows', '--release', '--no-pub']); + + final String updatedYaml = shorebirdYamlFile.readAsStringSync(); + expect(updatedYaml, contains('app_id: my-app-id')); + expect(updatedYaml, contains('patch_public_key: my_public_key')); + }, overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => processManager, + Platform: () => windowsPlatformWithPublicKey, + FeatureFlags: () => TestFeatureFlags(isWindowsEnabled: true), + }); } class FakeVisualStudio extends Fake implements VisualStudio { diff --git a/packages/flutter_tools/test/general.shard/base/build_test.dart b/packages/flutter_tools/test/general.shard/base/build_test.dart index de41b2e569322..90cc0c26ac1db 100644 --- a/packages/flutter_tools/test/general.shard/base/build_test.dart +++ b/packages/flutter_tools/test/general.shard/base/build_test.dart @@ -37,6 +37,17 @@ const kDefaultClang = [ 'build/foo/snapshot_assembly.o', ]; +// Shorebird link info arguments added for iOS/macOS builds. +// These correspond to the dumpLinkInfoArgs in AOTSnapshotter.build(). +const kLinkInfoArgs = [ + '--print_class_table_link_debug_info_to=build/App.class_table.json', + '--print_class_table_link_info_to=build/App.ct.link', + '--print_field_table_link_debug_info_to=build/App.field_table.json', + '--print_field_table_link_info_to=build/App.ft.link', + '--print_dispatch_table_link_debug_info_to=build/App.dispatch_table.json', + '--print_dispatch_table_link_info_to=build/App.dt.link', +]; + void main() { group('GenSnapshot', () { late GenSnapshot genSnapshot; @@ -203,6 +214,7 @@ void main() { command: [ genSnapshotPath, '--deterministic', + ...kLinkInfoArgs, '--snapshot_kind=app-aot-assembly', '--assembly=$assembly', '--dwarf-stack-traces', @@ -278,6 +290,7 @@ void main() { command: [ genSnapshotPath, '--deterministic', + ...kLinkInfoArgs, '--snapshot_kind=app-aot-assembly', '--assembly=$assembly', '--obfuscate', @@ -349,6 +362,7 @@ void main() { command: [ genSnapshotPath, '--deterministic', + ...kLinkInfoArgs, '--snapshot_kind=app-aot-assembly', '--assembly=${fileSystem.path.join(outputPath, 'snapshot_assembly.S')}', 'main.dill', diff --git a/packages/flutter_tools/test/general.shard/build_system/build_trace_test.dart b/packages/flutter_tools/test/general.shard/build_system/build_trace_test.dart new file mode 100644 index 0000000000000..2916c7562f14e --- /dev/null +++ b/packages/flutter_tools/test/general.shard/build_system/build_trace_test.dart @@ -0,0 +1,229 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:file/memory.dart'; +import 'package:flutter_tools/src/base/file_system.dart'; +import 'package:flutter_tools/src/build_system/build_trace.dart'; +import 'package:flutter_tools/src/convert.dart'; + +import '../../src/common.dart'; + +void main() { + group('BuildTraceEvent', () { + testWithoutContext('toJson produces Chrome Trace Event Format', () { + final event = BuildTraceEvent( + name: 'test_target', + cat: 'assemble', + ts: 1000000, + dur: 500000, + pid: 1, + tid: 3, + args: {'key': 'value'}, + ); + + final result = event.toJson(); + + expect(result['ph'], 'X'); + expect(result['name'], 'test_target'); + expect(result['cat'], 'assemble'); + expect(result['ts'], 1000000); + expect(result['dur'], 500000); + expect(result['pid'], 1); + expect(result['tid'], 3); + expect(result['args'], {'key': 'value'}); + }); + + testWithoutContext('toJson omits args when null', () { + final event = BuildTraceEvent( + name: 'test', + cat: 'flutter', + ts: 0, + dur: 100, + tid: 1, + ); + + final result = event.toJson(); + + expect(result.containsKey('args'), isFalse); + }); + + testWithoutContext('fromJson round-trips correctly', () { + final original = { + 'ph': 'X', + 'name': 'test', + 'cat': 'flutter', + 'ts': 1000, + 'dur': 500, + 'pid': 1, + 'tid': 2, + 'args': {'foo': 'bar'}, + }; + + final event = BuildTraceEvent.fromJson(original); + final result = event.toJson(); + + expect(result['name'], 'test'); + expect(result['cat'], 'flutter'); + expect(result['ts'], 1000); + expect(result['dur'], 500); + expect(result['pid'], 1); + expect(result['tid'], 2); + }); + }); + + group('BuildTracer', () { + late FileSystem fileSystem; + + setUp(() { + fileSystem = MemoryFileSystem.test(); + }); + + testWithoutContext('addCompleteEvent adds event with correct duration', () { + final tracer = BuildTracer(); + + tracer.addCompleteEvent( + name: 'gradle build', + cat: 'gradle', + tid: 2, + startMicros: 1000, + endMicros: 5000, + ); + + final outFile = fileSystem.file('trace.json'); + tracer.writeToFile(outFile); + + final events = json.decode(outFile.readAsStringSync()) as List; + expect(events, hasLength(1)); + + final event = events.first! as Map; + expect(event['name'], 'gradle build'); + expect(event['cat'], 'gradle'); + expect(event['tid'], 2); + expect(event['ts'], 1000); + expect(event['dur'], 4000); + expect(event['ph'], 'X'); + expect(event['pid'], 1); + }); + + testWithoutContext('mergeEventsFromFile reads and appends events', () { + final tracer = BuildTracer(); + + // Write a trace file to merge. + final sourceFile = fileSystem.file('source_trace.json'); + sourceFile.writeAsStringSync(json.encode(>[ + { + 'ph': 'X', + 'name': 'KernelSnapshot', + 'cat': 'assemble', + 'ts': 2000, + 'dur': 1000, + 'pid': 1, + 'tid': 3, + }, + ])); + + tracer.addCompleteEvent( + name: 'gradle build', + cat: 'gradle', + tid: 2, + startMicros: 1000, + endMicros: 5000, + ); + + tracer.mergeEventsFromFile(sourceFile); + + final outFile = fileSystem.file('trace.json'); + tracer.writeToFile(outFile); + + final events = json.decode(outFile.readAsStringSync()) as List; + expect(events, hasLength(2)); + expect((events[0]! as Map)['name'], 'gradle build'); + expect((events[1]! as Map)['name'], 'KernelSnapshot'); + }); + + testWithoutContext('mergeEventsFromFile does nothing for non-existent file', () { + final tracer = BuildTracer(); + + tracer.addCompleteEvent( + name: 'test', + cat: 'flutter', + tid: 1, + startMicros: 0, + endMicros: 100, + ); + + // Should not throw. + tracer.mergeEventsFromFile(fileSystem.file('does_not_exist.json')); + + final outFile = fileSystem.file('trace.json'); + tracer.writeToFile(outFile); + + final events = json.decode(outFile.readAsStringSync()) as List; + expect(events, hasLength(1)); + }); + + testWithoutContext('writeToFile creates parent directories', () { + final tracer = BuildTracer(); + tracer.addCompleteEvent( + name: 'test', + cat: 'flutter', + tid: 1, + startMicros: 0, + endMicros: 100, + ); + + final outFile = fileSystem.file('/a/b/c/trace.json'); + tracer.writeToFile(outFile); + + expect(outFile.existsSync(), isTrue); + }); + + testWithoutContext('output is valid Chrome Trace Event Format', () { + final tracer = BuildTracer(); + + tracer.addCompleteEvent( + name: 'flutter build apk', + cat: 'flutter', + tid: 1, + startMicros: 0, + endMicros: 15000000, + ); + tracer.addCompleteEvent( + name: 'gradle assembleRelease', + cat: 'gradle', + tid: 2, + startMicros: 500000, + endMicros: 12500000, + ); + tracer.addCompleteEvent( + name: 'KernelSnapshot', + cat: 'assemble', + tid: 3, + startMicros: 2000000, + endMicros: 5000000, + args: {'skipped': false}, + ); + + final outFile = fileSystem.file('trace.json'); + tracer.writeToFile(outFile); + + // Verify it's a valid JSON array. + final events = json.decode(outFile.readAsStringSync()) as List; + expect(events, hasLength(3)); + + // Verify all required fields are present in each event. + for (final item in events) { + final event = item! as Map; + expect(event.containsKey('ph'), isTrue); + expect(event.containsKey('name'), isTrue); + expect(event.containsKey('cat'), isTrue); + expect(event.containsKey('ts'), isTrue); + expect(event.containsKey('dur'), isTrue); + expect(event.containsKey('pid'), isTrue); + expect(event.containsKey('tid'), isTrue); + expect(event['ph'], 'X'); + } + }); + }); +} diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart index 2e0f99300c3be..c0dda6b07591e 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/common_test.dart @@ -25,6 +25,17 @@ const kBoundaryKey = '4d2d9609-c662-4571-afde-31410f96caa6'; const kElfAot = '--snapshot_kind=app-aot-elf'; const kAssemblyAot = '--snapshot_kind=app-aot-assembly'; +/// Generate Shorebird link info arguments for iOS/macOS AOT builds. +/// The [buildPath] should be the build directory path (outputDir.parent.path). +List linkInfoArgsFor(String buildPath) => [ + '--print_class_table_link_debug_info_to=$buildPath/App.class_table.json', + '--print_class_table_link_info_to=$buildPath/App.ct.link', + '--print_field_table_link_debug_info_to=$buildPath/App.field_table.json', + '--print_field_table_link_info_to=$buildPath/App.ft.link', + '--print_dispatch_table_link_debug_info_to=$buildPath/App.dispatch_table.json', + '--print_dispatch_table_link_info_to=$buildPath/App.dt.link', +]; + final Platform macPlatform = FakePlatform( operatingSystem: 'macos', environment: {}, @@ -803,6 +814,7 @@ void main() { // This path is not known by the cache due to the iOS gen_snapshot split. 'Artifact.genSnapshotArm64.TargetPlatform.ios.profile', '--deterministic', + ...linkInfoArgsFor(build), '--write-v8-snapshot-profile-to=code_size_1/snapshot.arm64.json', '--trace-precompiler-to=code_size_1/trace.arm64.json', kAssemblyAot, diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart index aa31e06bfcddd..d57c76c444c7f 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/macos_test.dart @@ -20,6 +20,17 @@ import '../../../src/fake_process_manager.dart'; import '../../../src/fakes.dart'; import '../../../src/package_config.dart'; +/// Generate Shorebird link info arguments for iOS/macOS AOT builds. +/// The [buildPath] should be the build directory path (outputDir.parent.path). +List linkInfoArgsFor(String buildPath) => [ + '--print_class_table_link_debug_info_to=$buildPath/App.class_table.json', + '--print_class_table_link_info_to=$buildPath/App.ct.link', + '--print_field_table_link_debug_info_to=$buildPath/App.field_table.json', + '--print_field_table_link_info_to=$buildPath/App.ft.link', + '--print_dispatch_table_link_debug_info_to=$buildPath/App.dispatch_table.json', + '--print_dispatch_table_link_info_to=$buildPath/App.dt.link', +]; + void main() { late Environment environment; late MemoryFileSystem fileSystem; @@ -142,7 +153,14 @@ void main() { ); lipoExtractX86_64Command = FakeCommand( - command: ['lipo', '-output', binary.path, '-extract', 'x86_64', binary.path], + command: [ + 'lipo', + '-output', + binary.path, + '-extract', + 'x86_64', + binary.path + ], ); }); @@ -229,7 +247,8 @@ void main() { isException.having( (Exception exception) => exception.toString(), 'description', - contains('FlutterMacOS.framework/Versions/A/FlutterMacOS does not exist, cannot thin'), + contains( + 'FlutterMacOS.framework/Versions/A/FlutterMacOS does not exist, cannot thin'), ), ), ); @@ -249,7 +268,13 @@ void main() { copyFrameworkCommand, lipoInfoFatCommand, FakeCommand( - command: ['lipo', binary.path, '-verify_arch', 'arm64', 'x86_64'], + command: [ + 'lipo', + binary.path, + '-verify_arch', + 'arm64', + 'x86_64' + ], exitCode: 1, ), ]); @@ -285,7 +310,8 @@ void main() { expect( logger.traceText, - contains('Skipping lipo for non-fat file /FlutterMacOS.framework/Versions/A/FlutterMacOS'), + contains( + 'Skipping lipo for non-fat file /FlutterMacOS.framework/Versions/A/FlutterMacOS'), ); }); @@ -313,7 +339,8 @@ void main() { lipoVerifyX86_64Command, ]); - await const ReleaseUnpackMacOS().build(environment..defines[kBuildMode] = 'release'); + await const ReleaseUnpackMacOS() + .build(environment..defines[kBuildMode] = 'release'); expect(processManager, hasNoRemainingExpectations); }, @@ -335,7 +362,8 @@ void main() { copyFrameworkDsymCommand, ]); - await const ReleaseUnpackMacOS().build(environment..defines[kBuildMode] = 'release'); + await const ReleaseUnpackMacOS() + .build(environment..defines[kBuildMode] = 'release'); expect(processManager, hasNoRemainingExpectations); }, @@ -372,7 +400,8 @@ void main() { ]); await expectLater( - const ReleaseUnpackMacOS().build(environment..defines[kBuildMode] = 'release'), + const ReleaseUnpackMacOS() + .build(environment..defines[kBuildMode] = 'release'), throwsA( isException.having( (Exception exception) => exception.toString(), @@ -393,15 +422,19 @@ void main() { () async { fileSystem .directory( - artifacts.getArtifactPath(Artifact.flutterMacOSFramework, mode: BuildMode.debug), + artifacts.getArtifactPath(Artifact.flutterMacOSFramework, + mode: BuildMode.debug), ) .createSync(); - final String inputKernel = fileSystem.path.join(environment.buildDir.path, 'app.dill'); + final String inputKernel = + fileSystem.path.join(environment.buildDir.path, 'app.dill'); fileSystem.file(inputKernel) ..createSync(recursive: true) ..writeAsStringSync('testing'); - expect(() async => const DebugMacOSBundleFlutterAssets().build(environment), throwsException); + expect( + () async => const DebugMacOSBundleFlutterAssets().build(environment), + throwsException); }, overrides: { FileSystem: () => fileSystem, @@ -414,7 +447,8 @@ void main() { () async { fileSystem .directory( - artifacts.getArtifactPath(Artifact.flutterMacOSFramework, mode: BuildMode.debug), + artifacts.getArtifactPath(Artifact.flutterMacOSFramework, + mode: BuildMode.debug), ) .createSync(); fileSystem @@ -447,20 +481,25 @@ void main() { expect( fileSystem - .file('App.framework/Versions/A/Resources/flutter_assets/kernel_blob.bin') + .file( + 'App.framework/Versions/A/Resources/flutter_assets/kernel_blob.bin') .readAsStringSync(), 'testing', ); expect( - fileSystem.file('App.framework/Versions/A/Resources/Info.plist').readAsStringSync(), + fileSystem + .file('App.framework/Versions/A/Resources/Info.plist') + .readAsStringSync(), contains('io.flutter.flutter.app'), ); expect( - fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/vm_snapshot_data'), + fileSystem.file( + 'App.framework/Versions/A/Resources/flutter_assets/vm_snapshot_data'), exists, ); expect( - fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/isolate_snapshot_data'), + fileSystem.file( + 'App.framework/Versions/A/Resources/flutter_assets/isolate_snapshot_data'), exists, ); }, @@ -563,23 +602,30 @@ void main() { fileSystem .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') .createSync(recursive: true); - fileSystem.file('${environment.buildDir.path}/App.framework/App').createSync(recursive: true); - fileSystem.file('${environment.buildDir.path}/native_assets.json').createSync(); + fileSystem + .file('${environment.buildDir.path}/App.framework/App') + .createSync(recursive: true); + fileSystem + .file('${environment.buildDir.path}/native_assets.json') + .createSync(); await const ProfileMacOSBundleFlutterAssets().build( environment..defines[kBuildMode] = 'profile', ); expect( - fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/kernel_blob.bin'), + fileSystem.file( + 'App.framework/Versions/A/Resources/flutter_assets/kernel_blob.bin'), isNot(exists), ); expect( - fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/vm_snapshot_data'), + fileSystem.file( + 'App.framework/Versions/A/Resources/flutter_assets/vm_snapshot_data'), isNot(exists), ); expect( - fileSystem.file('App.framework/Versions/A/Resources/flutter_assets/isolate_snapshot_data'), + fileSystem.file( + 'App.framework/Versions/A/Resources/flutter_assets/isolate_snapshot_data'), isNot(exists), ); }, @@ -598,17 +644,23 @@ void main() { fileSystem .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') .createSync(recursive: true); - fileSystem.file('${environment.buildDir.path}/App.framework/App').createSync(recursive: true); fileSystem - .file('${environment.buildDir.path}/App.framework.dSYM/Contents/Resources/DWARF/App') + .file('${environment.buildDir.path}/App.framework/App') .createSync(recursive: true); - fileSystem.file('${environment.buildDir.path}/native_assets.json').createSync(); + fileSystem + .file( + '${environment.buildDir.path}/App.framework.dSYM/Contents/Resources/DWARF/App') + .createSync(recursive: true); + fileSystem + .file('${environment.buildDir.path}/native_assets.json') + .createSync(); await const ReleaseMacOSBundleFlutterAssets().build( environment..defines[kBuildMode] = 'release', ); - expect(fileSystem.file('App.framework.dSYM/Contents/Resources/DWARF/App'), exists); + expect(fileSystem.file('App.framework.dSYM/Contents/Resources/DWARF/App'), + exists); }, overrides: { FileSystem: () => fileSystem, @@ -625,17 +677,20 @@ void main() { fileSystem .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') .createSync(recursive: true); - final File inputFramework = - fileSystem.file(fileSystem.path.join(environment.buildDir.path, 'App.framework', 'App')) - ..createSync(recursive: true) - ..writeAsStringSync('ABC'); - fileSystem.file(environment.buildDir.childFile('native_assets.json')).createSync(); + final File inputFramework = fileSystem.file(fileSystem.path + .join(environment.buildDir.path, 'App.framework', 'App')) + ..createSync(recursive: true) + ..writeAsStringSync('ABC'); + fileSystem + .file(environment.buildDir.childFile('native_assets.json')) + .createSync(); await const ProfileMacOSBundleFlutterAssets().build( environment..defines[kBuildMode] = 'profile', ); final File outputFramework = fileSystem.file( - fileSystem.path.join(environment.outputDir.path, 'App.framework', 'App'), + fileSystem.path + .join(environment.outputDir.path, 'App.framework', 'App'), ); expect(outputFramework.readAsStringSync(), 'ABC'); @@ -666,9 +721,12 @@ void main() { .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') .createSync(recursive: true); fileSystem - .file(fileSystem.path.join(environment.buildDir.path, 'App.framework', 'App')) + .file(fileSystem.path + .join(environment.buildDir.path, 'App.framework', 'App')) .createSync(recursive: true); - fileSystem.file(environment.buildDir.childFile('native_assets.json')).createSync(); + fileSystem + .file(environment.buildDir.childFile('native_assets.json')) + .createSync(); await const ReleaseMacOSBundleFlutterAssets().build(environment); expect( @@ -702,7 +760,8 @@ void main() { expect( fakeAnalytics.sentEvents, contains( - Event.appleUsageEvent(workflow: 'assemble', parameter: 'macos-archive', result: 'fail'), + Event.appleUsageEvent( + workflow: 'assemble', parameter: 'macos-archive', result: 'fail'), ), ); }, @@ -739,7 +798,10 @@ void main() { '-install_name', '@rpath/App.framework/App', '-o', - environment.buildDir.childDirectory('App.framework').childFile('App').path, + environment.buildDir + .childDirectory('App.framework') + .childFile('App') + .path, ], ), ); @@ -753,6 +815,75 @@ void main() { }, ); + testUsingContext( + 'ReleaseMacOSBundleFlutterAssets updates shorebird.yaml if present', + () async { + environment.defines[kBuildMode] = 'release'; + environment.defines[kXcodeAction] = 'install'; + environment.defines[kFlavor] = 'internal'; + + // Set up engine artifacts + fileSystem + .file('bin/cache/artifacts/engine/darwin-x64/vm_isolate_snapshot.bin') + .createSync(recursive: true); + fileSystem + .file('bin/cache/artifacts/engine/darwin-x64/isolate_snapshot.bin') + .createSync(recursive: true); + + // Set up App.framework binary + fileSystem + .file(fileSystem.path + .join(environment.buildDir.path, 'App.framework', 'App')) + .createSync(recursive: true); + + // Set up native_assets.json (required by MacOSBundleFlutterAssets) + environment.buildDir.childFile('native_assets.json').createSync(); + + // Set up pubspec.yaml with shorebird.yaml as an asset + fileSystem.file('pubspec.yaml') + ..createSync() + ..writeAsStringSync(''' +name: example +flutter: + assets: + - shorebird.yaml +'''); + + // Create the shorebird.yaml asset file + fileSystem.file('shorebird.yaml') + ..createSync() + ..writeAsStringSync(''' +# Some other text that should be removed +app_id: base-app-id +flavors: + internal: internal-app-id + stable: stable-app-id +'''); + + // Set up package config + writePackageConfigFiles(directory: fileSystem.currentDirectory, mainLibName: 'example'); + + await const ReleaseMacOSBundleFlutterAssets().build(environment); + + // The output is in environment.outputDir, not buildDir + final String shorebirdYamlPath = fileSystem.path.join( + environment.outputDir.path, + 'App.framework', + 'Versions', + 'A', + 'Resources', + 'flutter_assets', + 'shorebird.yaml', + ); + expect(fileSystem.file(shorebirdYamlPath).readAsStringSync(), + 'app_id: internal-app-id'); + }, + overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => processManager, + }, + ); + testUsingContext( 'DebugMacOSFramework creates universal binary', () async { @@ -782,7 +913,10 @@ void main() { '-install_name', '@rpath/App.framework/App', '-o', - environment.buildDir.childDirectory('App.framework').childFile('App').path, + environment.buildDir + .childDirectory('App.framework') + .childFile('App') + .path, ], ), ); @@ -810,11 +944,13 @@ void main() { .childFile('x86_64/App.framework.dSYM/Contents/Resources/DWARF/App') .createSync(recursive: true); + final build = environment.buildDir.path; processManager.addCommands([ FakeCommand( command: [ 'Artifact.genSnapshotArm64.TargetPlatform.darwin.release', '--deterministic', + ...linkInfoArgsFor(build), '--snapshot_kind=app-aot-assembly', '--assembly=${environment.buildDir.childFile('arm64/snapshot_assembly.S').path}', environment.buildDir.childFile('app.dill').path, @@ -824,6 +960,7 @@ void main() { command: [ 'Artifact.genSnapshotX64.TargetPlatform.darwin.release', '--deterministic', + ...linkInfoArgsFor(build), '--snapshot_kind=app-aot-assembly', '--assembly=${environment.buildDir.childFile('x86_64/snapshot_assembly.S').path}', environment.buildDir.childFile('app.dill').path, @@ -951,14 +1088,18 @@ void main() { command: [ 'lipo', environment.buildDir - .childFile('arm64/App.framework.dSYM/Contents/Resources/DWARF/App') + .childFile( + 'arm64/App.framework.dSYM/Contents/Resources/DWARF/App') .path, environment.buildDir - .childFile('x86_64/App.framework.dSYM/Contents/Resources/DWARF/App') + .childFile( + 'x86_64/App.framework.dSYM/Contents/Resources/DWARF/App') .path, '-create', '-output', - environment.buildDir.childFile('App.framework.dSYM/Contents/Resources/DWARF/App').path, + environment.buildDir + .childFile('App.framework.dSYM/Contents/Resources/DWARF/App') + .path, ], ), ]); diff --git a/packages/flutter_tools/test/general.shard/cache_test.dart b/packages/flutter_tools/test/general.shard/cache_test.dart index 4a6152961b800..1e71e52385441 100644 --- a/packages/flutter_tools/test/general.shard/cache_test.dart +++ b/packages/flutter_tools/test/general.shard/cache_test.dart @@ -108,7 +108,8 @@ void main() { final Directory artifactDir = fileSystem.systemTempDirectory.createTempSync( 'flutter_cache_test_artifact.', ); - final Directory downloadDir = fileSystem.systemTempDirectory.createTempSync( + final Directory downloadDir = + fileSystem.systemTempDirectory.createTempSync( 'flutter_cache_test_download.', ); final Cache cache = FakeSecondaryCache() @@ -131,7 +132,8 @@ void main() { final Directory artifactDir = fileSystem.systemTempDirectory.createTempSync( 'flutter_cache_test_artifact.', ); - final Directory downloadDir = fileSystem.systemTempDirectory.createTempSync( + final Directory downloadDir = + fileSystem.systemTempDirectory.createTempSync( 'flutter_cache_test_download.', ); final Cache cache = FakeSecondaryCache() @@ -143,7 +145,8 @@ void main() { final artifact = FakeSimpleArtifact(cache); await artifact.update(FakeArtifactUpdater(), logger, fileSystem, FakeOperatingSystemUtils()); - expect(logger.warningText, contains('No known version for the artifact name "fake"')); + expect(logger.warningText, + contains('No known version for the artifact name "fake"')); }); testWithoutContext( @@ -156,16 +159,20 @@ void main() { fileSystem.path.join('artifacts', 'gradle_wrapper'), ); fileSystem - .file(fileSystem.path.join(directory.path, 'gradle', 'wrapper', 'gradle-wrapper.jar')) + .file(fileSystem.path.join( + directory.path, 'gradle', 'wrapper', 'gradle-wrapper.jar')) .createSync(recursive: true); expect(gradleWrapper.isUpToDateInner(fileSystem), false); }, ); - testWithoutContext('Gradle wrapper will delete .properties/NOTICES if they exist', () async { + testWithoutContext( + 'Gradle wrapper will delete .properties/NOTICES if they exist', + () async { final FileSystem fileSystem = MemoryFileSystem.test(); - final Directory artifactDir = fileSystem.systemTempDirectory.createTempSync( + final Directory artifactDir = + fileSystem.systemTempDirectory.createTempSync( 'flutter_cache_test_artifact.', ); final cache = FakeSecondaryCache() @@ -180,12 +187,15 @@ void main() { ); final gradleWrapper = GradleWrapper(cache); final File propertiesFile = fileSystem.file( - fileSystem.path.join(artifactDir.path, 'gradle', 'wrapper', 'gradle-wrapper.properties'), + fileSystem.path.join( + artifactDir.path, 'gradle', 'wrapper', 'gradle-wrapper.properties'), )..createSync(recursive: true); - final File noticeFile = fileSystem.file(fileSystem.path.join(artifactDir.path, 'NOTICE')) + final File noticeFile = fileSystem + .file(fileSystem.path.join(artifactDir.path, 'NOTICE')) ..createSync(recursive: true); - await gradleWrapper.updateInner(FakeArtifactUpdater(), fileSystem, operatingSystemUtils); + await gradleWrapper.updateInner( + FakeArtifactUpdater(), fileSystem, operatingSystemUtils); expect(propertiesFile, isNot(exists)); expect(noticeFile, isNot(exists)); @@ -201,7 +211,8 @@ void main() { fileSystem.path.join('artifacts', 'gradle_wrapper'), ); fileSystem - .file(fileSystem.path.join(directory.path, 'gradle', 'wrapper', 'gradle-wrapper.jar')) + .file(fileSystem.path.join( + directory.path, 'gradle', 'wrapper', 'gradle-wrapper.jar')) .createSync(recursive: true); fileSystem .file(fileSystem.path.join(directory.path, 'gradlew')) @@ -214,9 +225,12 @@ void main() { }, ); - testWithoutContext('should not be up to date, if some cached artifact is not', () async { - final CachedArtifact artifact1 = FakeSecondaryCachedArtifact()..upToDate = true; - final CachedArtifact artifact2 = FakeSecondaryCachedArtifact()..upToDate = false; + testWithoutContext( + 'should not be up to date, if some cached artifact is not', () async { + final CachedArtifact artifact1 = FakeSecondaryCachedArtifact() + ..upToDate = true; + final CachedArtifact artifact2 = FakeSecondaryCachedArtifact() + ..upToDate = false; final FileSystem fileSystem = MemoryFileSystem.test(); final cache = Cache.test( @@ -252,7 +266,8 @@ void main() { processManager: FakeProcessManager.any(), ); - await cache.updateAll({DevelopmentArtifact.universal}); + await cache + .updateAll({DevelopmentArtifact.universal}); expect(artifact1.didUpdate, false); expect(artifact2.didUpdate, true); }); @@ -299,7 +314,8 @@ void main() { logger: logger, ); await expectLater( - () => cache.updateAll({DevelopmentArtifact.universal}), + () => cache + .updateAll({DevelopmentArtifact.universal}), throwsException, ); expect(artifact1.didUpdate, true); @@ -311,7 +327,9 @@ void main() { testWithoutContext('Invalid URI for FLUTTER_STORAGE_BASE_URL throws ToolExit', () async { final cache = Cache.test( platform: FakePlatform( - environment: {'FLUTTER_STORAGE_BASE_URL': ' http://foo'}, + environment: { + 'FLUTTER_STORAGE_BASE_URL': ' http://foo' + }, ), processManager: FakeProcessManager.any(), ); @@ -329,7 +347,8 @@ void main() { ); expect(cache.storageBaseUrl, baseUrl); - expect(logger.warningText, contains('Flutter assets will be downloaded from $baseUrl')); + expect(logger.warningText, + contains('Flutter assets will be downloaded from $baseUrl')); expect(logger.statusText, isEmpty); }); @@ -354,15 +373,18 @@ void main() { testWithoutContext('flattenNameSubdirs', () { expect( - flattenNameSubdirs(Uri.parse('http://flutter.dev/foo/bar'), MemoryFileSystem.test()), + flattenNameSubdirs( + Uri.parse('http://flutter.dev/foo/bar'), MemoryFileSystem.test()), 'flutter.dev/foo/bar', ); expect( - flattenNameSubdirs(Uri.parse('http://api.flutter.dev/foo/bar'), MemoryFileSystem.test()), + flattenNameSubdirs( + Uri.parse('http://api.flutter.dev/foo/bar'), MemoryFileSystem.test()), 'api.flutter.dev/foo/bar', ); expect( - flattenNameSubdirs(Uri.parse('https://www.flutter.dev'), MemoryFileSystem.test()), + flattenNameSubdirs( + Uri.parse('https://www.flutter.dev'), MemoryFileSystem.test()), 'www.flutter.dev', ); }); @@ -372,10 +394,12 @@ void main() { () async { final operatingSystemUtils = FakeOperatingSystemUtils(); final FileSystem fileSystem = MemoryFileSystem.test(); - final Directory artifactDir = fileSystem.systemTempDirectory.createTempSync( + final Directory artifactDir = + fileSystem.systemTempDirectory.createTempSync( 'flutter_cache_test_artifact.', ); - final Directory downloadDir = fileSystem.systemTempDirectory.createTempSync( + final Directory downloadDir = + fileSystem.systemTempDirectory.createTempSync( 'flutter_cache_test_download.', ); final cache = FakeSecondaryCache() @@ -400,7 +424,10 @@ void main() { expect(dir, isNotNull); expect(dir.path, artifactDir.childDirectory('bin_dir').path); expect(operatingSystemUtils.chmods, >[ - ['/.tmp_rand0/flutter_cache_test_artifact.rand0/bin_dir', 'a+r,a+x'], + [ + '/.tmp_rand0/flutter_cache_test_artifact.rand0/bin_dir', + 'a+r,a+x' + ], ]); }, ); @@ -433,7 +460,8 @@ void main() { packageUrl = url; }; - await artifact.updateInner(artifactUpdater, fileSystem, operatingSystemUtils); + await artifact.updateInner( + artifactUpdater, fileSystem, operatingSystemUtils); expect(packageUrl, isNotNull); expect( packageUrl.toString(), @@ -483,8 +511,8 @@ void main() { platform: FakePlatform(operatingSystem: 'macos'), ); iosUsbArtifacts.location.createSync(); - final File ideviceScreenshotFile = iosUsbArtifacts.location.childFile('idevicescreenshot') - ..createSync(); + final File ideviceScreenshotFile = + iosUsbArtifacts.location.childFile('idevicescreenshot')..createSync(); iosUsbArtifacts.location.childFile('idevicesyslog').createSync(); expect(iosUsbArtifacts.isUpToDateInner(fileSystem), true); @@ -504,7 +532,8 @@ void main() { platform: FakePlatform(operatingSystem: 'macos'), ); iosUsbArtifacts.location.createSync(); - final File iproxy = iosUsbArtifacts.location.childFile('iproxy')..createSync(); + final File iproxy = iosUsbArtifacts.location.childFile('iproxy') + ..createSync(); expect(iosUsbArtifacts.isUpToDateInner(fileSystem), true); @@ -549,10 +578,13 @@ void main() { platform: FakePlatform(operatingSystem: 'macos'), ); - expect(iosUsbArtifacts.archiveUri.toString(), isNot(contains('/unsigned/'))); + expect( + iosUsbArtifacts.archiveUri.toString(), isNot(contains('/unsigned/'))); }); - testWithoutContext('FlutterRunnerDebugSymbols downloads Flutter runner debug symbols', () async { + testWithoutContext( + 'FlutterRunnerDebugSymbols downloads Flutter runner debug symbols', + () async { final FileSystem fileSystem = MemoryFileSystem.test(); final Cache cache = FakeSecondaryCache() ..artifactDirectory = fileSystem.currentDirectory @@ -623,7 +655,8 @@ void main() { testWithoutContext('FontSubset artifacts on macos', () { fakeProcessManager.addCommands([ - const FakeCommand(command: ['which', 'sysctl'], stdout: '/sbin/sysctl'), + const FakeCommand( + command: ['which', 'sysctl'], stdout: '/sbin/sysctl'), const FakeCommand( command: ['sysctl', 'hw.optional.arm64'], stdout: 'hw.optional.arm64: 0', @@ -672,7 +705,8 @@ void main() { ]); }); - testWithoutContext('FontSubset artifacts for all platforms on arm64 hosts', () { + testWithoutContext('FontSubset artifacts for all platforms on arm64 hosts', + () { fakeProcessManager.addCommand(unameCommandForArm64); final Cache cache = createCache(FakePlatform(operatingSystem: 'fuchsia')); @@ -732,11 +766,15 @@ void main() { expect( artifacts.getBinaryDirs(), - containsAll([contains(contains('profile')), contains(contains('release'))]), + containsAll([ + contains(contains('profile')), + contains(contains('release')) + ]), ); }); - testWithoutContext('Linux desktop artifacts ignore filtering when requested', () { + testWithoutContext('Linux desktop artifacts ignore filtering when requested', + () { fakeProcessManager.addCommand(unameCommandForX64); final Cache cache = createCache(FakePlatform()); @@ -747,7 +785,9 @@ void main() { expect(artifacts.getBinaryDirs(), isNotEmpty); }); - testWithoutContext('Linux desktop artifacts for x64 include profile and release artifacts', () { + testWithoutContext( + 'Linux desktop artifacts for x64 include profile and release artifacts', + () { fakeProcessManager.addCommand(unameCommandForX64); final Cache cache = createCache(FakePlatform()); @@ -755,12 +795,20 @@ void main() { expect(artifacts.getBinaryDirs(), >[ ['linux-x64', 'linux-x64-debug/linux-x64-flutter-gtk.zip'], - ['linux-x64-profile', 'linux-x64-profile/linux-x64-flutter-gtk.zip'], - ['linux-x64-release', 'linux-x64-release/linux-x64-flutter-gtk.zip'], + [ + 'linux-x64-profile', + 'linux-x64-profile/linux-x64-flutter-gtk.zip' + ], + [ + 'linux-x64-release', + 'linux-x64-release/linux-x64-flutter-gtk.zip' + ], ]); }); - testWithoutContext('Linux desktop artifacts for arm64 include profile and release artifacts', () { + testWithoutContext( + 'Linux desktop artifacts for arm64 include profile and release artifacts', + () { fakeProcessManager.addCommand(unameCommandForArm64); final Cache cache = createCache(FakePlatform()); @@ -768,8 +816,14 @@ void main() { expect(artifacts.getBinaryDirs(), >[ ['linux-arm64', 'linux-arm64-debug/linux-arm64-flutter-gtk.zip'], - ['linux-arm64-profile', 'linux-arm64-profile/linux-arm64-flutter-gtk.zip'], - ['linux-arm64-release', 'linux-arm64-release/linux-arm64-flutter-gtk.zip'], + [ + 'linux-arm64-profile', + 'linux-arm64-profile/linux-arm64-flutter-gtk.zip' + ], + [ + 'linux-arm64-release', + 'linux-arm64-release/linux-arm64-flutter-gtk.zip' + ], ]); }); @@ -799,7 +853,8 @@ void main() { expect(toolStampFile, isNot(exists)); }); - testWithoutContext('Cache does not attempt to delete already missing stamp files', () { + testWithoutContext( + 'Cache does not attempt to delete already missing stamp files', () { final FileSystem fileSystem = MemoryFileSystem.test(); final artifactSet = FakeIosUsbArtifacts(); final logger = BufferLogger.test(); @@ -824,7 +879,8 @@ void main() { expect(toolStampFile, isNot(exists)); }); - testWithoutContext('Cache catches file system exception from missing tool stamp file', () { + testWithoutContext( + 'Cache catches file system exception from missing tool stamp file', () { final FileSystem fileSystem = MemoryFileSystem.test(); final artifactSet = FakeIosUsbArtifacts(); final logger = BufferLogger.test(); @@ -851,7 +907,8 @@ void main() { final Directory internalDir = fileSystem.currentDirectory .childDirectory('bin') .childDirectory('internal'); - final File canvasKitVersionFile = internalDir.childFile('canvaskit.version'); + final File canvasKitVersionFile = + internalDir.childFile('canvaskit.version'); canvasKitVersionFile.createSync(recursive: true); canvasKitVersionFile.writeAsStringSync('abcdefg'); @@ -879,12 +936,13 @@ void main() { }; webCacheDirectory.childFile('bar').createSync(recursive: true); - await webSdk.updateInner(artifactUpdater, fileSystem, FakeOperatingSystemUtils()); + await webSdk.updateInner( + artifactUpdater, fileSystem, FakeOperatingSystemUtils()); expect(messages, ['Downloading Web SDK...']); expect(downloads, [ - 'https://storage.googleapis.com/flutter_infra_release/flutter/hijklmnop/flutter-web-sdk.zip', + 'https://download.shorebird.dev/flutter_infra_release/flutter/hijklmnop/flutter-web-sdk.zip', ]); expect(locations, ['/bin/cache/flutter_web_sdk']); @@ -901,7 +959,8 @@ void main() { final Directory internalDir = fileSystem.currentDirectory .childDirectory('bin') .childDirectory('internal'); - final File canvasKitVersionFile = internalDir.childFile('canvaskit.version'); + final File canvasKitVersionFile = + internalDir.childFile('canvaskit.version'); canvasKitVersionFile.createSync(recursive: true); canvasKitVersionFile.writeAsStringSync('abcdefg'); @@ -935,7 +994,8 @@ void main() { }; webCacheDirectory.childFile('bar').createSync(recursive: true); - await webSdk.updateInner(artifactUpdater, fileSystem, FakeOperatingSystemUtils()); + await webSdk.updateInner( + artifactUpdater, fileSystem, FakeOperatingSystemUtils()); expect(downloads, [ 'https://flutter.storage.com/override/flutter_infra_release/flutter/hijklmnop/flutter-web-sdk.zip', @@ -970,9 +1030,11 @@ void main() { ); await expectLater( - () => webSdk.updateInner(artifactUpdater, fileSystem, FakeOperatingSystemUtils()), + () => webSdk.updateInner( + artifactUpdater, fileSystem, FakeOperatingSystemUtils()), throwsToolExit( - message: RegExp('Unable to delete file or directory at "/bin/cache/flutter_web_sdk"'), + message: RegExp( + 'Unable to delete file or directory at "/bin/cache/flutter_web_sdk"'), ), ); }); @@ -1005,7 +1067,7 @@ void main() { expect(messages, ['Downloading engine information...']); expect(downloads, [ - 'https://storage.googleapis.com/flutter_infra_release/flutter/hijklmnop/engine_stamp.json', + 'https://download.shorebird.dev/flutter_infra_release/flutter/hijklmnop/engine_stamp.json', ]); expect(locations, ['/bin/cache']); // file copy is done by the real uploader; not the fake. @@ -1018,7 +1080,8 @@ void main() { final fileSystem = MemoryFileSystem.test(opHandle: handler.opHandle); final cache = Cache.test(processManager: FakeProcessManager.any(), fileSystem: fileSystem); final File canvasKitWasm = fileSystem.file( - fileSystem.path.join(cache.getRoot().path, 'canvaskit', 'canvaskit.wasm'), + fileSystem.path + .join(cache.getRoot().path, 'canvaskit', 'canvaskit.wasm'), ); canvasKitWasm.createSync(recursive: true); canvasKitWasm.writeAsStringSync('hello world'); @@ -1053,7 +1116,8 @@ void main() { expect(cache.getStampFor('foo'), null); file.createSync(); - exceptionHandler.addError(file, FileSystemOp.read, const FileSystemException()); + exceptionHandler.addError( + file, FileSystemOp.read, const FileSystemException()); expect(cache.getStampFor('foo'), null); }); @@ -1074,7 +1138,8 @@ void main() { expect(cache.getStampFor('foo'), 'ABC'); }); - testWithoutContext('PubDependencies needs to be updated if the package config' + testWithoutContext( + 'PubDependencies needs to be updated if the package config' ' file or the source directories are missing', () async { final logger = BufferLogger.test(); final fileSystem = MemoryFileSystem.test(); @@ -1085,7 +1150,8 @@ void main() { projectFactory: FakeFlutterProjectFactory(), ); - expect(await pubDependencies.isUpToDate(fileSystem), false); // no package config + expect(await pubDependencies.isUpToDate(fileSystem), + false); // no package config fileSystem.file('packages/flutter_tools/.dart_tool/package_config.json') ..createSync(recursive: true) @@ -1106,7 +1172,8 @@ void main() { } '''); - expect(await pubDependencies.isUpToDate(fileSystem), false); // dependencies are missing. + expect(await pubDependencies.isUpToDate(fileSystem), + false); // dependencies are missing. fileSystem .file('.pub-cache/hosted/pub.dartlang.org/example-7.0.0/pubspec.yaml') @@ -1137,7 +1204,8 @@ void main() { expect( pub.invocations.first, predicate( - (FakePubInvocation invocation) => invocation.outputMode == PubOutputMode.failuresOnly, + (FakePubInvocation invocation) => + invocation.outputMode == PubOutputMode.failuresOnly, 'Pub invoked with PubOutputMode.none', ), ); @@ -1165,7 +1233,9 @@ void main() { setUp(() { memoryFileSystem = MemoryFileSystem.test(); - cache = Cache.test(fileSystem: memoryFileSystem, processManager: FakeProcessManager.any()); + cache = Cache.test( + fileSystem: memoryFileSystem, + processManager: FakeProcessManager.any()); fakeAndroidSdk = FakeAndroidSdk(); }); @@ -1175,7 +1245,8 @@ void main() { java: FakeJava(), platform: FakePlatform(), ); - expect(mavenArtifacts.developmentArtifact, DevelopmentArtifact.androidMaven); + expect( + mavenArtifacts.developmentArtifact, DevelopmentArtifact.androidMaven); }); testUsingContext( @@ -1191,10 +1262,13 @@ void main() { ); expect(await mavenArtifacts.isUpToDate(memoryFileSystem!), isFalse); - final Directory gradleWrapperDir = cache!.getArtifactDirectory('gradle_wrapper') + final Directory gradleWrapperDir = cache! + .getArtifactDirectory('gradle_wrapper') ..createSync(recursive: true); gradleWrapperDir.childFile('gradlew').writeAsStringSync('irrelevant'); - gradleWrapperDir.childFile('gradlew.bat').writeAsStringSync('irrelevant'); + gradleWrapperDir + .childFile('gradlew.bat') + .writeAsStringSync('irrelevant'); await mavenArtifacts.update( FakeArtifactUpdater(), @@ -1283,7 +1357,8 @@ class FakeCachedArtifact extends EngineCachedArtifact { } class FakeSimpleArtifact extends CachedArtifact { - FakeSimpleArtifact(Cache cache) : super('fake', cache, DevelopmentArtifact.universal); + FakeSimpleArtifact(Cache cache) + : super('fake', cache, DevelopmentArtifact.universal); @override Future updateInner( @@ -1432,12 +1507,14 @@ class FakeArtifactUpdater extends Fake implements ArtifactUpdater { void Function(String, Uri, Directory)? onDownloadFile; @override - Future downloadZippedTarball(String message, Uri url, Directory location) async { + Future downloadZippedTarball( + String message, Uri url, Directory location) async { onDownloadZipTarball?.call(message, url, location); } @override - Future downloadZipArchive(String message, Uri url, Directory location) async { + Future downloadZipArchive( + String message, Uri url, Directory location) async { onDownloadZipArchive?.call(message, url, location); } diff --git a/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart b/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart new file mode 100644 index 0000000000000..7e05fec171409 --- /dev/null +++ b/packages/flutter_tools/test/general.shard/shorebird/shorebird_yaml_test.dart @@ -0,0 +1,104 @@ +// Copyright 2024 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'package:flutter_tools/src/shorebird/shorebird_yaml.dart'; +import 'package:test/test.dart'; +import 'package:yaml/yaml.dart'; + +void main() { + group('ShorebirdYaml', () { + test('yaml ignores comments', () { + const String yamlContents = ''' +# This file is used to configure the Shorebird updater used by your app. +app_id: 6160a7d8-cc18-4928-1233-05b51c0bb02c + +# auto_update controls if Shorebird should automatically update in the background on launch. +auto_update: false +'''; + final YamlDocument input = loadYamlDocument(yamlContents); + final YamlMap yamlMap = input.contents as YamlMap; + final Map compiled = + compileShorebirdYaml(yamlMap, flavor: null, environment: {}); + expect(compiled, { + 'app_id': '6160a7d8-cc18-4928-1233-05b51c0bb02c', + 'auto_update': false, + }); + }); + test('flavors', () { + // These are invalid app_ids but make for easy testing. + const String yamlContents = ''' +app_id: 1-a +auto_update: false +flavors: + foo: 2-a + bar: 3-a +'''; + final YamlDocument input = loadYamlDocument(yamlContents); + final YamlMap yamlMap = input.contents as YamlMap; + expect(appIdForFlavor(yamlMap, flavor: null), '1-a'); + expect(appIdForFlavor(yamlMap, flavor: 'foo'), '2-a'); + expect(appIdForFlavor(yamlMap, flavor: 'bar'), '3-a'); + expect(() => appIdForFlavor(yamlMap, flavor: 'unknown'), throwsException); + }); + test('all values', () { + // These are invalid app_ids but make for easy testing. + const String yamlContents = ''' +app_id: 1-a +auto_update: false +flavors: + foo: 2-a + bar: 3-a +base_url: https://example.com +patch_verification: strict +'''; + final YamlDocument input = loadYamlDocument(yamlContents); + final YamlMap yamlMap = input.contents as YamlMap; + final Map compiled1 = + compileShorebirdYaml(yamlMap, flavor: null, environment: {}); + expect(compiled1, { + 'app_id': '1-a', + 'auto_update': false, + 'base_url': 'https://example.com', + 'patch_verification': 'strict', + }); + final Map compiled2 = + compileShorebirdYaml(yamlMap, flavor: 'foo', environment: {'SHOREBIRD_PUBLIC_KEY': '4-a'}); + expect(compiled2, { + 'app_id': '2-a', + 'auto_update': false, + 'base_url': 'https://example.com', + 'patch_verification': 'strict', + 'patch_public_key': '4-a', + }); + }); + test('edit in place', () { + const String yamlContents = ''' +app_id: 1-a +auto_update: false +flavors: + foo: 2-a + bar: 3-a +base_url: https://example.com +'''; + // Make a temporary file to test editing in place. + final Directory tempDir = Directory.systemTemp.createTempSync('shorebird_yaml_test.'); + final File tempFile = File('${tempDir.path}/shorebird.yaml'); + tempFile.writeAsStringSync(yamlContents); + updateShorebirdYaml( + 'foo', + tempFile.path, + environment: {'SHOREBIRD_PUBLIC_KEY': '4-a'}, + ); + final String updatedContents = tempFile.readAsStringSync(); + // Order is not guaranteed, so parse as YAML to compare. + final YamlDocument updated = loadYamlDocument(updatedContents); + final YamlMap yamlMap = updated.contents as YamlMap; + expect(yamlMap['app_id'], '2-a'); + expect(yamlMap['auto_update'], false); + expect(yamlMap['base_url'], 'https://example.com'); + }); + }); +} diff --git a/packages/flutter_tools/test/general.shard/version_test.dart b/packages/flutter_tools/test/general.shard/version_test.dart index 1d8360c14866c..5d66fee5f3529 100644 --- a/packages/flutter_tools/test/general.shard/version_test.dart +++ b/packages/flutter_tools/test/general.shard/version_test.dart @@ -67,6 +67,20 @@ void main() { required int commitsBetweenRefs, }) { return [ + // Shorebird release branch check (returns empty to fall through to + // regular version lookup). + FakeCommand( + command: [ + 'git', + 'for-each-ref', + '--contains', + headRef, + '--format', + '%(refname:short)', + 'refs/remotes/origin/flutter_release/*', + ], + stdout: '', + ), FakeCommand( command: const [ 'git', diff --git a/packages/shorebird_tests/.gitignore b/packages/shorebird_tests/.gitignore new file mode 100644 index 0000000000000..3a85790408401 --- /dev/null +++ b/packages/shorebird_tests/.gitignore @@ -0,0 +1,3 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ diff --git a/packages/shorebird_tests/README.md b/packages/shorebird_tests/README.md new file mode 100644 index 0000000000000..b87fb24a480da --- /dev/null +++ b/packages/shorebird_tests/README.md @@ -0,0 +1,2 @@ +A dart project that includes tests that perform asserts in the modifications +made on the Flutter framework by the Shorebird team. diff --git a/packages/shorebird_tests/analysis_options.yaml b/packages/shorebird_tests/analysis_options.yaml new file mode 100644 index 0000000000000..a767d79d7f4b1 --- /dev/null +++ b/packages/shorebird_tests/analysis_options.yaml @@ -0,0 +1,2 @@ +# This file configures the static analysis results for your project (errors, +include: package:lints/recommended.yaml diff --git a/packages/shorebird_tests/pubspec.yaml b/packages/shorebird_tests/pubspec.yaml new file mode 100644 index 0000000000000..d0fba0c1fa95d --- /dev/null +++ b/packages/shorebird_tests/pubspec.yaml @@ -0,0 +1,16 @@ +name: shorebird_tests +description: Shorebird's Flutter customizations tests +version: 1.0.0 + +environment: + sdk: ^3.3.4 + +dependencies: + archive: ^3.5.1 + path: ^1.9.0 + yaml: ^3.1.2 + +dev_dependencies: + lints: ^3.0.0 + meta: ^1.15.0 + test: ^1.24.0 diff --git a/packages/shorebird_tests/test/android_test.dart b/packages/shorebird_tests/test/android_test.dart new file mode 100644 index 0000000000000..1e0a6da4f3824 --- /dev/null +++ b/packages/shorebird_tests/test/android_test.dart @@ -0,0 +1,94 @@ +import 'package:test/test.dart'; + +import 'shorebird_tests.dart'; + +void main() { + setUpAll(warmUpTemplateProject); + + group('shorebird android projects', () { + testWithShorebirdProject('can build an apk', (projectDirectory) async { + await projectDirectory.runFlutterBuildApk(); + + expect(projectDirectory.apkFile().existsSync(), isTrue); + expect(projectDirectory.shorebirdFile.existsSync(), isTrue); + expect(projectDirectory.getGeneratedAndroidShorebirdYaml(), completes); + }); + + group('when passing the public key through the environment variable', () { + testWithShorebirdProject( + 'adds the public key on top of the original file', + (projectDirectory) async { + final originalYaml = projectDirectory.shorebirdYaml; + + const base64PublicKey = 'public_123'; + await projectDirectory.runFlutterBuildApk( + environment: { + 'SHOREBIRD_PUBLIC_KEY': base64PublicKey, + }, + ); + + final generatedYaml = + await projectDirectory.getGeneratedAndroidShorebirdYaml(); + + expect( + generatedYaml.keys, + containsAll(originalYaml.keys), + ); + + expect( + generatedYaml['patch_public_key'], + equals(base64PublicKey), + ); + }, + ); + }); + + group('when building with a flavor', () { + testWithShorebirdProject( + 'correctly changes the app id', + (projectDirectory) async { + await projectDirectory.addProjectFlavors(); + projectDirectory.addShorebirdFlavors(); + + await projectDirectory.runFlutterBuildApk(flavor: 'internal'); + + final generatedYaml = + await projectDirectory.getGeneratedAndroidShorebirdYaml( + flavor: 'internal', + ); + + expect(generatedYaml['app_id'], equals('internal_123')); + }, + ); + + group('when public key passed through environment variable', () { + testWithShorebirdProject( + 'correctly changes the app id and adds the public key', + (projectDirectory) async { + const base64PublicKey = 'public_123'; + await projectDirectory.addProjectFlavors(); + projectDirectory.addShorebirdFlavors(); + + await projectDirectory.runFlutterBuildApk( + flavor: 'internal', + environment: { + 'SHOREBIRD_PUBLIC_KEY': base64PublicKey, + }, + ); + + final generatedYaml = + await projectDirectory.getGeneratedAndroidShorebirdYaml( + flavor: 'internal', + ); + + expect(generatedYaml['app_id'], equals('internal_123')); + expect( + generatedYaml['patch_public_key'], + equals(base64PublicKey), + ); + }, + ); + }); + }); + }); +} diff --git a/packages/shorebird_tests/test/base_test.dart b/packages/shorebird_tests/test/base_test.dart new file mode 100644 index 0000000000000..ce9b0564b1396 --- /dev/null +++ b/packages/shorebird_tests/test/base_test.dart @@ -0,0 +1,17 @@ +import 'package:test/test.dart'; + +import 'shorebird_tests.dart'; + +void main() { + setUpAll(warmUpTemplateProject); + + group('shorebird helpers', () { + testWithShorebirdProject('can build a base project', + (projectDirectory) async { + expect(projectDirectory.existsSync(), isTrue); + + expect(projectDirectory.pubspecFile.existsSync(), isTrue); + expect(projectDirectory.shorebirdFile.existsSync(), isTrue); + }); + }); +} diff --git a/packages/shorebird_tests/test/ios_test.dart b/packages/shorebird_tests/test/ios_test.dart new file mode 100644 index 0000000000000..bb694405ae946 --- /dev/null +++ b/packages/shorebird_tests/test/ios_test.dart @@ -0,0 +1,93 @@ +import 'package:test/test.dart'; + +import 'shorebird_tests.dart'; + +void main() { + setUpAll(warmUpTemplateProject); + + group( + 'shorebird ios projects', + () { + testWithShorebirdProject('can build', (projectDirectory) async { + await projectDirectory.runFlutterBuildIos(); + + expect(projectDirectory.iosArchiveFile().existsSync(), isTrue); + expect(projectDirectory.getGeneratedIosShorebirdYaml(), completes); + }); + + group('when passing the public key through the environment variable', () { + testWithShorebirdProject( + 'adds the public key on top of the original file', + (projectDirectory) async { + final originalYaml = projectDirectory.shorebirdYaml; + + const base64PublicKey = 'public_123'; + await projectDirectory.runFlutterBuildIos( + environment: { + 'SHOREBIRD_PUBLIC_KEY': base64PublicKey, + }, + ); + + final generatedYaml = + await projectDirectory.getGeneratedIosShorebirdYaml(); + + expect( + generatedYaml.keys, + containsAll(originalYaml.keys), + ); + + expect( + generatedYaml['patch_public_key'], + equals(base64PublicKey), + ); + }, + ); + }); + + group('when building with a flavor', () { + testWithShorebirdProject( + 'correctly changes the app id', + (projectDirectory) async { + await projectDirectory.addProjectFlavors(); + projectDirectory.addShorebirdFlavors(); + + await projectDirectory.runFlutterBuildIos(flavor: 'internal'); + + final generatedYaml = + await projectDirectory.getGeneratedIosShorebirdYaml(); + + expect(generatedYaml['app_id'], equals('internal_123')); + }, + ); + + group('when public key passed through environment variable', () { + testWithShorebirdProject( + 'correctly changes the app id and adds the public key', + (projectDirectory) async { + const base64PublicKey = 'public_123'; + await projectDirectory.addProjectFlavors(); + projectDirectory.addShorebirdFlavors(); + + await projectDirectory.runFlutterBuildIos( + flavor: 'internal', + environment: { + 'SHOREBIRD_PUBLIC_KEY': base64PublicKey, + }, + ); + + final generatedYaml = + await projectDirectory.getGeneratedIosShorebirdYaml(); + + expect(generatedYaml['app_id'], equals('internal_123')); + expect( + generatedYaml['patch_public_key'], + equals(base64PublicKey), + ); + }, + ); + }); + }); + }, + testOn: 'mac-os', + ); +} diff --git a/packages/shorebird_tests/test/shorebird_tests.dart b/packages/shorebird_tests/test/shorebird_tests.dart new file mode 100644 index 0000000000000..409f3963e95e1 --- /dev/null +++ b/packages/shorebird_tests/test/shorebird_tests.dart @@ -0,0 +1,385 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:archive/archive_io.dart'; +import 'package:path/path.dart' as path; + +import 'package:meta/meta.dart'; +import 'package:test/test.dart'; +import 'package:yaml/yaml.dart'; + +/// This will be the path to the flutter binary housed in this flutter repository. +/// +/// Which since we are running the tests from this inner package , we need to go up two directories +/// in order to find the flutter binary in the bin folder. +File get _flutterBinaryFile => File( + path.join( + Directory.current.path, + '..', + '..', + 'bin', + 'flutter${Platform.isWindows ? '.bat' : ''}', + ), + ); + +/// Runs a flutter command using the correct binary ([_flutterBinaryFile]) with the given arguments. +/// +/// Streams stdout and stderr to the test output in real time so that +/// CI logs show progress even if the process hangs or times out. +Future _runFlutterCommand( + List arguments, { + required Directory workingDirectory, + Map? environment, +}) async { + final String command = 'flutter ${arguments.join(' ')}'; + print('[$command] starting...'); + final stopwatch = Stopwatch()..start(); + + final Process process = await Process.start( + _flutterBinaryFile.absolute.path, + arguments, + workingDirectory: workingDirectory.path, + environment: { + 'FLUTTER_STORAGE_BASE_URL': 'https://download.shorebird.dev', + if (environment != null) ...environment, + }, + ); + + final StringBuffer stdoutBuffer = StringBuffer(); + final StringBuffer stderrBuffer = StringBuffer(); + + process.stdout.transform(utf8.decoder).listen((String data) { + stdoutBuffer.write(data); + // Print each line with a prefix so it's easy to identify in CI logs. + for (final String line in data.split('\n')) { + if (line.isNotEmpty) { + print(' [$command] $line'); + } + } + }); + + process.stderr.transform(utf8.decoder).listen((String data) { + stderrBuffer.write(data); + for (final String line in data.split('\n')) { + if (line.isNotEmpty) { + print(' [$command] (stderr) $line'); + } + } + }); + + final int exitCode = await process.exitCode; + stopwatch.stop(); + print('[$command] completed in ${stopwatch.elapsed} ' + '(exit code $exitCode)'); + + return ProcessResult( + process.pid, + exitCode, + stdoutBuffer.toString(), + stderrBuffer.toString(), + ); +} + +Future _createFlutterProject(Directory projectDirectory) async { + final result = await _runFlutterCommand( + ['create', '--empty', '.'], + workingDirectory: projectDirectory, + ); + if (result.exitCode != 0) { + throw Exception('Failed to create Flutter project: ${result.stderr}'); + } +} + +/// Cached template project directory, created once and reused across tests. +/// +/// This avoids running `flutter create` for every test, which saves +/// significant time (especially the first Gradle/SDK download). +Directory? _templateProject; + +/// Creates (or returns the cached) template Flutter project with +/// shorebird.yaml configured. The first call runs `flutter create` and +/// `flutter build apk` to warm up Gradle caches. +/// +/// Call this from `setUpAll` so the expensive setup runs outside per-test +/// timeouts. +Future warmUpTemplateProject() => _getTemplateProject(); + +Future _getTemplateProject() async { + if (_templateProject != null) { + return _templateProject!; + } + + final Directory templateDir = Directory( + path.join(Directory.systemTemp.createTempSync().path, 'shorebird_template'), + )..createSync(); + + await _createFlutterProject(templateDir); + + templateDir.pubspecFile.writeAsStringSync(''' +${templateDir.pubspecFile.readAsStringSync()} + assets: + - shorebird.yaml +'''); + + File( + path.join(templateDir.path, 'shorebird.yaml'), + ).writeAsStringSync(''' +app_id: "123" +'''); + + // Warm up the Gradle cache with a throwaway build so subsequent + // per-test builds are fast and don't hit the per-test timeout. + // Skip if Gradle cache is already populated (e.g., from GHA cache restore). + final Directory gradleCache = Directory( + path.join(Platform.environment['HOME'] ?? '', '.gradle', 'caches'), + ); + final bool hasGradleCache = + gradleCache.existsSync() && gradleCache.listSync().isNotEmpty; + if (hasGradleCache) { + print('[warmup] Gradle cache exists, skipping warm-up build'); + } else { + await _runFlutterCommand( + ['build', 'apk'], + workingDirectory: templateDir, + ); + } + + _templateProject = templateDir; + return templateDir; +} + +/// Copies the template project to a fresh directory for test isolation. +Future _copyTemplateProject() async { + final Directory template = await _getTemplateProject(); + final Directory testDir = Directory( + path.join(Directory.systemTemp.createTempSync().path, 'shorebird_test'), + ); + + // Use platform copy to preserve the full directory tree efficiently. + final ProcessResult result; + if (Platform.isWindows) { + result = await Process.run('xcopy', [ + template.path, + testDir.path, + '/E', + '/I', + '/Q', + ]); + } else { + result = await Process.run('cp', ['-R', template.path, testDir.path]); + } + if (result.exitCode != 0) { + throw Exception('Failed to copy template project: ${result.stderr}'); + } + + return testDir; +} + +@isTest +Future testWithShorebirdProject(String name, + FutureOr Function(Directory projectDirectory) testFn) async { + test( + name, + () async { + final Directory projectDirectory = await _copyTemplateProject(); + + try { + await testFn(projectDirectory); + } finally { + projectDirectory.deleteSync(recursive: true); + } + }, + timeout: Timeout( + // Per-test timeout can be shorter now since the template project + // creation and Gradle warm-up happen outside the test timeout. + Duration(minutes: 6), + ), + ); +} + +extension ShorebirdProjectDirectoryOnDirectory on Directory { + File get pubspecFile => File( + path.join(this.path, 'pubspec.yaml'), + ); + + File get shorebirdFile => File( + path.join(this.path, 'shorebird.yaml'), + ); + + YamlMap get shorebirdYaml => + loadYaml(shorebirdFile.readAsStringSync()) as YamlMap; + + File get appGradleFile => File( + path.join(this.path, 'android', 'app', 'build.gradle'), + ); + + Future addPubDependency(String name, {bool dev = false}) async { + final result = await _runFlutterCommand( + ['pub', 'add', if (dev) '--dev', name], + workingDirectory: this, + ); + if (result.exitCode != 0) { + throw Exception( + 'Failed to run `flutter pub add $name`: ${result.stderr}'); + } + } + + Future addProjectFlavors() async { + await addPubDependency( + // TODO(felangel): revert to using published version once 3.29.0 support is released. + // https://github.com/AngeloAvv/flutter_flavorizr/pull/291 + 'dev:flutter_flavorizr:{"git":{"url":"https://github.com/wjlee611/flutter_flavorizr.git","ref":"chore/temp-migrate-3-29","path":"."}}', + ); + + await File( + path.join( + this.path, + 'flavorizr.yaml', + ), + ).writeAsString(''' +flavors: + playStore: + app: + name: "App" + + android: + applicationId: "com.example.shorebird_test" + ios: + bundleId: "com.example.shorebird_test" + internal: + app: + name: "App (Internal)" + + android: + applicationId: "com.example.shorebird_test.internal" + ios: + bundleId: "com.example.shorebird_test.internal" + global: + app: + name: "App (Global)" + + android: + applicationId: "com.example.shorebird_test.global" + ios: + bundleId: "com.example.shorebird_test.global" +'''); + + final result = await _runFlutterCommand( + ['pub', 'run', 'flutter_flavorizr'], + workingDirectory: this, + ); + if (result.exitCode != 0) { + throw Exception( + 'Failed to run `flutter pub run flutter_flavorizr`: ${result.stderr}'); + } + } + + void addShorebirdFlavors() { + const flavors = ''' +flavors: + global: global_123 + internal: internal_123 + playStore: playStore_123 +'''; + + final currentShorebirdContent = shorebirdFile.readAsStringSync(); + shorebirdFile.writeAsStringSync( + ''' +$currentShorebirdContent +$flavors +''', + ); + } + + Future runFlutterBuildApk({ + String? flavor, + Map? environment, + }) async { + final result = await _runFlutterCommand( + [ + 'build', + 'apk', + if (flavor != null) '--flavor=$flavor', + ], + workingDirectory: this, + environment: environment, + ); + if (result.exitCode != 0) { + throw Exception('Failed to run `flutter build apk`: ${result.stderr}'); + } + } + + Future runFlutterBuildIos({ + Map? environment, + String? flavor, + }) async { + final result = await _runFlutterCommand( + // The projects used to test are generated on spot, to make it simpler we don't + // configure any apple accounts on it, so we skip code signing here. + ['build', 'ipa', '--no-codesign', if (flavor != null) '--flavor=$flavor'], + workingDirectory: this, + environment: environment, + ); + + if (result.exitCode != 0) { + throw Exception('Failed to run `flutter build ios`: ${result.stderr}'); + } + } + + File apkFile({String? flavor}) => File( + path.join( + this.path, + 'build', + 'app', + 'outputs', + 'flutter-apk', + 'app-${flavor != null ? '$flavor-' : ''}release.apk', + ), + ); + + Directory iosArchiveFile() => Directory( + path.join( + this.path, + 'build', + 'ios', + 'archive', + 'Runner.xcarchive', + ), + ); + + Future getGeneratedAndroidShorebirdYaml({String? flavor}) async { + final decodedBytes = + ZipDecoder().decodeBytes(apkFile(flavor: flavor).readAsBytesSync()); + + await extractArchiveToDisk( + decodedBytes, path.join(this.path, 'apk-extracted')); + + final yamlString = File( + path.join( + this.path, + 'apk-extracted', + 'assets', + 'flutter_assets', + 'shorebird.yaml', + ), + ).readAsStringSync(); + return loadYaml(yamlString) as YamlMap; + } + + Future getGeneratedIosShorebirdYaml() async { + final yamlString = File( + path.join( + iosArchiveFile().path, + 'Products', + 'Applications', + 'Runner.app', + 'Frameworks', + 'App.framework', + 'flutter_assets', + 'shorebird.yaml', + ), + ).readAsStringSync(); + return loadYaml(yamlString) as YamlMap; + } +} diff --git a/shorebird/ci/artifacts_manifest.template.yaml b/shorebird/ci/artifacts_manifest.template.yaml new file mode 100644 index 0000000000000..33223fced2de8 --- /dev/null +++ b/shorebird/ci/artifacts_manifest.template.yaml @@ -0,0 +1,65 @@ +# Template for artifacts_manifest.yaml +# This file is processed by shard_runner:finalize +# Variable substitution: {{flutter_engine_revision}} is replaced at generation time +# The $engine placeholder is kept as-is for Shorebird's artifact proxy + +flutter_engine_revision: {{flutter_engine_revision}} +storage_bucket: download.shorebird.dev +artifact_overrides: + # Android release artifacts + - flutter_infra_release/flutter/$engine/android-arm64-release/artifacts.zip + - flutter_infra_release/flutter/$engine/android-arm64-release/darwin-x64.zip + - flutter_infra_release/flutter/$engine/android-arm64-release/linux-x64.zip + - flutter_infra_release/flutter/$engine/android-arm64-release/symbols.zip + - flutter_infra_release/flutter/$engine/android-arm64-release/windows-x64.zip + + - flutter_infra_release/flutter/$engine/android-arm-release/artifacts.zip + - flutter_infra_release/flutter/$engine/android-arm-release/darwin-x64.zip + - flutter_infra_release/flutter/$engine/android-arm-release/linux-x64.zip + - flutter_infra_release/flutter/$engine/android-arm-release/symbols.zip + - flutter_infra_release/flutter/$engine/android-arm-release/windows-x64.zip + + - flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip + - flutter_infra_release/flutter/$engine/android-x64-release/darwin-x64.zip + - flutter_infra_release/flutter/$engine/android-x64-release/linux-x64.zip + - flutter_infra_release/flutter/$engine/android-x64-release/symbols.zip + - flutter_infra_release/flutter/$engine/android-x64-release/windows-x64.zip + + # engine_stamp.json + - flutter_infra_release/flutter/$engine/engine_stamp.json + + # Dart SDK + - flutter_infra_release/flutter/$engine/dart-sdk-darwin-arm64.zip + - flutter_infra_release/flutter/$engine/dart-sdk-darwin-x64.zip + - flutter_infra_release/flutter/$engine/dart-sdk-linux-x64.zip + - flutter_infra_release/flutter/$engine/dart-sdk-windows-x64.zip + + # Maven artifacts + - download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom + - download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar + - download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom + - download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar + - download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom + - download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.jar + - download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom + - download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.jar + + # Common release artifacts + - flutter_infra_release/flutter/$engine/flutter_patched_sdk_product.zip + + # iOS release artifacts + - flutter_infra_release/flutter/$engine/ios-release/artifacts.zip + - flutter_infra_release/flutter/$engine/ios-release/Flutter.framework.dSYM.zip + + # Linux release artifacts + - flutter_infra_release/flutter/$engine/linux-x64/artifacts.zip + - flutter_infra_release/flutter/$engine/linux-x64-release/linux-x64-flutter-gtk.zip + + # macOS release artifacts + - flutter_infra_release/flutter/$engine/darwin-x64-release/artifacts.zip + - flutter_infra_release/flutter/$engine/darwin-x64-release/framework.zip + - flutter_infra_release/flutter/$engine/darwin-x64-release/gen_snapshot.zip + + # Windows release artifacts + - flutter_infra_release/flutter/$engine/windows-x64/artifacts.zip + - flutter_infra_release/flutter/$engine/windows-x64-release/windows-x64-flutter.zip diff --git a/shorebird/ci/compose.json b/shorebird/ci/compose.json new file mode 100644 index 0000000000000..1f6ca744d1e5d --- /dev/null +++ b/shorebird/ci/compose.json @@ -0,0 +1,30 @@ +{ + "ios-framework": { + "requires": ["ios-release", "ios-release-ext", "ios-sim-x64", "ios-sim-x64-ext", "ios-sim-arm64", "ios-sim-arm64-ext"], + "script": "flutter/sky/tools/create_ios_framework.py", + "flags": ["--dsym", "--strip"], + "path_args": { + "--arm64-out-dir": "ios_release", + "--simulator-x64-out-dir": "ios_debug_sim", + "--simulator-arm64-out-dir": "ios_debug_sim_arm64" + } + }, + "macos-framework": { + "requires": ["mac-arm64", "mac-x64"], + "script": "flutter/sky/tools/create_macos_framework.py", + "flags": ["--dsym", "--strip", "--zip"], + "path_args": { + "--arm64-out-dir": "mac_release_arm64", + "--x64-out-dir": "mac_release" + } + }, + "macos-gen-snapshot": { + "requires": ["mac-arm64", "mac-x64"], + "script": "flutter/sky/tools/create_macos_gen_snapshots.py", + "flags": ["--zip"], + "path_args": { + "--arm64-path": "mac_release_arm64/universal/gen_snapshot_arm64", + "--x64-path": "mac_release/universal/gen_snapshot_x64" + } + } +} diff --git a/shorebird/ci/internal/generate_manifest.sh b/shorebird/ci/internal/generate_manifest.sh new file mode 100755 index 0000000000000..ee9ef7ddfdf67 --- /dev/null +++ b/shorebird/ci/internal/generate_manifest.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +# This script outputs an artifact_manifest.yaml mapping +# a shorebird engine revision to a flutter engine revision. +# Usage: +# ./generate_manifest.sh > artifact_manifest.yaml + +set -e + +# NOTE: If you edit this file you also may need to edit the global list +# of all known artifacts in the artifact_proxy's config.dart + +if [ "$#" -ne 1 ]; then + echo "Usage: ./generate_manifest.sh " + exit 1 +fi + +FLUTTER_ENGINE_REVISION=$1 + +cat <, but that no longer seems needed. +# We always use the hermetic NDK from the engine repo. +# The "unmodified" CIPD package keeps the NDK at the standard Android SDK path. +ANDROID_NDK_HOME=$(echo "$ENGINE_SRC/flutter/third_party/android_tools/sdk/ndk"/*) \ +cargo ndk \ + --target armv7-linux-androideabi \ + --target aarch64-linux-android \ + --target i686-linux-android \ + --target x86_64-linux-android \ + build --release + +cargo build --release --target x86_64-unknown-linux-gnu + +# Build the patch tool. +cd $UPDATER_SRC/patch +cargo build --release + +# Compile the engine using the steps here: +# https://github.com/flutter/flutter/wiki/Compiling-the-engine#compiling-for-android-from-macos-or-linux +cd $ENGINE_SRC + +NINJA="ninja" +GN=./flutter/tools/gn +# We could probably use our own prebuilt dart SDK, by modifying the gn files. +GN_ARGS="--no-rbe --no-enable-unittests" + +# We could use Linux to generate all of our Android binaries, but we don't yet. +# https://github.com/flutter/engine/blob/e590b24f3962fda3ec9144dcee3f7565b195839a/ci/builders/linux_android_aot_engine.json#L40 + +# Build the default and gen_snapshot targets. +# +# Linux doesn't seem to use "archive_gen_snapshot" as a target name yet. +# https://github.com/flutter/flutter/issues/105351#issuecomment-1650686247 +ANDROID_TARGETS="default gen_snapshot" + +# Android arm64 release +$GN $GN_ARGS --android --android-cpu=arm64 --runtime-mode=release +$NINJA -C ./out/android_release_arm64 $ANDROID_TARGETS + +# Android arm32 release +$GN $GN_ARGS --runtime-mode=release --android +$NINJA -C out/android_release $ANDROID_TARGETS + +# Android x64 release +$GN $GN_ARGS --android --android-cpu=x64 --runtime-mode=release +$NINJA -C ./out/android_release_x64 $ANDROID_TARGETS + +# Build Dart and Flutter +$GN $GN_ARGS --runtime-mode=release --no-prebuilt-dart-sdk +# build Dart and the linux shell and flutter_patched_sdk_product.zip +$NINJA -C out/host_release dart_sdk flutter/shell/platform/linux:flutter_gtk flutter/build/archives:flutter_patched_sdk +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 + +# Build debug Linux artifacts +# These are output to the `linux-x64` directory in host_debug, and are used +# by `flutter build linux --release`. +$GN $GN_ARGS --no-prebuilt-dart-sdk +$NINJA -C ./out/host_debug flutter/build/archives:artifacts + +# Shorebird AOT Tools (Linker) +mkdir -p $ENGINE_OUT/host_release/aot_tools + +# Dart kernel (.dill) files are not stable and can change with the version of Dart, so we +# can't use this machine's `dart`. Here we're using the version of Dart that this +# version of the engine depends on, which should be the same version that +# `flutter` ends up depending on. +DART=$ENGINE_OUT/host_release/dart-sdk/bin/dart +AOT_TOOLS_PKG=$ENGINE_SRC/flutter/third_party/dart/pkg/aot_tools +# This should be part of `gclient sync` https://github.com/shorebirdtech/_build_engine/issues/113 +(cd $AOT_TOOLS_PKG; $DART pub get) +# This should be built as part of Dart and then pulled down as part of the engine build. +# https://github.com/shorebirdtech/_build_engine/issues/88 +$DART compile kernel $AOT_TOOLS_PKG/bin/aot_tools.dart -o $ENGINE_OUT/host_release/aot_tools/aot-tools.dill + +mkdir -p $ENGINE_OUT/host_release/updater_tools +UPDATER_TOOLS_PKG=$ENGINE_SRC/flutter/third_party/updater/updater_tools +# This should be part of `gclient sync` https://github.com/shorebirdtech/_build_engine/issues/113 + +# We could also build the `patch` tool for Linux here. diff --git a/shorebird/ci/internal/linux_setup.sh b/shorebird/ci/internal/linux_setup.sh new file mode 100755 index 0000000000000..626ad211536eb --- /dev/null +++ b/shorebird/ci/internal/linux_setup.sh @@ -0,0 +1,30 @@ +#!/bin/bash -e + +# Usage: +# ./linux_setup.sh + +# Per https://github.com/flutter/flutter/wiki/Setting-up-the-Engine-development-environment +# Subset of ./flutter/build/install-build-deps-linux-desktop.sh +sudo apt -y install libfreetype6-dev pkg-config + +# This assumes rust is installed, but could also install rust/cargo. + +# Need NDK from https://developer.android.com/ndk/downloads +# The NDK version should match that of DEPS, e.g. +# https://github.com/flutter/flutter/blame/b45fa18946ecc2d9b4009952c636ba7e2ffbb787/DEPS#L615 +# Example: +# curl -O https://dl.google.com/android/repository/android-ndk-r27d-linux.zip +# unzip android-ndk-r27d-linux.zip +# On the GHA runners we set this in .github/workflows/build_engine.yaml +# env: +# NDK_HOME: /home/gha/bin/android-ndk-r27d + +# We require an old version of cargo-ndk to support the old NDK Flutter +# engine currently uses. +cargo install cargo-ndk +rustup target add \ + aarch64-linux-android \ + armv7-linux-androideabi \ + x86_64-linux-android \ + i686-linux-android \ + x86_64-unknown-linux-gnu diff --git a/shorebird/ci/internal/linux_test_build.sh b/shorebird/ci/internal/linux_test_build.sh new file mode 100755 index 0000000000000..5d5fb9b0b2538 --- /dev/null +++ b/shorebird/ci/internal/linux_test_build.sh @@ -0,0 +1,29 @@ +#!/bin/bash -e + +# The path to the Flutter engine. +# Convert to an absolute path so we don't need to worry about cd'ing back to +# the root directory between commands. +ENGINE_ROOT=$(realpath $1) +ENGINE_SRC=$ENGINE_ROOT/src + +cd $ENGINE_SRC + +UPDATER_SRC=$ENGINE_SRC/flutter/third_party/updater +(cd $UPDATER_SRC && + ANDROID_NDK_HOME=$(echo "$ENGINE_SRC/flutter/third_party/android_tools/sdk/ndk"/*) \ + cargo ndk \ + --target armv7-linux-androideabi \ + --target aarch64-linux-android \ + --target i686-linux-android \ + --target x86_64-linux-android \ + build --release && + cargo build --release --target x86_64-unknown-linux-gnu +) + +# Generate an unoptimized debug build of the engine (expected by the test script). +./flutter/tools/gn --unoptimized --no-rbe +ninja -C out/host_debug_unopt + +# Generate an unoptimized android debug build for java engine tests +./flutter/tools/gn --android --unoptimized --no-rbe +ninja -C out/android_debug_unopt diff --git a/shorebird/ci/internal/linux_upload.sh b/shorebird/ci/internal/linux_upload.sh new file mode 100755 index 0000000000000..873f0f3edb969 --- /dev/null +++ b/shorebird/ci/internal/linux_upload.sh @@ -0,0 +1,138 @@ +#!/bin/bash -e + +# Usage: +# ./linux_upload.sh engine_path git_hash + +# Convert to an absolute path so we don't need to worry about cd'ing back to +# the root directory between commands. +ENGINE_ROOT=$(realpath $1) +ENGINE_HASH=$2 + +STORAGE_BUCKET="download.shorebird.dev" +SHOREBIRD_ROOT=gs://$STORAGE_BUCKET/shorebird/$ENGINE_HASH + +ENGINE_SRC=$ENGINE_ROOT/src +ENGINE_OUT=$ENGINE_SRC/out +ENGINE_FLUTTER=$ENGINE_SRC/flutter +# FLUTTER_ROOT is the Flutter monorepo root (parent of engine/) +FLUTTER_ROOT=$(dirname $ENGINE_ROOT) + +cd $FLUTTER_ROOT + +# Compute the content-aware hash for the Dart SDK. +# This allows Flutter checkouts that haven't changed engine content to share +# the same pre-built Dart SDK, even if they have different git commit SHAs. +CONTENT_HASH=$($FLUTTER_ROOT/bin/internal/content_aware_hash.sh) + +# We do not generate a manifest file, we assume another builder did that. +# TODO(bryanoltman): should we generate a manifest file as part of an upload +# script, or should it be done once all build and uploads have completed? +# See https://github.com/shorebirdtech/build_engine/issues/25 + +# TODO(eseidel): This should not be in shell, it's too complicated/repetitive. + +HOST_ARCH='linux-x64' + +INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$ENGINE_HASH" +MAVEN_VER="1.0.0-$ENGINE_HASH" +MAVEN_ROOT="gs://$STORAGE_BUCKET/download.flutter.io/io/flutter" + +# Dart SDK +# This gets uploaded to flutter_infra_release/flutter/\$engine/dart-sdk-$HOST_ARCH.zip +# We also upload to the content-aware hash path to support local development branches. +CONTENT_INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$CONTENT_HASH" + +HOST_RELEASE=$ENGINE_OUT/host_release +DART_ZIP_FILE=dart-sdk-$HOST_ARCH.zip +( + cd $HOST_RELEASE; + zip -r $DART_ZIP_FILE dart-sdk +) +ZIPS_DEST=$INFRA_ROOT/$DART_ZIP_FILE +gsutil cp $HOST_RELEASE/$DART_ZIP_FILE $ZIPS_DEST +# Also upload to content-aware hash path +gsutil cp $HOST_RELEASE/$DART_ZIP_FILE $CONTENT_INFRA_ROOT/$DART_ZIP_FILE + +# Android Arm64 release Flutter artifacts +ARCH_OUT=$ENGINE_OUT/android_release_arm64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm64-release +ZIPS_DEST=$INFRA_ROOT/android-arm64-release +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip +gsutil cp $ZIPS_OUT/symbols.zip $ZIPS_DEST/symbols.zip +# Android Arm64 release Maven artifacts +ARCH_PATH=$ARCH_OUT/arm64_v8a_release +MAVEN_PATH=$MAVEN_ROOT/arm64_v8a_release/$MAVEN_VER/arm64_v8a_release-$MAVEN_VER +gsutil cp $ARCH_PATH.pom $MAVEN_PATH.pom +gsutil cp $ARCH_PATH.jar $MAVEN_PATH.jar +gsutil cp $ARCH_PATH.maven-metadata.xml $MAVEN_PATH.maven-metadata.xml + +# Android Arm32 release Flutter artifacts +ARCH_OUT=$ENGINE_OUT/android_release +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm-release +ZIPS_DEST=$INFRA_ROOT/android-arm-release +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip +gsutil cp $ZIPS_OUT/symbols.zip $ZIPS_DEST/symbols.zip +# Android Arm32 release Maven artifacts +ARCH_PATH=$ARCH_OUT/armeabi_v7a_release +MAVEN_PATH=$MAVEN_ROOT/armeabi_v7a_release/$MAVEN_VER/armeabi_v7a_release-$MAVEN_VER +gsutil cp $ARCH_PATH.pom $MAVEN_PATH.pom +gsutil cp $ARCH_PATH.jar $MAVEN_PATH.jar +gsutil cp $ARCH_PATH.maven-metadata.xml $MAVEN_PATH.maven-metadata.xml + +# Not sure which flutter_embedding_release files to use? 32 or 64 bit? +# It does not seem to contain the libflutter.so file, but does seem to +# differ between the two build dirs. +ARCH_OUT=$ENGINE_OUT/android_release +ARCH_PATH=$ARCH_OUT/flutter_embedding_release +MAVEN_PATH=$MAVEN_ROOT/flutter_embedding_release/$MAVEN_VER/flutter_embedding_release-$MAVEN_VER +gsutil cp $ARCH_PATH.pom $MAVEN_PATH.pom +gsutil cp $ARCH_PATH.jar $MAVEN_PATH.jar +gsutil cp $ARCH_PATH.maven-metadata.xml $MAVEN_PATH.maven-metadata.xml + +# Android x64 release Flutter artifacts +ARCH_OUT=$ENGINE_OUT/android_release_x64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-x64-release +ZIPS_DEST=$INFRA_ROOT/android-x64-release +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip +gsutil cp $ZIPS_OUT/symbols.zip $ZIPS_DEST/symbols.zip +# Android x64 release Maven artifacts +ARCH_PATH=$ARCH_OUT/x86_64_release +MAVEN_PATH=$MAVEN_ROOT/x86_64_release/$MAVEN_VER/x86_64_release-$MAVEN_VER +gsutil cp $ARCH_PATH.pom $MAVEN_PATH.pom +gsutil cp $ARCH_PATH.jar $MAVEN_PATH.jar +gsutil cp $ARCH_PATH.maven-metadata.xml $MAVEN_PATH.maven-metadata.xml + +# Shorebird AOT Tools (Linker) +gsutil cp $ENGINE_OUT/host_release/aot_tools/aot-tools.dill $SHOREBIRD_ROOT/aot-tools.dill + +# Common Product-mode artifacts +ARCH_OUT=$ENGINE_OUT/host_release +ZIPS_OUT=$ARCH_OUT/zip_archives +ZIPS_DEST=$INFRA_ROOT +gsutil cp $ZIPS_OUT/flutter_patched_sdk_product.zip $ZIPS_DEST/flutter_patched_sdk_product.zip + +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 +# Linux x64 host_release font_subset (ConstFinder) +# ARCH_OUT=$ENGINE_OUT/host_release +# ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +# ZIPS_DEST=$INFRA_ROOT/linux-x64-release +# gsutil cp $ZIPS_OUT/font-subset.zip $ZIPS_DEST/font-subset.zip + +# Linux Desktop Support +ARCH_OUT=$ENGINE_OUT/host_release +ZIPS_OUT=$ARCH_OUT/zip_archives/linux-x64-release +ZIPS_DEST=$INFRA_ROOT/linux-x64-release +gsutil cp $ZIPS_OUT/linux-x64-flutter-gtk.zip $ZIPS_DEST/linux-x64-flutter-gtk.zip + +ARCH_OUT=$ENGINE_OUT/host_debug +ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +ZIPS_DEST=$INFRA_ROOT/$HOST_ARCH +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip + +# We could upload patch if we built it here. +# gsutil cp $ENGINE_OUT/host_release/patch.zip $SHOREBIRD_ROOT/patch-win-x64.zip diff --git a/shorebird/ci/internal/mac_build.sh b/shorebird/ci/internal/mac_build.sh new file mode 100755 index 0000000000000..a860da431f269 --- /dev/null +++ b/shorebird/ci/internal/mac_build.sh @@ -0,0 +1,204 @@ +#!/bin/bash -e + +# FIXME: This script should be deleted and instead these steps be part +# of the GN build process. +# I haven't investigated how to build rust from GN with the Android NDK yet. + +# Usage: +# ./mac_build.sh engine_path + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 engine_path" + exit 1 +fi + +# Convert to an absolute path so we don't need to worry about cd'ing back to +# the root directory between commands. +ENGINE_ROOT=$(realpath $1) + +ENGINE_SRC=$ENGINE_ROOT/src +ENGINE_OUT=$ENGINE_SRC/out +UPDATER_SRC=$ENGINE_SRC/flutter/third_party/updater +HOST_ARCH='darwin-x64' + +# Build the Rust library. +cd $UPDATER_SRC/library + +# Build iOS and MacOS +cargo build \ + --target aarch64-apple-ios \ + --target x86_64-apple-ios \ + --target aarch64-apple-darwin \ + --target x86_64-apple-darwin \ + --release + +# Build the patch tool. +# Again, this belongs as part of the gn build. +cd $UPDATER_SRC/patch +cargo build --release + +# Compile the engine using the steps here: +# https://github.com/flutter/flutter/wiki/Compiling-the-engine#compiling-for-android-from-macos-or-linux +cd $ENGINE_SRC + +NINJA="ninja" +GN=./flutter/tools/gn +ET=./flutter/bin/et +# We could probably use our own prebuilt dart SDK, by modifying the gn files. +# `--no-enable-unittests` is needed on Flutter 3.10.1 and 3.10.2 to avoid +# https://github.com/flutter/flutter/issues/128135 +GN_ARGS="--no-rbe --no-enable-unittests" + +# FIXME: These build commands likely could build fewer targets. + +# Mac doesn't seem to use "archive_gen_snapshot" as a target name yet. +# https://github.com/flutter/flutter/issues/105351#issuecomment-1650686247 +ANDROID_TARGETS="flutter/shell/platform/android:gen_snapshot" + +# Because Flutter does not yet build universal binaries for macOS, we need to +# ensure we're building for x64 for the time being so we can support both Intel +# and Apple Silicon Macs. We do this by telling gn to use host_cpu="x64". + +# Android arm64 release +$GN $GN_ARGS --android --android-cpu=arm64 --runtime-mode=release --gn-args='host_cpu="x64"' +$NINJA -C ./out/android_release_arm64 $ANDROID_TARGETS + +# Android arm32 release +$GN $GN_ARGS --runtime-mode=release --android --gn-args='host_cpu="x64"' +$NINJA -C out/android_release $ANDROID_TARGETS + +# Android x64 release +$GN $GN_ARGS --android --android-cpu=x64 --runtime-mode=release --gn-args='host_cpu="x64"' +$NINJA -C ./out/android_release_x64 $ANDROID_TARGETS + +# We only need two targets (per the mac builders): +# "flutter/shell/platform/darwin/ios:flutter_framework", +# "flutter/lib/snapshot:generate_snapshot_bins", which builds both gen_snapshot and analyze_snapshot binaries. +# https://github.com/flutter/engine/blob/main/ci/builders/mac_ios_engine.json#L139 +# https://github.com/flutter/engine/blob/main/ci/builders/README.md +# The files created by these targets are packaged into a framework and an artifacts.zip file +# by the create_full_ios_framework.py and create_macos_framework.py scripts. + +IOS_TARGETS="flutter/shell/platform/darwin/ios:flutter_framework flutter/lib/snapshot:generate_snapshot_bins" +# You will also need to build vm_platform_strong.dill if you're using a local engine build. + +# From ci/builders/mac_host_engine.json in the engine repo +MACOS_TARGETS="flutter/shell/platform/darwin/macos:zip_macos_flutter_framework flutter/lib/snapshot:generate_snapshot_bins flutter/build/archives:artifacts" + +# Build x64 Dart SDK +$GN $GN_ARGS --runtime-mode=release --mac-cpu=x64 --no-prebuilt-dart-sdk +$NINJA -C out/host_release dart_sdk +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 + +# Build arm64 Dart SDK +$GN $GN_ARGS --runtime-mode=release --mac-cpu=arm64 --no-prebuilt-dart-sdk +$NINJA -C out/host_release_arm64 dart_sdk +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 + +# iOS arm64 release +$GN $GN_ARGS --runtime-mode=release --ios --gn-arg='shorebird_runtime=true' +$NINJA -C out/ios_release $IOS_TARGETS + +$GN $GN_ARGS --ios --runtime-mode=release --darwin-extension-safe --xcode-symlinks --gn-arg='shorebird_runtime=true' +$NINJA -C out/ios_release_extension_safe $IOS_TARGETS + +# iOS simulator-x64 release +$GN $GN_ARGS --runtime-mode=debug --ios --simulator +$NINJA -C out/ios_debug_sim $IOS_TARGETS + +$GN $GN_ARGS --runtime-mode=debug --darwin-extension-safe --ios --simulator +$NINJA -C out/ios_debug_sim_extension_safe $IOS_TARGETS + +# iOS simulator-arm64 release +$GN $GN_ARGS --runtime-mode=debug --ios --simulator --simulator-cpu=arm64 +$NINJA -C out/ios_debug_sim_arm64 $IOS_TARGETS + +$GN $GN_ARGS --runtime-mode=debug --darwin-extension-safe --ios --simulator --simulator-cpu=arm64 +$NINJA -C out/ios_debug_sim_arm64_extension_safe $IOS_TARGETS + +# macOS arm64 release +$GN $GN_ARGS --runtime-mode=release --mac --mac-cpu=arm64 +$NINJA -C out/mac_release_arm64 $MACOS_TARGETS + +# macOS x64 release +# Note: we don't enable the simulator here because the simulator is an arm64 simulator, +# which won't work for x64 apps. +$GN $GN_ARGS --runtime-mode=release --mac --mac-cpu=x64 +$NINJA -C out/mac_release $MACOS_TARGETS + +# The python scripts below fail if the out/release directory already exists. +rm -rf out/release + +# We have to create a composite Flutter.framework for iOS and macOS, matching +# what the Flutter builders do: +IOS_FRAMEWORK_OUT=out/release +echo "Building Flutter.framework for iOS" +python3 flutter/sky/tools/create_ios_framework.py \ + --dst $IOS_FRAMEWORK_OUT \ + --arm64-out-dir out/ios_release \ + --simulator-x64-out-dir out/ios_debug_sim \ + --simulator-arm64-out-dir out/ios_debug_sim_arm64 \ + --dsym \ + --strip +echo "Built Flutter.framework for iOS" + +MAC_FRAMEWORK_OUT=out/release/framework +echo "Building Flutter.framework for macOS" +python3 flutter/sky/tools/create_macos_framework.py \ + --dst $MAC_FRAMEWORK_OUT \ + --arm64-out-dir out/mac_release_arm64 \ + --x64-out-dir out/mac_release \ + --dsym \ + --strip \ + --zip +echo "Built Flutter.framework for macOS" + +echo "Creating macOS gen_snapshot" +python3 flutter/sky/tools/create_macos_gen_snapshots.py \ + --dst out/release/snapshot \ + --arm64-path out/mac_release_arm64/universal/gen_snapshot_arm64 \ + --x64-path out/mac_release/universal/gen_snapshot_x64 \ + --zip +echo "Created macOS gen_snapshot" + +# Zip the dSYMs +zip -r $IOS_FRAMEWORK_OUT/Flutter.framework.dSYM.zip $IOS_FRAMEWORK_OUT/Flutter.framework.dSYM +zip -r $MAC_FRAMEWORK_OUT/FlutterMacOS.framework.dSYM.zip $MAC_FRAMEWORK_OUT/FlutterMacOS.framework.dSYM + +sign_flutter_xcframework() { + pushd $ENGINE_OUT/release + + # Unzip the artifacts zip file, which contains the Flutter.xcframework. + rm -rf artifacts + unzip artifacts.zip -d artifacts + + # Keep a copy of the old artifacts.zip for now, we may decide to remove this later + mv artifacts.zip artifacts.old.zip + + # Sign the Flutter.xcframework + cd artifacts + + # In case the artifacts are already signed, remove the signature + codesign -v --remove-signature Flutter.xcframework + codesign -v --sign "Apple Distribution: Code Town Inc (6V53YACS2W)" Flutter.xcframework + + # Zip the artifacts back up + zip -r "../artifacts.zip" * + + # Cleanup + cd .. + rm -rf artifacts + + popd +} + +sign_flutter_xcframework + +# Create out/engine_stamp.json +# We can remove this explicit step once we're using et in any of the lines +# above. +$ET stamp diff --git a/shorebird/ci/internal/mac_setup.sh b/shorebird/ci/internal/mac_setup.sh new file mode 100755 index 0000000000000..8453f8b9b1a9e --- /dev/null +++ b/shorebird/ci/internal/mac_setup.sh @@ -0,0 +1,11 @@ +#!/bin/bash -e + +# Usage: +# ./mac_setup.sh + +# This assumes rust is installed, but could also install rust/cargo. +rustup target add \ + x86_64-apple-ios \ + aarch64-apple-ios \ + aarch64-apple-darwin \ + x86_64-apple-darwin diff --git a/shorebird/ci/internal/mac_upload.sh b/shorebird/ci/internal/mac_upload.sh new file mode 100755 index 0000000000000..8ec266246a8e1 --- /dev/null +++ b/shorebird/ci/internal/mac_upload.sh @@ -0,0 +1,180 @@ +#!/bin/bash -e + +# Usage: +# ./mac_upload.sh engine_path git_hash + +# Convert to an absolute path so we don't need to worry about cd'ing back to +# the root directory between commands. +ENGINE_ROOT=$(realpath $1) +ENGINE_HASH=$2 + +# Get the absolute path to the directory of this script. +SCRIPT_DIR=$(cd $(dirname $0) && pwd) + +STORAGE_BUCKET="download.shorebird.dev" +SHOREBIRD_ROOT=gs://$STORAGE_BUCKET/shorebird/$ENGINE_HASH + +ENGINE_SRC=$ENGINE_ROOT/src +ENGINE_OUT=$ENGINE_SRC/out +ENGINE_FLUTTER=$ENGINE_SRC/flutter +# FLUTTER_ROOT is the Flutter monorepo root (parent of engine/) +FLUTTER_ROOT=$(dirname $ENGINE_ROOT) + +cd $FLUTTER_ROOT + +# Compute the content-aware hash for the Dart SDK. +# This allows Flutter checkouts that haven't changed engine content to share +# the same pre-built Dart SDK, even if they have different git commit SHAs. +CONTENT_HASH=$($FLUTTER_ROOT/bin/internal/content_aware_hash.sh) +# Can't just `git merge-base` because the engine branches for each +# major version (e.g. 3.7, 3.8) (e.g. upstream/flutter-3.7-candidate.1) +# but it's not clear which branch we're forked from, only that we took +# some tag and added our commits (but we don't know what tag). +BASE_FLUTTER_TAG=`git describe --tags --abbrev=0` +# Read the first line from bin/internal/engine.version file and trim whitespace. +BASE_ENGINE_HASH=`git show $BASE_FLUTTER_TAG:bin/internal/engine.version | head -n 1 | tr -d '[:space:]'` + +# Build the artifacts manifest: +MANIFEST_FILE=`mktemp` +# Note that any uploads which are *not* listed in the manifest will be +# ignored by the artifact proxy. +# if you add uploads here, they also need to be reflected in the manifest. +$SCRIPT_DIR/generate_manifest.sh $BASE_ENGINE_HASH > $MANIFEST_FILE + +# FIXME: This should not be in shell, it's too complicated/repetitive. +# Only need the libflutter.so (and flutter.jar) artifacts +# Artifact list: https://github.com/shorebirdtech/shorebird/blob/main/packages/artifact_proxy/lib/config.dart + +HOST_ARCH='darwin-x64' +ARM64_HOST_ARCH='darwin-arm64' + +INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$ENGINE_HASH" + +# engine_stamp.json +ENGINE_STAMP_FILE=$ENGINE_OUT/engine_stamp.json +gsutil cp $ENGINE_STAMP_FILE $INFRA_ROOT/engine_stamp.json + +# Dart SDK +# This gets uploaded to flutter_infra_release/flutter/\$engine/dart-sdk-$HOST_ARCH.zip +# We also upload to the content-aware hash path to support local development branches. +CONTENT_INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$CONTENT_HASH" + +# x64 Dart SDK +HOST_RELEASE=$ENGINE_OUT/host_release +DART_ZIP_FILE=dart-sdk-$HOST_ARCH.zip +( + cd $HOST_RELEASE; + zip -r $DART_ZIP_FILE dart-sdk +) +ZIPS_DEST=$INFRA_ROOT/$DART_ZIP_FILE +gsutil cp $HOST_RELEASE/$DART_ZIP_FILE $ZIPS_DEST +# Also upload to content-aware hash path +gsutil cp $HOST_RELEASE/$DART_ZIP_FILE $CONTENT_INFRA_ROOT/$DART_ZIP_FILE + +# arm64 Dart SDK +HOST_RELEASE_ARM64=$ENGINE_OUT/host_release_arm64 +DART_ZIP_FILE=dart-sdk-$ARM64_HOST_ARCH.zip +( + cd $HOST_RELEASE_ARM64; + zip -r $DART_ZIP_FILE dart-sdk +) +ZIPS_DEST=$INFRA_ROOT/$DART_ZIP_FILE +gsutil cp $HOST_RELEASE_ARM64/$DART_ZIP_FILE $ZIPS_DEST +# Also upload to content-aware hash path +gsutil cp $HOST_RELEASE_ARM64/$DART_ZIP_FILE $CONTENT_INFRA_ROOT/$DART_ZIP_FILE + +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 +# # mac x64 host_release font_subset (ConstFinder) +# ARCH_OUT=$ENGINE_OUT/host_release +# ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +# ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +# gsutil cp $ZIPS_OUT/font-subset.zip $ZIPS_DEST/font-subset.zip + +# # mac arm64 host_release font_subset (ConstFinder) +# ARCH_OUT=$ENGINE_OUT/host_release_arm64 +# ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +# ZIPS_DEST=$INFRA_ROOT/darwin-arm64-release +# gsutil cp $ZIPS_OUT/font-subset.zip $ZIPS_DEST/font-subset.zip + +# Android Arm64 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release_arm64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm64-release +ZIPS_DEST=$INFRA_ROOT/android-arm64-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Android Arm32 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm-release +ZIPS_DEST=$INFRA_ROOT/android-arm-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Android x64 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release_x64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-x64-release +ZIPS_DEST=$INFRA_ROOT/android-x64-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Match the upload pattern from iOS: +# https://github.com/flutter/engine/commit/1d7f0c66c316a37105601b13136f890f6595aebc + +# iOS release Flutter artifacts +ARCH_OUT=$ENGINE_OUT/release +ZIPS_OUT=$ARCH_OUT +ZIPS_DEST=$INFRA_ROOT/ios-release +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip + +# iOS dSYM +gsutil cp $ZIPS_OUT/Flutter.framework.dSYM.zip $ZIPS_DEST/Flutter.framework.dSYM.zip + +# macOS framework +ARCH_OUT=$ENGINE_OUT/release +ZIPS_OUT=$ARCH_OUT/framework +ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +gsutil cp $ZIPS_OUT/framework.zip $ZIPS_DEST/framework.zip + +# macOS gen_snapshot +ARCH_OUT=$ENGINE_OUT/release +ZIPS_OUT=$ARCH_OUT/snapshot +ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +gsutil cp $ZIPS_OUT/gen_snapshot.zip $ZIPS_DEST/gen_snapshot.zip + +# FIXME: these should go where we're putting the arm64 macOS artifacts +# (darwin-x64-release), however, arm macs use darwin-x64-release and we +# currently only support those. We need to find a way to support both. +# macOS x64 release artifacts +# ARCH_OUT=$ENGINE_OUT/mac_release +# ZIPS_OUT=$ARCH_OUT/zip_archives/darwin-x64-release +# ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +# gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip + +# macOS arm64 release artifacts +ARCH_OUT=$ENGINE_OUT/mac_release_arm64 +ZIPS_OUT=$ARCH_OUT/zip_archives/darwin-arm64-release +# This looks wrong - why are we putting arm64 artifacts in darwin-x64-release +# instead of darwin-arm64-release? This is because arm macs use darwin-x64-release +# and we need to use the artifacts we've built for arm64 macs. +ZIPS_DEST=$INFRA_ROOT/darwin-x64-release +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip + +# macOS dSYM (used for symbolication, not by Flutter) +ARCH_OUT=$ENGINE_OUT/release +ZIPS_OUT=$ARCH_OUT/framework +ZIPS_DEST=$INFRA_ROOT/darwin-x64 +gsutil cp $ZIPS_OUT/FlutterMacOS.framework.dSYM.zip $ZIPS_DEST/FlutterMacOS.framework.dSYM.zip + +TMP_DIR=$(mktemp -d) + +PATCH_VERSION=0.2.1 +GH_RELEASE=https://github.com/shorebirdtech/updater/releases/download/patch-v$PATCH_VERSION/ +cd $TMP_DIR +curl -L $GH_RELEASE/patch-x86_64-apple-darwin.zip -o patch-x86_64-apple-darwin.zip +curl -L $GH_RELEASE/patch-x86_64-pc-windows-msvc.zip -o patch-x86_64-pc-windows-msvc.zip +curl -L $GH_RELEASE/patch-x86_64-unknown-linux-musl.zip -o patch-x86_64-unknown-linux-musl.zip + +gsutil cp patch-x86_64-apple-darwin.zip $SHOREBIRD_ROOT/patch-darwin-x64.zip +gsutil cp patch-x86_64-pc-windows-msvc.zip $SHOREBIRD_ROOT/patch-windows-x64.zip +gsutil cp patch-x86_64-unknown-linux-musl.zip $SHOREBIRD_ROOT/patch-linux-x64.zip + +gsutil cp $MANIFEST_FILE $SHOREBIRD_ROOT/artifacts_manifest.yaml diff --git a/shorebird/ci/internal/win_build.sh b/shorebird/ci/internal/win_build.sh new file mode 100755 index 0000000000000..2df02670aa78f --- /dev/null +++ b/shorebird/ci/internal/win_build.sh @@ -0,0 +1,63 @@ +#!/bin/bash -e + +# Usage: +# ./win_build.sh engine_path + +ENGINE_ROOT=$1 + +ENGINE_SRC=$ENGINE_ROOT/src +ENGINE_OUT=$ENGINE_SRC/out +UPDATER_SRC=$ENGINE_SRC/flutter/third_party/updater +HOST_ARCH='windows-x64' + +# Build the Rust library. +cd $UPDATER_SRC/library + +cargo build --release \ + --target x86_64-pc-windows-msvc + +# Compile the engine using the steps here: +# https://github.com/flutter/flutter/wiki/Compiling-the-engine#compiling-for-android-from-macos-or-linux +cd $ENGINE_SRC + +NINJA="ninja" +GN=./flutter/tools/gn +# We could probably use our own prebuilt dart SDK, by modifying the gn files. +GN_ARGS="--no-rbe --no-enable-unittests" + +# Windows only needs gen_snapshot for each Android CPU type. +# See https://github.com/flutter/engine/blob/e590b24f3962fda3ec9144dcee3f7565b195839a/ci/builders/windows_android_aot_engine.json + +TARGETS="archive_win_gen_snapshot" + +# Build host_release +$GN $GN_ARGS --runtime-mode=release --no-prebuilt-dart-sdk +$NINJA -C out/host_release dart_sdk +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 + +# Build windows desktop targets +$GN $GN_ARGS --runtime-mode=release --no-prebuilt-dart-sdk +$NINJA -C ./out/host_release flutter/build/archives:windows_flutter gen_snapshot windows flutter/build/archives:artifacts + +# Build debug Windows artifacts +# These are output to the `windows-x64` directory in host_debug, and are used +# by `flutter build windows --release`. +$GN $GN_ARGS --no-prebuilt-dart-sdk +$NINJA -C ./out/host_debug flutter/build/archives:artifacts + +# If this gives you trouble, try using VS2019 instead. I had trouble with 2022. +# Android arm64 release +$GN $GN_ARGS --android --android-cpu=arm64 --runtime-mode=release +$NINJA -C ./out/android_release_arm64 $TARGETS + +# Android arm32 release +$GN $GN_ARGS --runtime-mode=release --android +$NINJA -C out/android_release $TARGETS + +# Android x64 release +$GN $GN_ARGS --android --android-cpu=x64 --runtime-mode=release +$NINJA -C ./out/android_release_x64 $TARGETS + +# We could also build the `patch` tool for Windows here. diff --git a/shorebird/ci/internal/win_setup.sh b/shorebird/ci/internal/win_setup.sh new file mode 100755 index 0000000000000..971135644f66b --- /dev/null +++ b/shorebird/ci/internal/win_setup.sh @@ -0,0 +1,8 @@ +#!/bin/bash -e + +# Usage: +# ./windows_setup.sh + +# Add the MSVC toolchain to Rust. +rustup target add \ + x86_64-pc-windows-msvc diff --git a/shorebird/ci/internal/win_upload.sh b/shorebird/ci/internal/win_upload.sh new file mode 100755 index 0000000000000..952457109e40c --- /dev/null +++ b/shorebird/ci/internal/win_upload.sh @@ -0,0 +1,90 @@ +#!/bin/bash -e + +# Usage: +# ./win_upload.sh engine_path git_hash +ENGINE_ROOT=$1 +ENGINE_HASH=$2 + +STORAGE_BUCKET="download.shorebird.dev" +SHOREBIRD_ROOT=gs://$STORAGE_BUCKET/shorebird/$ENGINE_HASH + +ENGINE_SRC=$ENGINE_ROOT/src +ENGINE_OUT=$ENGINE_SRC/out +ENGINE_FLUTTER=$ENGINE_SRC/flutter +# FLUTTER_ROOT is the Flutter monorepo root (parent of engine/) +FLUTTER_ROOT=$(dirname $ENGINE_ROOT) + +cd $FLUTTER_ROOT + +# Compute the content-aware hash for the Dart SDK. +# This allows Flutter checkouts that haven't changed engine content to share +# the same pre-built Dart SDK, even if they have different git commit SHAs. +CONTENT_HASH=$($FLUTTER_ROOT/bin/internal/content_aware_hash.sh) + +# We do not generate a manifest file, we assume another builder did that. + +# TODO(eseidel): This should not be in shell, it's too complicated/repetitive. + +HOST_ARCH='windows-x64' + +INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$ENGINE_HASH" + +# Dart SDK +# This gets uploaded to flutter_infra_release/flutter/\$engine/dart-sdk-$HOST_ARCH.zip +# We also upload to the content-aware hash path to support local development branches. +CONTENT_INFRA_ROOT="gs://$STORAGE_BUCKET/flutter_infra_release/flutter/$CONTENT_HASH" + +DART_SDK_DIR=$ENGINE_OUT/host_release/dart-sdk +DART_ZIP_FILE=dart-sdk-$HOST_ARCH.zip + +# Use 7zip to compress the Dart SDK, as zip isn't available on Windows and +# Powershell, which we would normally use in the form of +# `powershell Compress-Archive dart-sdk dart-sdk.zip`, doesn't play nicely +# with git bash paths (e.g. /c/Users/... instead of C:/Users/...) +/c/Program\ Files/7-Zip/7z a $DART_ZIP_FILE $DART_SDK_DIR +ZIPS_DEST=$INFRA_ROOT/$DART_ZIP_FILE +gsutil cp $DART_ZIP_FILE $ZIPS_DEST +# Also upload to content-aware hash path +gsutil cp $DART_ZIP_FILE $CONTENT_INFRA_ROOT/$DART_ZIP_FILE + +# Android Arm64 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release_arm64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm64-release +ZIPS_DEST=$INFRA_ROOT/android-arm64-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Android Arm32 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release +ZIPS_OUT=$ARCH_OUT/zip_archives/android-arm-release +ZIPS_DEST=$INFRA_ROOT/android-arm-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# Android x64 release gen_snapshot +ARCH_OUT=$ENGINE_OUT/android_release_x64 +ZIPS_OUT=$ARCH_OUT/zip_archives/android-x64-release +ZIPS_DEST=$INFRA_ROOT/android-x64-release +gsutil cp $ZIPS_OUT/$HOST_ARCH.zip $ZIPS_DEST/$HOST_ARCH.zip + +# We could upload patch if we built it here. +# gsutil cp $ENGINE_OUT/host_release/patch.zip $SHOREBIRD_ROOT/patch-win-x64.zip + +# Engine release artifacts +ARCH_OUT=$ENGINE_OUT/host_release +ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH-release +ZIPS_DEST=$INFRA_ROOT/$HOST_ARCH-release +gsutil cp $ZIPS_OUT/$HOST_ARCH-flutter.zip $ZIPS_DEST/$HOST_ARCH-flutter.zip + +# We want to build flutter/tools/font_subset, but that doesn't work with +# --no-prebuilt-dart-sdk. +# https://github.com/flutter/flutter/issues/164531 +# # Windows x64 host_release font_subset (ConstFinder) +# ARCH_OUT=$ENGINE_OUT/host_release +# ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +# ZIPS_DEST=$INFRA_ROOT/windows-x64-release +# gsutil cp $ZIPS_OUT/font-subset.zip $ZIPS_DEST/font-subset.zip + +# Engine debug artifacts (not sure why this is needed?) +ARCH_OUT=$ENGINE_OUT/host_debug +ZIPS_OUT=$ARCH_OUT/zip_archives/$HOST_ARCH +ZIPS_DEST=$INFRA_ROOT/$HOST_ARCH +gsutil cp $ZIPS_OUT/artifacts.zip $ZIPS_DEST/artifacts.zip diff --git a/shorebird/ci/linux_build_and_upload.sh b/shorebird/ci/linux_build_and_upload.sh new file mode 100755 index 0000000000000..9657e14178695 --- /dev/null +++ b/shorebird/ci/linux_build_and_upload.sh @@ -0,0 +1,32 @@ +#!/bin/bash -e + +# Usage: +# ./linux_build_and_upload.sh flutter_root engine_hash +# +# This is the main entrypoint for building and uploading Linux engine artifacts. +# It is called from the _build_engine repository's CI scripts. + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 flutter_root engine_hash" + exit 1 +fi + +FLUTTER_ROOT=$1 +ENGINE_HASH=$2 +ENGINE_ROOT=$FLUTTER_ROOT/engine + +# Get the absolute path to the directory of this script. +SCRIPT_DIR=$(cd $(dirname $0) && pwd) + +echo "Building engine at $ENGINE_ROOT and uploading to gs://download.shorebird.dev" + +cd $SCRIPT_DIR + +# Run the setup script. +./internal/linux_setup.sh + +# Then run the build. +./internal/linux_build.sh $ENGINE_ROOT + +# Copy Shorebird engine artifacts to Google Cloud Storage. +./internal/linux_upload.sh $ENGINE_ROOT $ENGINE_HASH diff --git a/shorebird/ci/mac_build_and_upload.sh b/shorebird/ci/mac_build_and_upload.sh new file mode 100755 index 0000000000000..9ee302620d207 --- /dev/null +++ b/shorebird/ci/mac_build_and_upload.sh @@ -0,0 +1,32 @@ +#!/bin/bash -e + +# Usage: +# ./mac_build_and_upload.sh flutter_root engine_hash +# +# This is the main entrypoint for building and uploading macOS engine artifacts. +# It is called from the _build_engine repository's CI scripts. + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 flutter_root engine_hash" + exit 1 +fi + +FLUTTER_ROOT=$1 +ENGINE_HASH=$2 +ENGINE_ROOT=$FLUTTER_ROOT/engine + +# Get the absolute path to the directory of this script. +SCRIPT_DIR=$(cd $(dirname $0) && pwd) + +echo "Building engine at $ENGINE_ROOT and uploading to gs://download.shorebird.dev" + +cd $SCRIPT_DIR + +# Run the setup script. +./internal/mac_setup.sh + +# Then run the build. +./internal/mac_build.sh $ENGINE_ROOT + +# Copy Shorebird engine artifacts to Google Cloud Storage. +./internal/mac_upload.sh $ENGINE_ROOT $ENGINE_HASH diff --git a/shorebird/ci/shard_runner/.gitignore b/shorebird/ci/shard_runner/.gitignore new file mode 100644 index 0000000000000..929f3d7e632c0 --- /dev/null +++ b/shorebird/ci/shard_runner/.gitignore @@ -0,0 +1,2 @@ +# Override root .gitignore to track our lockfile +!pubspec.lock diff --git a/shorebird/ci/shard_runner/analysis_options.yaml b/shorebird/ci/shard_runner/analysis_options.yaml new file mode 100644 index 0000000000000..d04adaf90938a --- /dev/null +++ b/shorebird/ci/shard_runner/analysis_options.yaml @@ -0,0 +1,7 @@ +include: package:lints/recommended.yaml + +analyzer: + language: + strict-casts: true + strict-inference: true + strict-raw-types: true diff --git a/shorebird/ci/shard_runner/bin/compare_buckets.dart b/shorebird/ci/shard_runner/bin/compare_buckets.dart new file mode 100644 index 0000000000000..958c238bb0230 --- /dev/null +++ b/shorebird/ci/shard_runner/bin/compare_buckets.dart @@ -0,0 +1,158 @@ +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:path/path.dart' as p; +import 'package:shard_runner/config.dart'; + +/// Compares artifacts between two GCS buckets for a given engine revision. +/// +/// Usage: dart run shard_runner:compare_buckets [options] +/// +/// Example: +/// dart run shard_runner:compare_buckets \ +/// --engine-revision abc123 \ +/// --test-bucket shorebird-build-test \ +/// --production-bucket download.shorebird.dev +Future main(List args) async { + final ArgParser parser = ArgParser() + ..addOption('engine-revision', + abbr: 'r', help: 'Engine revision (git hash)', mandatory: true) + ..addOption('test-bucket', + abbr: 't', help: 'Test bucket to compare', mandatory: true) + ..addOption('production-bucket', + abbr: 'p', + help: 'Production bucket (default: download.shorebird.dev)', + defaultsTo: 'download.shorebird.dev') + ..addOption('config-dir', + abbr: 'c', help: 'Config directory containing shards/*.json') + ..addFlag('verbose', abbr: 'v', help: 'Show detailed output') + ..addFlag('help', abbr: 'h', negatable: false, help: 'Show this help'); + + final ArgResults results = parser.parse(args); + + if (results['help'] as bool) { + print('Usage: dart run shard_runner:compare_buckets [options]'); + print(''); + print('Compares artifacts between test and production GCS buckets.'); + print('Uses gsutil hash to compare file checksums (MD5 + CRC32C).'); + print(''); + print(parser.usage); + exit(0); + } + + final String engineRevision = results['engine-revision'] as String; + final String testBucket = results['test-bucket'] as String; + final String productionBucket = results['production-bucket'] as String; + final String? configDirPath = results['config-dir'] as String?; + final bool verbose = results['verbose'] as bool; + + // Find config directory + final String configDir = configDirPath ?? + p.join(p.dirname(p.dirname(Platform.script.toFilePath())), 'shards'); + + print('=' * 60); + print('Compare Buckets'); + print('=' * 60); + print('Engine revision: $engineRevision'); + print('Test bucket: $testBucket'); + print('Production bucket: $productionBucket'); + print('Config dir: $configDir'); + print(''); + + // Load configs to get artifact paths + const List platforms = ['linux', 'macos', 'windows']; + final Map configs = {}; + for (final String platform in platforms) { + configs[platform] = PlatformConfig.load(platform, configDir); + } + + // Collect all artifact paths + final List artifacts = []; + for (final PlatformConfig config in configs.values) { + for (final ShardDef shard in config.shards.values) { + for (final ArtifactDef artifact in shard.artifacts) { + final String dstPath = + artifact.dst.replaceAll(r'$engine', engineRevision); + artifacts.add(dstPath); + } + } + } + + // Add manifest + artifacts.add('shorebird/$engineRevision/artifacts_manifest.yaml'); + + print('Comparing ${artifacts.length} artifacts...\n'); + + int matches = 0; + int mismatches = 0; + int missing = 0; + + for (final String artifact in artifacts) { + final String testUri = 'gs://$testBucket/$artifact'; + final String prodUri = 'gs://$productionBucket/$artifact'; + + // Get hash from test bucket + final String? testHash = await _getHash(testUri); + if (testHash == null) { + if (verbose) print('[MISSING] $artifact (not in test bucket)'); + missing++; + continue; + } + + // Get hash from production bucket + final String? prodHash = await _getHash(prodUri); + if (prodHash == null) { + if (verbose) print('[MISSING] $artifact (not in production bucket)'); + missing++; + continue; + } + + // Compare hashes + if (testHash == prodHash) { + if (verbose) print('[OK] $artifact'); + matches++; + } else { + print('[MISMATCH] $artifact'); + print(' Test: $testHash'); + print(' Prod: $prodHash'); + mismatches++; + } + } + + print(''); + print('=' * 60); + print('Results:'); + print(' Matches: $matches'); + print(' Mismatches: $mismatches'); + print(' Missing: $missing'); + print('=' * 60); + + if (mismatches > 0) { + print('\nWARNING: Found $mismatches mismatched artifacts!'); + exit(1); + } else if (missing > 0) { + print('\nWARNING: Found $missing missing artifacts.'); + exit(2); + } else { + print('\nSUCCESS: All artifacts match!'); + } +} + +/// Gets the MD5 hash of a GCS object using gsutil hash. +Future _getHash(String uri) async { + final ProcessResult result = + await Process.run('gsutil', ['hash', uri]); + if (result.exitCode != 0) { + return null; + } + + // Parse output for MD5 hash + // Example output: + // Hashes [hex] for gs://bucket/path: + // Hash (crc32c): abc123== + // Hash (md5): xyz789== + final String output = result.stdout as String; + final RegExpMatch? md5Match = + RegExp(r'Hash \(md5\):\s+(\S+)').firstMatch(output); + return md5Match?.group(1); +} diff --git a/shorebird/ci/shard_runner/bin/compose.dart b/shorebird/ci/shard_runner/bin/compose.dart new file mode 100644 index 0000000000000..73cdb44393bb7 --- /dev/null +++ b/shorebird/ci/shard_runner/bin/compose.dart @@ -0,0 +1,104 @@ +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:path/path.dart' as p; +import 'package:shard_runner/cli.dart'; +import 'package:shard_runner/compose_config.dart'; +import 'package:shard_runner/gcs.dart'; +import 'package:shard_runner/process.dart'; + +/// Composes artifacts from multiple shards into final outputs. +/// +/// Usage: dart run shard_runner:compose [options] +/// +/// Example: +/// dart run shard_runner:compose ios-framework --engine-src ~/.engine_checkout/engine/src +Future main(List args) async { + final ArgParser parser = ArgParser(); + CliConfig.addCommonOptions(parser, includeUpload: false); + parser.addFlag('download', + defaultsTo: true, help: 'Download artifacts from GCS staging'); + + final ArgResults results = parser.parse(args); + + if (results['help'] as bool || results.rest.isEmpty) { + print('Usage: dart run shard_runner:compose [options]'); + print(''); + print('Compose names: ios-framework, macos-framework, macos-gen-snapshot'); + print(''); + print(parser.usage); + exit(results['help'] as bool ? 0 : 1); + } + + final String composeName = results.rest[0]; + final CliConfig cli = + CliConfig.fromArgs(results, scriptPath: Platform.script.toFilePath()); + final bool shouldDownload = results['download'] as bool; + + cli.printHeader('Compose Runner', { + 'Compose:': composeName, + 'Download:': shouldDownload.toString(), + }); + + // Load compose config + final ComposeConfig config; + try { + config = ComposeConfig.load(cli.configDir); + } on FileSystemException catch (e) { + print('Error: ${e.message} at ${e.path}'); + exit(1); + } + + final ComposeDef composeDef; + try { + composeDef = config.getCompose(composeName); + } on ArgumentError catch (e) { + print('Error: ${e.message}'); + exit(1); + } + + print('\n[Compose] Requires shards: ${composeDef.requires.join(', ')}'); + + // Download artifacts from each required shard + if (shouldDownload) { + for (final String shard in composeDef.requires) { + print('\n[Download] Fetching $shard artifacts...'); + await downloadFromStaging( + runId: cli.runId, + platform: 'macos', // Compose only runs for macOS currently + shard: shard, + destDir: p.join(cli.engineSrc, 'out'), + ); + } + } + + // Build script arguments + final String outDir = p.join(cli.engineSrc, 'out', 'release'); + + // Ensure output directory exists + Directory(outDir).createSync(recursive: true); + + // Build script arguments: expand path_args to absolute paths, pass flags as-is. + final List expandedArgs = ['--dst', outDir]; + for (final MapEntry entry in composeDef.pathArgs.entries) { + expandedArgs + .addAll([entry.key, p.join(cli.engineSrc, 'out', entry.value)]); + } + expandedArgs.addAll(composeDef.flags); + + // Run the composition script + print('\n[Compose] Running ${composeDef.script}...'); + print('[Compose] Args: ${expandedArgs.join(' ')}'); + + await runChecked( + 'python3', + [p.join(cli.engineSrc, composeDef.script), ...expandedArgs], + workingDirectory: cli.engineSrc, + description: 'Compose ($composeName)', + ); + + print('[Compose] Complete'); + print('\n${'='.padRight(60, '=')}'); + print('Compose $composeName completed successfully'); + print('='.padRight(60, '=')); +} diff --git a/shorebird/ci/shard_runner/bin/finalize.dart b/shorebird/ci/shard_runner/bin/finalize.dart new file mode 100644 index 0000000000000..6534aedec8162 --- /dev/null +++ b/shorebird/ci/shard_runner/bin/finalize.dart @@ -0,0 +1,206 @@ +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:path/path.dart' as p; +import 'package:shard_runner/cli.dart'; +import 'package:shard_runner/config.dart'; +import 'package:shard_runner/gcs.dart'; +import 'package:shard_runner/manifest.dart'; +import 'package:shard_runner/process.dart'; + +/// Finalizes a sharded build by generating manifest and uploading artifacts. +/// +/// Usage: dart run shard_runner:finalize [options] +/// +/// Example: +/// dart run shard_runner:finalize --engine-revision abc123 +Future main(List args) async { + final ArgParser parser = ArgParser() + ..addOption('engine-revision', + abbr: 'r', help: 'Engine revision (git hash)', mandatory: true) + ..addOption('base-engine-revision', + help: 'Base Flutter engine revision for manifest') + ..addOption('content-hash', help: 'Content-aware hash for Dart SDK') + ..addOption('bucket', + abbr: 'b', + help: 'GCS bucket for uploads (default: download.shorebird.dev)', + defaultsTo: 'download.shorebird.dev') + ..addFlag('download', + defaultsTo: true, help: 'Download artifacts from GCS staging') + ..addFlag('upload', + defaultsTo: true, help: 'Upload artifacts to GCS bucket'); + + CliConfig.addCommonOptions(parser, includeUpload: false); + + final ArgResults results = parser.parse(args); + + if (results['help'] as bool) { + print('Usage: dart run shard_runner:finalize [options]'); + print(''); + print(parser.usage); + exit(0); + } + + final CliConfig cli = + CliConfig.fromArgs(results, scriptPath: Platform.script.toFilePath()); + final String engineRevision = results['engine-revision'] as String; + final String baseEngineRevision = + results['base-engine-revision'] as String? ?? engineRevision; + final String? contentHash = results['content-hash'] as String?; + final String bucket = results['bucket'] as String; + final bool shouldDownload = results['download'] as bool; + final bool shouldUpload = results['upload'] as bool; + + cli.printHeader('Finalize Build', { + 'Engine:': engineRevision, + 'Base Engine:': baseEngineRevision, + 'Bucket:': bucket, + 'Download:': shouldDownload.toString(), + 'Upload:': shouldUpload.toString(), + }); + + // Load shard configs for each platform + const List platforms = ['linux', 'macos', 'windows']; + final Map configs = {}; + for (final String platform in platforms) { + configs[platform] = PlatformConfig.load(platform, cli.configDir); + } + + // Download all artifacts from staging + if (shouldDownload) { + final String outDir = p.join(cli.engineSrc, 'out'); + Directory(outDir).createSync(recursive: true); + + for (final String platform in platforms) { + final PlatformConfig config = configs[platform]!; + for (final String shardName in config.shards.keys) { + print('\n[Download] Fetching $platform/$shardName...'); + await downloadFromStaging( + runId: cli.runId, + platform: platform, + shard: shardName, + destDir: outDir, + ); + } + } + } + + // Generate manifest + print('\n[Manifest] Generating artifacts_manifest.yaml...'); + final String manifest = + generateManifest(baseEngineRevision, configDir: cli.configDir); + final File manifestFile = + File(p.join(cli.engineSrc, 'artifacts_manifest.yaml')); + manifestFile.writeAsStringSync(manifest); + print('[Manifest] Written to ${manifestFile.path}'); + + // Upload to production + if (shouldUpload) { + print('\n[Upload] Uploading to $bucket...'); + await uploadToProduction( + engineSrc: cli.engineSrc, + engineRevision: engineRevision, + contentHash: contentHash, + configs: configs, + bucket: bucket, + ); + } + + print('\n${'=' * 60}'); + print('Finalize completed successfully'); + print('=' * 60); +} + +/// Production storage bucket name (without gs:// prefix). +const String productionBucket = 'download.shorebird.dev'; + +/// Uploads artifacts to a GCS bucket based on config definitions. +Future uploadToProduction({ + required String engineSrc, + required String engineRevision, + required String? contentHash, + required Map configs, + required String bucket, +}) async { + final String outDir = p.join(engineSrc, 'out'); + final String bucketUri = 'gs://$bucket'; + + // Helper to run gsutil cp + Future gscp(String src, String dest) async { + print('[Upload] $src -> $dest'); + await runChecked('gsutil', ['cp', src, dest], + description: 'gsutil cp $src'); + } + + // Helper to zip a directory and upload + Future zipAndUpload(String srcPath, String dest) async { + final String tempZip = '$srcPath.zip'; + print('[Zip] Creating $tempZip...'); + await runChecked( + 'zip', + ['-r', tempZip, '.'], + workingDirectory: srcPath, + description: 'zip $tempZip', + ); + await gscp(tempZip, dest); + File(tempZip).deleteSync(); + } + + // Process artifacts from all configs + for (final MapEntry entry in configs.entries) { + final String platform = entry.key; + final PlatformConfig config = entry.value; + + for (final MapEntry shardEntry in config.shards.entries) { + final String shardName = shardEntry.key; + final ShardDef shard = shardEntry.value; + + print('\n[Upload] Processing $platform/$shardName...'); + + for (final ArtifactDef artifact in shard.artifacts) { + // Resolve source path + final String srcPath = p.join(outDir, artifact.src); + + // Resolve destination path (replace $engine with actual revision) + final String dstPath = + artifact.dst.replaceAll(r'$engine', engineRevision); + final String fullDest = '$bucketUri/$dstPath'; + + // Check if source exists + final File srcFile = File(srcPath); + final Directory srcDir = Directory(srcPath); + final bool srcExists = srcFile.existsSync() || srcDir.existsSync(); + + if (!srcExists) { + print('[Skip] $srcPath (not found)'); + continue; + } + + // Handle zip flag + if (artifact.zip && srcDir.existsSync()) { + await zipAndUpload(srcPath, fullDest); + } else { + await gscp(srcPath, fullDest); + } + + // Handle content-hash uploads (for Dart SDK) + if (artifact.contentHash && contentHash != null) { + final String contentDstPath = + artifact.dst.replaceAll(r'$engine', contentHash); + final String contentFullDest = '$bucketUri/$contentDstPath'; + await gscp(srcPath, contentFullDest); + } + } + } + } + + // Upload manifest + final String manifestFile = p.join(engineSrc, 'artifacts_manifest.yaml'); + if (File(manifestFile).existsSync()) { + final String manifestDest = + '$bucketUri/shorebird/$engineRevision/artifacts_manifest.yaml'; + await gscp(manifestFile, manifestDest); + } + + print('\n[Upload] Production upload complete'); +} diff --git a/shorebird/ci/shard_runner/bin/run_shard.dart b/shorebird/ci/shard_runner/bin/run_shard.dart new file mode 100644 index 0000000000000..ed8c4cddac435 --- /dev/null +++ b/shorebird/ci/shard_runner/bin/run_shard.dart @@ -0,0 +1,84 @@ +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:shard_runner/cli.dart'; +import 'package:shard_runner/config.dart'; +import 'package:shard_runner/gcs.dart'; + +/// Runs a single build shard. +/// +/// Usage: dart run shard_runner:run_shard [options] +/// +/// Example: +/// dart run shard_runner:run_shard linux android-arm64 --engine-src ~/.engine_checkout/engine/src +Future main(List args) async { + final ArgParser parser = ArgParser(); + CliConfig.addCommonOptions(parser); + + final ArgResults results = parser.parse(args); + + if (results['help'] as bool || results.rest.length < 2) { + print( + 'Usage: dart run shard_runner:run_shard [options]'); + print(''); + print('Platforms: linux, macos, windows'); + print(''); + print(parser.usage); + exit(results['help'] as bool ? 0 : 1); + } + + final String platform = results.rest[0]; + final String shard = results.rest[1]; + final CliConfig cli = + CliConfig.fromArgs(results, scriptPath: Platform.script.toFilePath()); + + cli.printHeader('Shard Runner', { + 'Platform:': platform, + 'Shard:': shard, + 'Upload:': cli.shouldUpload.toString(), + }); + + cli.verifyEngineSrc(); + + // Load config + print('\n[Config] Loading $platform.json...'); + final PlatformConfig config = PlatformConfig.load(platform, cli.configDir); + final ShardDef shardDef = config.getShard(shard); + + print('[Config] Found ${shardDef.steps.length} step(s)'); + + // Collect output directories from GnNinja steps for upload + final List outDirs = [ + for (final BuildStep step in shardDef.steps) + if (step is GnNinjaStep) step.outDir, + ]; + + // Execute steps + final Stopwatch stopwatch = Stopwatch()..start(); + + for (int i = 0; i < shardDef.steps.length; i++) { + final BuildStep step = shardDef.steps[i]; + print( + '\n[${'Step ${i + 1}/${shardDef.steps.length}'}] ${step.runtimeType}'); + + await step.execute(cli.engineSrc); + } + + stopwatch.stop(); + print('\n[Build] Complete in ${stopwatch.elapsed}'); + + // Upload to GCS staging + if (cli.shouldUpload && outDirs.isNotEmpty) { + await uploadToStaging( + runId: cli.runId, + platform: platform, + shard: shard, + engineSrc: cli.engineSrc, + outDirs: outDirs, + ); + } + + print('\n${'='.padRight(60, '=')}'); + print('Shard $platform/$shard completed successfully'); + print('='.padRight(60, '=')); +} diff --git a/shorebird/ci/shard_runner/lib/cli.dart b/shorebird/ci/shard_runner/lib/cli.dart new file mode 100644 index 0000000000000..23a2a0b99a5a3 --- /dev/null +++ b/shorebird/ci/shard_runner/lib/cli.dart @@ -0,0 +1,83 @@ +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as p; + +/// Common CLI configuration used by shard runner scripts. +@immutable +class CliConfig { + CliConfig({ + required this.engineSrc, + required this.configDir, + required this.runId, + required this.shouldUpload, + }); + + /// Parses common options from ArgResults. + /// + /// [scriptPath] should be Platform.script.toFilePath() from the calling script. + factory CliConfig.fromArgs(ArgResults results, {required String scriptPath}) { + final String engineSrc = p.canonicalize(results['engine-src'] as String); + + // Config directory defaults to shorebird/ci (grandparent of bin/*.dart) + final String configDir = results['config-dir'] as String? ?? + p.dirname(p.dirname(p.dirname(scriptPath))); + + final String runId = results['run-id'] as String; + + final bool shouldUpload = + !results.options.contains('upload') || results['upload'] as bool; + + return CliConfig( + engineSrc: engineSrc, + configDir: configDir, + runId: runId, + shouldUpload: shouldUpload, + ); + } + final String engineSrc; + final String configDir; + final String runId; + final bool shouldUpload; + + /// Creates common argument parser options. + static void addCommonOptions(ArgParser parser, {bool includeUpload = true}) { + parser + ..addOption('engine-src', + abbr: 'e', help: 'Path to engine/src directory', mandatory: true) + ..addOption('run-id', + help: 'Build run identifier (use "local" for local development)', + mandatory: true) + ..addOption('config-dir', + abbr: 'c', help: 'Path to config directory (shorebird/ci)') + ..addFlag('help', abbr: 'h', negatable: false, help: 'Show this help'); + + if (includeUpload) { + parser.addFlag('upload', + defaultsTo: true, help: 'Upload artifacts to GCS staging'); + } + } + + /// Prints a standard header with configuration info. + void printHeader(String title, Map extra) { + print('='.padRight(60, '=')); + print(title); + print('='.padRight(60, '=')); + print('Engine: $engineSrc'); + print('Config: $configDir'); + print('Run ID: $runId'); + for (final MapEntry entry in extra.entries) { + print('${entry.key.padRight(12)}${entry.value}'); + } + print('='.padRight(60, '=')); + } + + /// Verifies that the engine source directory exists. + void verifyEngineSrc() { + if (!Directory(engineSrc).existsSync()) { + print('Error: Engine source not found at $engineSrc'); + exit(1); + } + } +} diff --git a/shorebird/ci/shard_runner/lib/compose_config.dart b/shorebird/ci/shard_runner/lib/compose_config.dart new file mode 100644 index 0000000000000..927748d326484 --- /dev/null +++ b/shorebird/ci/shard_runner/lib/compose_config.dart @@ -0,0 +1,76 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as p; + +/// Configuration for all compose operations. +@immutable +class ComposeConfig { + ComposeConfig({required this.composes}); + + factory ComposeConfig.fromJson(Map json) { + return ComposeConfig( + composes: json.map( + (String key, value) => + MapEntry(key, ComposeDef.fromJson(value as Map)), + ), + ); + } + final Map composes; + + static ComposeConfig load(String configDir) { + final File file = File(p.join(configDir, 'compose.json')); + if (!file.existsSync()) { + throw FileSystemException('compose.json not found', file.path); + } + final String content = file.readAsStringSync(); + final Map json = + jsonDecode(content) as Map; + return ComposeConfig.fromJson(json); + } + + ComposeDef getCompose(String name) { + final ComposeDef? compose = composes[name]; + if (compose == null) { + throw ArgumentError( + 'Unknown compose: $name. Available: ${composes.keys.join(', ')}'); + } + return compose; + } +} + +/// Definition of a single compose operation. +@immutable +class ComposeDef { + ComposeDef({ + required this.requires, + required this.script, + this.flags = const [], + this.pathArgs = const {}, + }); + + factory ComposeDef.fromJson(Map json) { + return ComposeDef( + requires: (json['requires'] as List).cast(), + script: json['script'] as String, + flags: (json['flags'] as List?)?.cast() ?? [], + pathArgs: (json['path_args'] as Map?) + ?.cast() ?? + {}, + ); + } + + /// Shards that must complete before this compose can run. + final List requires; + + /// Path to the Python script to execute (relative to engine/src). + final String script; + + /// Boolean flags to pass to the script (e.g., --dsym, --strip, --zip). + final List flags; + + /// Arguments whose values are paths relative to out/ (e.g., --arm64-out-dir: ios_release). + /// These are expanded to absolute paths at runtime. + final Map pathArgs; +} diff --git a/shorebird/ci/shard_runner/lib/config.dart b/shorebird/ci/shard_runner/lib/config.dart new file mode 100644 index 0000000000000..9fecdabe1cfd2 --- /dev/null +++ b/shorebird/ci/shard_runner/lib/config.dart @@ -0,0 +1,268 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as p; +import 'package:shard_runner/process.dart'; + +/// Configuration for all shards on a platform. +@immutable +class PlatformConfig { + PlatformConfig({required this.shards}); + + factory PlatformConfig.fromJson(Map json) { + return PlatformConfig( + shards: json.map( + (String key, value) => + MapEntry(key, ShardDef.fromJson(value as Map)), + ), + ); + } + final Map shards; + + ShardDef getShard(String name) { + final ShardDef? shard = shards[name]; + if (shard == null) { + throw ArgumentError( + 'Unknown shard: $name. Available: ${shards.keys.join(', ')}', + ); + } + return shard; + } + + static PlatformConfig load(String platform, String configDir) { + final File file = File(p.join(configDir, 'shards', '$platform.json')); + if (!file.existsSync()) { + throw FileSystemException('Config file not found', file.path); + } + final String content = file.readAsStringSync(); + final Map json = + jsonDecode(content) as Map; + return PlatformConfig.fromJson(json); + } +} + +/// Definition of a single build shard. +@immutable +class ShardDef { + ShardDef({ + required this.steps, + this.composeInput, + this.artifacts = const [], + }); + + factory ShardDef.fromJson(Map json) { + final List steps = (json['steps'] as List) + .map((s) => BuildStep.fromJson(s as Map)) + .toList(); + + final List artifacts = (json['artifacts'] as List?) + ?.map((a) => ArtifactDef.fromJson(a as Map)) + .toList() ?? + []; + + return ShardDef( + steps: steps, + composeInput: json['compose_input'] as String?, + artifacts: artifacts, + ); + } + + /// Build steps to execute. For simple shards, this is a single GnNinja step. + final List steps; + + /// If set, this shard contributes to a compose operation. + final String? composeInput; + + /// Artifacts produced by this shard (paths relative to out_dir). + /// Used by finalize to know what to upload. + final List artifacts; +} + +/// Definition of an artifact to upload. +@immutable +class ArtifactDef { + ArtifactDef({ + required this.src, + required this.dst, + this.zip = false, + this.contentHash = false, + }); + + factory ArtifactDef.fromJson(Map json) { + return ArtifactDef( + src: json['src'] as String, + dst: json['dst'] as String, + zip: json['zip'] as bool? ?? false, + contentHash: json['content_hash'] as bool? ?? false, + ); + } + + /// Source path relative to out/ (or out// for single-step shards) + final String src; + + /// Destination path (relative to storage bucket root). + /// Supports placeholders: $engine (engine hash) + final String dst; + + /// If true, zip the source directory before uploading. + final bool zip; + + /// If true, also upload to content-hash path (for Dart SDK). + final bool contentHash; +} + +/// Base class for build steps. +sealed class BuildStep { + factory BuildStep.fromJson(Map json) { + final String type = json['type'] as String; + return switch (type) { + 'gn_ninja' => GnNinjaStep.fromJson(json), + 'rust' => RustStep.fromJson(json), + _ => throw ArgumentError('Unknown step type: $type'), + }; + } + Future execute(String engineSrc); +} + +/// A GN + Ninja build step. +@immutable +class GnNinjaStep implements BuildStep { + GnNinjaStep({ + required this.gnArgs, + required this.ninjaTargets, + required this.outDir, + }); + + factory GnNinjaStep.fromJson(Map json) { + return GnNinjaStep( + gnArgs: (json['gn_args'] as List).cast(), + ninjaTargets: (json['ninja_targets'] as List).cast(), + outDir: json['out_dir'] as String, + ); + } + final List gnArgs; + final List ninjaTargets; + final String outDir; + + @override + Future execute(String engineSrc) async { + // Import gn.dart functions + await _runGn(engineSrc, gnArgs, outDir); + await _runNinja(engineSrc, outDir, ninjaTargets); + } +} + +/// A Rust/Cargo build step. +@immutable +class RustStep implements BuildStep { + RustStep({required this.targets}); + + factory RustStep.fromJson(Map json) { + return RustStep(targets: (json['targets'] as List).cast()); + } + final List targets; + + @override + Future execute(String engineSrc) async { + await _runRust(engineSrc, targets); + } +} + +// Internal execution functions (to be moved to separate files) +Future _runGn(String engineSrc, List args, String outDir) async { + print('[GN] Building $outDir with args: ${args.join(' ')}'); + await runChecked( + 'python3', + [ + p.join(engineSrc, 'flutter', 'tools', 'gn'), + '--no-rbe', + '--no-enable-unittests', + '--target-dir', + outDir, + ...args, + ], + workingDirectory: engineSrc, + description: 'GN ($outDir)', + ); + print('[GN] Complete'); +} + +Future _runNinja( + String engineSrc, + String outDir, + List targets, +) async { + print('[Ninja] Building ${targets.join(' ')} in out/$outDir'); + await runChecked( + 'ninja', + ['-C', p.join(engineSrc, 'out', outDir), ...targets], + workingDirectory: engineSrc, + description: 'Ninja ($outDir)', + ); + print('[Ninja] Complete'); +} + +Future _runRust(String engineSrc, List targets) async { + final String updaterPath = p.join( + engineSrc, + 'flutter', + 'third_party', + 'updater', + 'library', + ); + + // Separate Android and non-Android targets + final List androidTargets = + targets.where((String t) => t.contains('android')).toList(); + final List otherTargets = + targets.where((String t) => !t.contains('android')).toList(); + + // Build all Android targets together with cargo-ndk + if (androidTargets.isNotEmpty) { + print('[Rust] Building Android targets: ${androidTargets.join(', ')}'); + + final List args = ['ndk']; + for (final String target in androidTargets) { + args.addAll(['--target', target]); + } + args.addAll(['build', '--release']); + + // The "unmodified" CIPD package keeps the NDK at the standard Android + // SDK path: android_tools/sdk/ndk/. + final Directory ndkParent = Directory( + p.join( + engineSrc, + 'flutter', + 'third_party', + 'android_tools', + 'sdk', + 'ndk', + ), + ); + final String ndkHome = + ndkParent.listSync().whereType().first.path; + + await runChecked( + 'cargo', + args, + workingDirectory: updaterPath, + environment: {'ANDROID_NDK_HOME': ndkHome}, + description: 'Cargo ndk (${androidTargets.join(', ')})', + ); + } + + // Build non-Android targets individually + for (final String target in otherTargets) { + print('[Rust] Building for target: $target'); + + await runChecked( + 'cargo', + ['build', '--release', '--target', target], + workingDirectory: updaterPath, + description: 'Cargo ($target)', + ); + } + + print('[Rust] Complete'); +} diff --git a/shorebird/ci/shard_runner/lib/gcs.dart b/shorebird/ci/shard_runner/lib/gcs.dart new file mode 100644 index 0000000000000..c4263658bc275 --- /dev/null +++ b/shorebird/ci/shard_runner/lib/gcs.dart @@ -0,0 +1,112 @@ +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:shard_runner/process.dart'; + +/// Staging bucket for intermediate artifacts. +const String stagingBucket = 'gs://shorebird-build-staging'; + +/// Uploads shard artifacts to GCS staging bucket. +/// +/// Artifacts are uploaded to: +/// gs://shorebird-build-staging/builds/{runId}/{platform}/{shard}/ +Future uploadToStaging({ + required String runId, + required String platform, + required String shard, + required String engineSrc, + required List outDirs, +}) async { + final String stagingRoot = '$stagingBucket/builds/$runId/$platform/$shard'; + + print('[GCS] Uploading to $stagingRoot'); + + for (final String outDir in outDirs) { + final String outPath = p.join(engineSrc, 'out', outDir); + if (!Directory(outPath).existsSync()) { + print('[GCS] Skipping $outDir (not found)'); + continue; + } + + // Create a tarball of the out directory + final String tarFile = '$outDir.tar.gz'; + print('[GCS] Creating $tarFile...'); + + await runChecked( + 'tar', + ['-czf', tarFile, '-C', p.join(engineSrc, 'out'), outDir], + workingDirectory: engineSrc, + description: 'tar create $tarFile', + ); + + // Upload to GCS + print('[GCS] Uploading $tarFile...'); + await runChecked( + 'gsutil', + ['-m', 'cp', p.join(engineSrc, tarFile), '$stagingRoot/'], + description: 'gsutil upload $tarFile', + ); + + // Clean up local tarball + File(p.join(engineSrc, tarFile)).deleteSync(); + } + + // Upload status file + final File statusFile = File(p.join(engineSrc, 'status.json')); + statusFile.writeAsStringSync('{"status": "success", "shard": "$shard"}'); + await runChecked('gsutil', ['cp', statusFile.path, '$stagingRoot/'], + description: 'gsutil upload status.json'); + statusFile.deleteSync(); + + print('[GCS] Upload complete'); +} + +/// Downloads artifacts from GCS staging bucket. +Future downloadFromStaging({ + required String runId, + required String platform, + required String shard, + required String destDir, +}) async { + final String stagingRoot = '$stagingBucket/builds/$runId/$platform/$shard'; + + print('[GCS] Downloading from $stagingRoot'); + + // List files in the staging location + final ProcessResult lsResult = await runChecked( + 'gsutil', + ['ls', stagingRoot], + description: 'gsutil ls $stagingRoot', + ); + + final List files = (lsResult.stdout as String) + .split('\n') + .where((String f) => f.endsWith('.tar.gz')) + .toList(); + + for (final String file in files) { + final String fileName = p.basename(file); + print('[GCS] Downloading $fileName...'); + + // Download + await runChecked( + 'gsutil', + ['cp', file, p.join(destDir, fileName)], + description: 'gsutil download $fileName', + ); + + // Extract + print('[GCS] Extracting $fileName...'); + await runChecked( + 'tar', + ['-xzf', fileName], + workingDirectory: destDir, + description: 'tar extract $fileName', + ); + + // Clean up tarball + File(p.join(destDir, fileName)).deleteSync(); + } + + print('[GCS] Download complete'); +} diff --git a/shorebird/ci/shard_runner/lib/manifest.dart b/shorebird/ci/shard_runner/lib/manifest.dart new file mode 100644 index 0000000000000..db5c3a5789248 --- /dev/null +++ b/shorebird/ci/shard_runner/lib/manifest.dart @@ -0,0 +1,26 @@ +import 'dart:io'; + +import 'package:path/path.dart' as p; + +/// Generates the artifacts manifest YAML from template. +/// +/// The manifest maps a Shorebird engine revision to a Flutter engine revision +/// and lists all artifact paths that should be proxied. +/// +/// [flutterEngineRevision] is the base Flutter engine revision this build is based on. +/// [configDir] is the path to the ci/ directory containing the template. +String generateManifest( + String flutterEngineRevision, { + required String configDir, +}) { + final templatePath = p.join(configDir, 'artifacts_manifest.template.yaml'); + final templateFile = File(templatePath); + + if (!templateFile.existsSync()) { + throw ArgumentError('Manifest template not found: $templatePath'); + } + + final template = templateFile.readAsStringSync(); + return template.replaceAll( + '{{flutter_engine_revision}}', flutterEngineRevision); +} diff --git a/shorebird/ci/shard_runner/lib/process.dart b/shorebird/ci/shard_runner/lib/process.dart new file mode 100644 index 0000000000000..1fb251b609d97 --- /dev/null +++ b/shorebird/ci/shard_runner/lib/process.dart @@ -0,0 +1,66 @@ +import 'dart:io'; + +/// Resolves an executable name for Windows, appending .cmd if needed. +/// +/// On Windows, many tools like gsutil and gcloud are installed as .cmd files. +/// This function checks if a .cmd version exists and uses it. +String _resolveExecutable(String executable) { + if (!Platform.isWindows) return executable; + + // Don't modify if it already has an extension + if (executable.endsWith('.exe') || + executable.endsWith('.cmd') || + executable.endsWith('.bat')) { + return executable; + } + + // Check if .cmd version exists in PATH + final String? path = Platform.environment['PATH']; + if (path == null) return executable; + + for (final String dir in path.split(';')) { + final File cmdFile = File('$dir\\$executable.cmd'); + if (cmdFile.existsSync()) { + return '$executable.cmd'; + } + } + + return executable; +} + +/// Runs a process and throws if it exits with a non-zero code. +/// +/// Returns the [ProcessResult] for callers that need stdout/stderr. +Future runChecked( + String executable, + List arguments, { + String? workingDirectory, + Map? environment, + String? description, +}) async { + final String resolvedExecutable = _resolveExecutable(executable); + + final ProcessResult result = await Process.run( + resolvedExecutable, + arguments, + workingDirectory: workingDirectory, + environment: environment, + ); + + if (result.exitCode != 0) { + final String desc = description ?? '$executable ${arguments.join(' ')}'; + final String stderr = (result.stderr as String).trim(); + final String stdout = (result.stdout as String).trim(); + final StringBuffer message = + StringBuffer('$desc failed (exit ${result.exitCode})'); + if (stdout.isNotEmpty) { + message.write('\nSTDOUT: $stdout'); + } + if (stderr.isNotEmpty) { + message.write('\nSTDERR: $stderr'); + } + throw Exception(message.toString()); + } + + return result; +} diff --git a/shorebird/ci/shard_runner/pubspec.lock b/shorebird/ci/shard_runner/pubspec.lock new file mode 100644 index 0000000000000..14ad0cf1c27c0 --- /dev/null +++ b/shorebird/ci/shard_runner/pubspec.lock @@ -0,0 +1,389 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + _fe_analyzer_shared: + dependency: transitive + description: + name: _fe_analyzer_shared + sha256: "796d97d925add7ffcdf5595f33a2066a6e3cee97971e6dbef09b76b7880fd760" + url: "https://pub.dev" + source: hosted + version: "94.0.0" + analyzer: + dependency: transitive + description: + name: analyzer + sha256: "9c8ebb304d72c0a0c8764344627529d9503fc83d7d73e43ed727dc532f822e4b" + url: "https://pub.dev" + source: hosted + version: "10.0.2" + args: + dependency: "direct main" + description: + name: args + sha256: d0481093c50b1da8910eb0bb301626d4d8eb7284aa739614d2b394ee09e3ea04 + url: "https://pub.dev" + source: hosted + version: "2.7.0" + async: + dependency: transitive + description: + name: async + sha256: "758e6d74e971c3e5aceb4110bfd6698efc7f501675bcfe0c775459a8140750eb" + url: "https://pub.dev" + source: hosted + version: "2.13.0" + boolean_selector: + dependency: transitive + description: + name: boolean_selector + sha256: "8aab1771e1243a5063b8b0ff68042d67334e3feab9e95b9490f9a6ebf73b42ea" + url: "https://pub.dev" + source: hosted + version: "2.1.2" + cli_config: + dependency: transitive + description: + name: cli_config + sha256: ac20a183a07002b700f0c25e61b7ee46b23c309d76ab7b7640a028f18e4d99ec + url: "https://pub.dev" + source: hosted + version: "0.2.0" + collection: + dependency: "direct main" + description: + name: collection + sha256: "2f5709ae4d3d59dd8f7cd309b4e023046b57d8a6c82130785d2b0e5868084e76" + url: "https://pub.dev" + source: hosted + version: "1.19.1" + convert: + dependency: transitive + description: + name: convert + sha256: b30acd5944035672bc15c6b7a8b47d773e41e2f17de064350988c5d02adb1c68 + url: "https://pub.dev" + source: hosted + version: "3.1.2" + coverage: + dependency: transitive + description: + name: coverage + sha256: "5da775aa218eaf2151c721b16c01c7676fbfdd99cebba2bf64e8b807a28ff94d" + url: "https://pub.dev" + source: hosted + version: "1.15.0" + crypto: + dependency: transitive + description: + name: crypto + sha256: c8ea0233063ba03258fbcf2ca4d6dadfefe14f02fab57702265467a19f27fadf + url: "https://pub.dev" + source: hosted + version: "3.0.7" + file: + dependency: transitive + description: + name: file + sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4 + url: "https://pub.dev" + source: hosted + version: "7.0.1" + frontend_server_client: + dependency: transitive + description: + name: frontend_server_client + sha256: f64a0333a82f30b0cca061bc3d143813a486dc086b574bfb233b7c1372427694 + url: "https://pub.dev" + source: hosted + version: "4.0.0" + glob: + dependency: transitive + description: + name: glob + sha256: c3f1ee72c96f8f78935e18aa8cecced9ab132419e8625dc187e1c2408efc20de + url: "https://pub.dev" + source: hosted + version: "2.1.3" + http_multi_server: + dependency: transitive + description: + name: http_multi_server + sha256: aa6199f908078bb1c5efb8d8638d4ae191aac11b311132c3ef48ce352fb52ef8 + url: "https://pub.dev" + source: hosted + version: "3.2.2" + http_parser: + dependency: transitive + description: + name: http_parser + sha256: "178d74305e7866013777bab2c3d8726205dc5a4dd935297175b19a23a2e66571" + url: "https://pub.dev" + source: hosted + version: "4.1.2" + io: + dependency: transitive + description: + name: io + sha256: dfd5a80599cf0165756e3181807ed3e77daf6dd4137caaad72d0b7931597650b + url: "https://pub.dev" + source: hosted + version: "1.0.5" + lints: + dependency: "direct dev" + description: + name: lints + sha256: cbf8d4b858bb0134ef3ef87841abdf8d63bfc255c266b7bf6b39daa1085c4290 + url: "https://pub.dev" + source: hosted + version: "3.0.0" + logging: + dependency: transitive + description: + name: logging + sha256: c8245ada5f1717ed44271ed1c26b8ce85ca3228fd2ffdb75468ab01979309d61 + url: "https://pub.dev" + source: hosted + version: "1.3.0" + matcher: + dependency: transitive + description: + name: matcher + sha256: "12956d0ad8390bbcc63ca2e1469c0619946ccb52809807067a7020d57e647aa6" + url: "https://pub.dev" + source: hosted + version: "0.12.18" + meta: + dependency: "direct main" + description: + name: meta + sha256: "9f29b9bcc8ee287b1a31e0d01be0eae99a930dbffdaecf04b3f3d82a969f296f" + url: "https://pub.dev" + source: hosted + version: "1.18.1" + mime: + dependency: transitive + description: + name: mime + sha256: "41a20518f0cb1256669420fdba0cd90d21561e560ac240f26ef8322e45bb7ed6" + url: "https://pub.dev" + source: hosted + version: "2.0.0" + node_preamble: + dependency: transitive + description: + name: node_preamble + sha256: "6e7eac89047ab8a8d26cf16127b5ed26de65209847630400f9aefd7cd5c730db" + url: "https://pub.dev" + source: hosted + version: "2.0.2" + package_config: + dependency: transitive + description: + name: package_config + sha256: f096c55ebb7deb7e384101542bfba8c52696c1b56fca2eb62827989ef2353bbc + url: "https://pub.dev" + source: hosted + version: "2.2.0" + path: + dependency: "direct main" + description: + name: path + sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5" + url: "https://pub.dev" + source: hosted + version: "1.9.1" + pool: + dependency: transitive + description: + name: pool + sha256: "978783255c543aa3586a1b3c21f6e9d720eb315376a915872c61ef8b5c20177d" + url: "https://pub.dev" + source: hosted + version: "1.5.2" + pub_semver: + dependency: transitive + description: + name: pub_semver + sha256: "5bfcf68ca79ef689f8990d1160781b4bad40a3bd5e5218ad4076ddb7f4081585" + url: "https://pub.dev" + source: hosted + version: "2.2.0" + shelf: + dependency: transitive + description: + name: shelf + sha256: e7dd780a7ffb623c57850b33f43309312fc863fb6aa3d276a754bb299839ef12 + url: "https://pub.dev" + source: hosted + version: "1.4.2" + shelf_packages_handler: + dependency: transitive + description: + name: shelf_packages_handler + sha256: "89f967eca29607c933ba9571d838be31d67f53f6e4ee15147d5dc2934fee1b1e" + url: "https://pub.dev" + source: hosted + version: "3.0.2" + shelf_static: + dependency: transitive + description: + name: shelf_static + sha256: c87c3875f91262785dade62d135760c2c69cb217ac759485334c5857ad89f6e3 + url: "https://pub.dev" + source: hosted + version: "1.1.3" + shelf_web_socket: + dependency: transitive + description: + name: shelf_web_socket + sha256: "3632775c8e90d6c9712f883e633716432a27758216dfb61bd86a8321c0580925" + url: "https://pub.dev" + source: hosted + version: "3.0.0" + source_map_stack_trace: + dependency: transitive + description: + name: source_map_stack_trace + sha256: c0713a43e323c3302c2abe2a1cc89aa057a387101ebd280371d6a6c9fa68516b + url: "https://pub.dev" + source: hosted + version: "2.1.2" + source_maps: + dependency: transitive + description: + name: source_maps + sha256: "190222579a448b03896e0ca6eca5998fa810fda630c1d65e2f78b3f638f54812" + url: "https://pub.dev" + source: hosted + version: "0.10.13" + source_span: + dependency: transitive + description: + name: source_span + sha256: "254ee5351d6cb365c859e20ee823c3bb479bf4a293c22d17a9f1bf144ce86f7c" + url: "https://pub.dev" + source: hosted + version: "1.10.1" + stack_trace: + dependency: transitive + description: + name: stack_trace + sha256: "8b27215b45d22309b5cddda1aa2b19bdfec9df0e765f2de506401c071d38d1b1" + url: "https://pub.dev" + source: hosted + version: "1.12.1" + stream_channel: + dependency: transitive + description: + name: stream_channel + sha256: "969e04c80b8bcdf826f8f16579c7b14d780458bd97f56d107d3950fdbeef059d" + url: "https://pub.dev" + source: hosted + version: "2.1.4" + string_scanner: + dependency: transitive + description: + name: string_scanner + sha256: "921cd31725b72fe181906c6a94d987c78e3b98c2e205b397ea399d4054872b43" + url: "https://pub.dev" + source: hosted + version: "1.4.1" + term_glyph: + dependency: transitive + description: + name: term_glyph + sha256: "7f554798625ea768a7518313e58f83891c7f5024f88e46e7182a4558850a4b8e" + url: "https://pub.dev" + source: hosted + version: "1.2.2" + test: + dependency: "direct dev" + description: + name: test + sha256: "54c516bbb7cee2754d327ad4fca637f78abfc3cbcc5ace83b3eda117e42cd71a" + url: "https://pub.dev" + source: hosted + version: "1.29.0" + test_api: + dependency: transitive + description: + name: test_api + sha256: "93167629bfc610f71560ab9312acdda4959de4df6fac7492c89ff0d3886f6636" + url: "https://pub.dev" + source: hosted + version: "0.7.9" + test_core: + dependency: transitive + description: + name: test_core + sha256: "394f07d21f0f2255ec9e3989f21e54d3c7dc0e6e9dbce160e5a9c1a6be0e2943" + url: "https://pub.dev" + source: hosted + version: "0.6.15" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006 + url: "https://pub.dev" + source: hosted + version: "1.4.0" + vm_service: + dependency: transitive + description: + name: vm_service + sha256: "45caa6c5917fa127b5dbcfbd1fa60b14e583afdc08bfc96dda38886ca252eb60" + url: "https://pub.dev" + source: hosted + version: "15.0.2" + watcher: + dependency: transitive + description: + name: watcher + sha256: "1398c9f081a753f9226febe8900fce8f7d0a67163334e1c94a2438339d79d635" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + web: + dependency: transitive + description: + name: web + sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a" + url: "https://pub.dev" + source: hosted + version: "1.1.1" + web_socket: + dependency: transitive + description: + name: web_socket + sha256: "34d64019aa8e36bf9842ac014bb5d2f5586ca73df5e4d9bf5c936975cae6982c" + url: "https://pub.dev" + source: hosted + version: "1.0.1" + web_socket_channel: + dependency: transitive + description: + name: web_socket_channel + sha256: d645757fb0f4773d602444000a8131ff5d48c9e47adfe9772652dd1a4f2d45c8 + url: "https://pub.dev" + source: hosted + version: "3.0.3" + webkit_inspection_protocol: + dependency: transitive + description: + name: webkit_inspection_protocol + sha256: "87d3f2333bb240704cd3f1c6b5b7acd8a10e7f0bc28c28dcf14e782014f4a572" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + yaml: + dependency: transitive + description: + name: yaml + sha256: b9da305ac7c39faa3f030eccd175340f968459dae4af175130b3fc47e40d76ce + url: "https://pub.dev" + source: hosted + version: "3.1.3" +sdks: + dart: ">=3.9.0 <4.0.0" diff --git a/shorebird/ci/shard_runner/pubspec.yaml b/shorebird/ci/shard_runner/pubspec.yaml new file mode 100644 index 0000000000000..8e0834004a88b --- /dev/null +++ b/shorebird/ci/shard_runner/pubspec.yaml @@ -0,0 +1,17 @@ +name: shard_runner +description: Dart-based build shard runner for Shorebird CI +version: 0.1.0 +publish_to: none + +environment: + sdk: ^3.0.0 + +dependencies: + args: ^2.4.0 + collection: ^1.18.0 + meta: ^1.11.0 + path: ^1.8.0 + +dev_dependencies: + lints: ^3.0.0 + test: ^1.24.0 diff --git a/shorebird/ci/shard_runner/test/compose_config_test.dart b/shorebird/ci/shard_runner/test/compose_config_test.dart new file mode 100644 index 0000000000000..24868a6affcc5 --- /dev/null +++ b/shorebird/ci/shard_runner/test/compose_config_test.dart @@ -0,0 +1,119 @@ +import 'package:test/test.dart'; +import 'package:shard_runner/compose_config.dart'; + +void main() { + group('ComposeConfig', () { + test('parses compose definitions', () { + final Map json = { + 'ios-framework': { + 'requires': ['ios-release', 'ios-sim-x64', 'ios-sim-arm64'], + 'script': 'flutter/sky/tools/create_ios_framework.py', + 'flags': ['--dsym', '--strip'], + 'path_args': { + '--arm64-out-dir': 'ios_release', + }, + }, + 'macos-framework': { + 'requires': ['mac-arm64', 'mac-x64'], + 'script': 'flutter/sky/tools/create_macos_framework.py', + 'flags': ['--zip'], + }, + }; + + final ComposeConfig config = ComposeConfig.fromJson(json); + + expect(config.composes.length, 2); + expect(config.composes.containsKey('ios-framework'), true); + expect(config.composes.containsKey('macos-framework'), true); + }); + + test('getCompose returns correct definition', () { + final Map json = { + 'ios-framework': { + 'requires': ['ios-release'], + 'script': 'create_ios_framework.py', + }, + }; + + final ComposeConfig config = ComposeConfig.fromJson(json); + final ComposeDef compose = config.getCompose('ios-framework'); + + expect(compose.requires, ['ios-release']); + expect(compose.script, 'create_ios_framework.py'); + }); + + test('getCompose throws for unknown name', () { + final ComposeConfig config = + ComposeConfig(composes: {}); + + expect( + () => config.getCompose('nonexistent'), + throwsA(isA()), + ); + }); + }); + + group('ComposeDef', () { + test('parses all fields', () { + final Map json = { + 'requires': ['shard-a', 'shard-b'], + 'script': 'path/to/script.py', + 'flags': ['--dsym', '--strip'], + 'path_args': { + '--arm64-out-dir': 'ios_release', + '--x64-out-dir': 'ios_debug_sim', + }, + }; + + final ComposeDef compose = ComposeDef.fromJson(json); + + expect(compose.requires, ['shard-a', 'shard-b']); + expect(compose.script, 'path/to/script.py'); + expect(compose.flags, ['--dsym', '--strip']); + expect(compose.pathArgs, { + '--arm64-out-dir': 'ios_release', + '--x64-out-dir': 'ios_debug_sim', + }); + }); + + test('defaults flags and path_args when missing', () { + final Map json = { + 'requires': ['shard-a'], + 'script': 'script.py', + }; + + final ComposeDef compose = ComposeDef.fromJson(json); + + expect(compose.flags, isEmpty); + expect(compose.pathArgs, isEmpty); + }); + + test('parses ios-framework config correctly', () { + final Map json = { + 'requires': [ + 'ios-release', + 'ios-release-ext', + 'ios-sim-x64', + 'ios-sim-x64-ext', + 'ios-sim-arm64', + 'ios-sim-arm64-ext', + ], + 'script': 'flutter/sky/tools/create_ios_framework.py', + 'flags': ['--dsym', '--strip'], + 'path_args': { + '--arm64-out-dir': 'ios_release', + '--simulator-x64-out-dir': 'ios_debug_sim', + '--simulator-arm64-out-dir': 'ios_debug_sim_arm64', + }, + }; + + final ComposeDef compose = ComposeDef.fromJson(json); + + expect(compose.requires.length, 6); + expect(compose.script, 'flutter/sky/tools/create_ios_framework.py'); + expect(compose.flags, ['--dsym', '--strip']); + expect(compose.pathArgs.keys, contains('--arm64-out-dir')); + expect(compose.pathArgs['--arm64-out-dir'], 'ios_release'); + }); + }); +} diff --git a/shorebird/ci/shard_runner/test/config_test.dart b/shorebird/ci/shard_runner/test/config_test.dart new file mode 100644 index 0000000000000..ec3d508864758 --- /dev/null +++ b/shorebird/ci/shard_runner/test/config_test.dart @@ -0,0 +1,264 @@ +import 'package:test/test.dart'; +import 'package:shard_runner/config.dart'; + +void main() { + group('PlatformConfig', () { + test('parses single-step shard', () { + final json = { + 'android-arm64': { + 'steps': [ + { + 'type': 'gn_ninja', + 'gn_args': [ + '--android', + '--android-cpu=arm64', + '--runtime-mode=release' + ], + 'ninja_targets': ['default', 'gen_snapshot'], + 'out_dir': 'android_release_arm64', + }, + ], + }, + }; + + final config = PlatformConfig.fromJson(json); + + expect(config.shards.length, 1); + expect(config.shards.containsKey('android-arm64'), true); + + final shard = config.getShard('android-arm64'); + expect(shard.steps.length, 1); + expect(shard.steps.first, isA()); + expect(shard.artifacts, isEmpty); + + final step = shard.steps.first as GnNinjaStep; + expect(step.gnArgs, + ['--android', '--android-cpu=arm64', '--runtime-mode=release']); + expect(step.ninjaTargets, ['default', 'gen_snapshot']); + expect(step.outDir, 'android_release_arm64'); + }); + + test('parses shard with artifacts', () { + final json = { + 'android-arm64': { + 'steps': [ + { + 'type': 'gn_ninja', + 'gn_args': ['--android'], + 'ninja_targets': ['default'], + 'out_dir': 'android_release_arm64', + }, + ], + 'artifacts': [ + { + 'src': 'zip_archives/artifacts.zip', + 'dst': 'flutter_infra/\$engine/artifacts.zip' + }, + {'src': 'maven.pom', 'dst': 'maven/\$engine/maven.pom'}, + ], + }, + }; + + final config = PlatformConfig.fromJson(json); + final shard = config.getShard('android-arm64'); + + expect(shard.artifacts.length, 2); + expect(shard.artifacts[0].src, 'zip_archives/artifacts.zip'); + expect(shard.artifacts[0].dst, 'flutter_infra/\$engine/artifacts.zip'); + expect(shard.artifacts[1].src, 'maven.pom'); + }); + + test('parses multi-step shard', () { + final json = { + 'host': { + 'steps': [ + { + 'type': 'rust', + 'targets': ['aarch64-linux-android', 'x86_64-unknown-linux-gnu'], + }, + { + 'type': 'gn_ninja', + 'gn_args': ['--runtime-mode=release'], + 'ninja_targets': ['dart_sdk'], + 'out_dir': 'host_release', + }, + ], + }, + }; + + final config = PlatformConfig.fromJson(json); + final shard = config.getShard('host'); + + expect(shard.steps.length, 2); + expect(shard.steps[0], isA()); + expect(shard.steps[1], isA()); + + final rustStep = shard.steps[0] as RustStep; + expect(rustStep.targets, + ['aarch64-linux-android', 'x86_64-unknown-linux-gnu']); + + final gnStep = shard.steps[1] as GnNinjaStep; + expect(gnStep.outDir, 'host_release'); + }); + + test('parses compose_input', () { + final json = { + 'ios-release': { + 'steps': [ + { + 'type': 'gn_ninja', + 'gn_args': ['--ios', '--runtime-mode=release'], + 'ninja_targets': ['flutter_framework'], + 'out_dir': 'ios_release', + }, + ], + 'compose_input': 'ios-framework', + }, + }; + + final config = PlatformConfig.fromJson(json); + final shard = config.getShard('ios-release'); + + expect(shard.composeInput, 'ios-framework'); + }); + + test('getShard throws for unknown shard', () { + final config = PlatformConfig(shards: {}); + + expect( + () => config.getShard('nonexistent'), + throwsA(isA()), + ); + }); + }); + + group('BuildStep.fromJson', () { + test('parses gn_ninja type', () { + final json = { + 'type': 'gn_ninja', + 'gn_args': ['--android'], + 'ninja_targets': ['default'], + 'out_dir': 'out_dir', + }; + + final step = BuildStep.fromJson(json); + expect(step, isA()); + }); + + test('parses rust type', () { + final json = { + 'type': 'rust', + 'targets': ['x86_64-unknown-linux-gnu'], + }; + + final step = BuildStep.fromJson(json); + expect(step, isA()); + }); + + test('throws for unknown type', () { + final json = { + 'type': 'unknown', + }; + + expect( + () => BuildStep.fromJson(json), + throwsA(isA()), + ); + }); + }); + + group('GnNinjaStep', () { + test('fromJson parses all fields', () { + final json = { + 'type': 'gn_ninja', + 'gn_args': ['--android', '--runtime-mode=release'], + 'ninja_targets': ['default', 'gen_snapshot'], + 'out_dir': 'android_release', + }; + + final step = GnNinjaStep.fromJson(json); + + expect(step.gnArgs, ['--android', '--runtime-mode=release']); + expect(step.ninjaTargets, ['default', 'gen_snapshot']); + expect(step.outDir, 'android_release'); + }); + }); + + group('RustStep', () { + test('fromJson parses targets', () { + final json = { + 'type': 'rust', + 'targets': [ + 'aarch64-linux-android', + 'armv7-linux-androideabi', + 'x86_64-unknown-linux-gnu', + ], + }; + + final step = RustStep.fromJson(json); + + expect(step.targets, [ + 'aarch64-linux-android', + 'armv7-linux-androideabi', + 'x86_64-unknown-linux-gnu', + ]); + }); + }); + + group('ArtifactDef', () { + test('fromJson parses src and dst', () { + final json = { + 'src': 'zip_archives/artifacts.zip', + 'dst': 'flutter_infra/\$engine/artifacts.zip', + }; + + final artifact = ArtifactDef.fromJson(json); + + expect(artifact.src, 'zip_archives/artifacts.zip'); + expect(artifact.dst, 'flutter_infra/\$engine/artifacts.zip'); + expect(artifact.zip, false); + expect(artifact.contentHash, false); + }); + + test('fromJson parses zip flag', () { + final json = { + 'src': 'dart-sdk', + 'dst': 'flutter_infra/dart-sdk.zip', + 'zip': true, + }; + + final artifact = ArtifactDef.fromJson(json); + + expect(artifact.zip, true); + }); + + test('fromJson parses content_hash flag', () { + final json = { + 'src': 'dart-sdk', + 'dst': 'flutter_infra/dart-sdk.zip', + 'content_hash': true, + }; + + final artifact = ArtifactDef.fromJson(json); + + expect(artifact.contentHash, true); + }); + + test('fromJson parses all flags together', () { + final json = { + 'src': 'host_release/dart-sdk', + 'dst': 'flutter_infra/flutter/\$engine/dart-sdk-linux-x64.zip', + 'zip': true, + 'content_hash': true, + }; + + final artifact = ArtifactDef.fromJson(json); + + expect(artifact.src, 'host_release/dart-sdk'); + expect(artifact.dst, + 'flutter_infra/flutter/\$engine/dart-sdk-linux-x64.zip'); + expect(artifact.zip, true); + expect(artifact.contentHash, true); + }); + }); +} diff --git a/shorebird/ci/shard_runner/test/manifest_test.dart b/shorebird/ci/shard_runner/test/manifest_test.dart new file mode 100644 index 0000000000000..a5606ae9b2f54 --- /dev/null +++ b/shorebird/ci/shard_runner/test/manifest_test.dart @@ -0,0 +1,208 @@ +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; +import 'package:shard_runner/manifest.dart'; + +void main() { + group('generateManifest', () { + // Path to the ci/ directory where the template lives + // Tests run from shard_runner/, so go up one level to ci/ + final String configDir = p.normalize(p.join(Directory.current.path, '..')); + + test('generates valid YAML structure', () { + final manifest = generateManifest('abc123def456', configDir: configDir); + + expect(manifest, contains('flutter_engine_revision: abc123def456')); + expect(manifest, contains('storage_bucket: download.shorebird.dev')); + expect(manifest, contains('artifact_overrides:')); + }); + + test('includes Android release artifacts', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + // Android arm64 + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-arm64-release/artifacts.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-arm64-release/linux-x64.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-arm64-release/darwin-x64.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-arm64-release/windows-x64.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-arm64-release/symbols.zip')); + + // Android arm32 + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-arm-release/artifacts.zip')); + + // Android x64 + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip')); + }); + + test('includes Dart SDK for all platforms', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/dart-sdk-darwin-arm64.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/dart-sdk-darwin-x64.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/dart-sdk-linux-x64.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/dart-sdk-windows-x64.zip')); + }); + + test('includes Maven artifacts', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + // flutter_embedding_release + expect( + manifest, + contains( + r'download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom')); + expect( + manifest, + contains( + r'download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar')); + + // arm64_v8a_release + expect( + manifest, + contains( + r'download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom')); + expect( + manifest, + contains( + r'download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar')); + + // armeabi_v7a_release + expect( + manifest, + contains( + r'download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom')); + + // x86_64_release + expect( + manifest, + contains( + r'download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom')); + }); + + test('includes iOS release artifacts', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/ios-release/artifacts.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/ios-release/Flutter.framework.dSYM.zip')); + }); + + test('includes Linux release artifacts', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/linux-x64/artifacts.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/linux-x64-release/linux-x64-flutter-gtk.zip')); + }); + + test('includes macOS release artifacts', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/darwin-x64-release/artifacts.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/darwin-x64-release/framework.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/darwin-x64-release/gen_snapshot.zip')); + }); + + test('includes Windows release artifacts', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/windows-x64/artifacts.zip')); + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/windows-x64-release/windows-x64-flutter.zip')); + }); + + test('includes engine_stamp.json', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect(manifest, + contains(r'flutter_infra_release/flutter/$engine/engine_stamp.json')); + }); + + test('includes flutter_patched_sdk_product', () { + final manifest = generateManifest('test-hash', configDir: configDir); + + expect( + manifest, + contains( + r'flutter_infra_release/flutter/$engine/flutter_patched_sdk_product.zip')); + }); + + test('uses \$engine placeholder (not hardcoded hash)', () { + final manifest = generateManifest('abc123', configDir: configDir); + + // The flutter_engine_revision should use the actual hash + expect(manifest, contains('flutter_engine_revision: abc123')); + + // But artifact paths should use $engine placeholder + expect(manifest, contains(r'$engine')); + // Should NOT contain the actual hash in artifact paths + expect(manifest.split('flutter_engine_revision:')[1], + isNot(contains('abc123/'))); + }); + + test('throws when template file not found', () { + expect( + () => generateManifest('test-hash', configDir: '/nonexistent'), + throwsA(isA()), + ); + }); + }); +} diff --git a/shorebird/ci/shards/linux.json b/shorebird/ci/shards/linux.json new file mode 100644 index 0000000000000..51b0aa86f0b6b --- /dev/null +++ b/shorebird/ci/shards/linux.json @@ -0,0 +1,104 @@ +{ + "android-arm64": { + "steps": [ + { + "type": "rust", + "targets": ["aarch64-linux-android"] + }, + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=arm64", "--runtime-mode=release"], + "ninja_targets": ["default", "gen_snapshot"], + "out_dir": "android_release_arm64" + } + ], + "artifacts": [ + {"src": "zip_archives/android-arm64-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/artifacts.zip"}, + {"src": "zip_archives/android-arm64-release/linux-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/linux-x64.zip"}, + {"src": "zip_archives/android-arm64-release/symbols.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/symbols.zip"}, + {"src": "arm64_v8a_release.pom", "dst": "download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom"}, + {"src": "arm64_v8a_release.jar", "dst": "download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar"}, + {"src": "arm64_v8a_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.maven-metadata.xml"} + ] + }, + "android-arm32": { + "steps": [ + { + "type": "rust", + "targets": ["armv7-linux-androideabi"] + }, + { + "type": "gn_ninja", + "gn_args": ["--android", "--runtime-mode=release"], + "ninja_targets": ["default", "gen_snapshot"], + "out_dir": "android_release" + } + ], + "artifacts": [ + {"src": "zip_archives/android-arm-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/artifacts.zip"}, + {"src": "zip_archives/android-arm-release/linux-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/linux-x64.zip"}, + {"src": "zip_archives/android-arm-release/symbols.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/symbols.zip"}, + {"src": "armeabi_v7a_release.pom", "dst": "download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom"}, + {"src": "armeabi_v7a_release.jar", "dst": "download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.jar"}, + {"src": "armeabi_v7a_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.maven-metadata.xml"}, + {"src": "flutter_embedding_release.pom", "dst": "download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom"}, + {"src": "flutter_embedding_release.jar", "dst": "download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar"}, + {"src": "flutter_embedding_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.maven-metadata.xml"} + ] + }, + "android-x64": { + "steps": [ + { + "type": "rust", + "targets": ["x86_64-linux-android"] + }, + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=x64", "--runtime-mode=release"], + "ninja_targets": ["default", "gen_snapshot"], + "out_dir": "android_release_x64" + } + ], + "artifacts": [ + {"src": "zip_archives/android-x64-release/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip"}, + {"src": "zip_archives/android-x64-release/linux-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/linux-x64.zip"}, + {"src": "zip_archives/android-x64-release/symbols.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/symbols.zip"}, + {"src": "x86_64_release.pom", "dst": "download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom"}, + {"src": "x86_64_release.jar", "dst": "download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.jar"}, + {"src": "x86_64_release.maven-metadata.xml", "dst": "download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.maven-metadata.xml"} + ] + }, + "host": { + "steps": [ + { + "type": "rust", + "targets": [ + "armv7-linux-androideabi", + "aarch64-linux-android", + "i686-linux-android", + "x86_64-linux-android", + "x86_64-unknown-linux-gnu" + ] + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--no-prebuilt-dart-sdk"], + "ninja_targets": ["dart_sdk", "flutter/shell/platform/linux:flutter_gtk", "flutter/build/archives:flutter_patched_sdk"], + "out_dir": "host_release" + }, + { + "type": "gn_ninja", + "gn_args": ["--no-prebuilt-dart-sdk"], + "ninja_targets": ["flutter/build/archives:artifacts"], + "out_dir": "host_debug" + } + ], + "artifacts": [ + {"src": "host_release/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-linux-x64.zip", "zip": true, "content_hash": true}, + {"src": "host_release/zip_archives/flutter_patched_sdk_product.zip", "dst": "flutter_infra_release/flutter/$engine/flutter_patched_sdk_product.zip"}, + {"src": "host_release/zip_archives/linux-x64-release/linux-x64-flutter-gtk.zip", "dst": "flutter_infra_release/flutter/$engine/linux-x64-release/linux-x64-flutter-gtk.zip"}, + {"src": "host_debug/zip_archives/linux-x64/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/linux-x64/artifacts.zip"}, + {"src": "host_release/aot_tools/aot-tools.dill", "dst": "shorebird/$engine/aot-tools.dill"} + ] + } +} diff --git a/shorebird/ci/shards/macos.json b/shorebird/ci/shards/macos.json new file mode 100644 index 0000000000000..db68dc3e1a2f8 --- /dev/null +++ b/shorebird/ci/shards/macos.json @@ -0,0 +1,169 @@ +{ + "android": { + "steps": [ + { + "type": "rust", + "targets": [ + "aarch64-apple-ios", + "x86_64-apple-ios", + "aarch64-apple-darwin", + "x86_64-apple-darwin" + ] + }, + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=arm64", "--runtime-mode=release", "--gn-args=host_cpu=\"x64\""], + "ninja_targets": ["flutter/shell/platform/android:gen_snapshot"], + "out_dir": "android_release_arm64" + }, + { + "type": "gn_ninja", + "gn_args": ["--android", "--runtime-mode=release", "--gn-args=host_cpu=\"x64\""], + "ninja_targets": ["flutter/shell/platform/android:gen_snapshot"], + "out_dir": "android_release" + }, + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=x64", "--runtime-mode=release", "--gn-args=host_cpu=\"x64\""], + "ninja_targets": ["flutter/shell/platform/android:gen_snapshot"], + "out_dir": "android_release_x64" + } + ], + "artifacts": [ + {"src": "android_release_arm64/zip_archives/android-arm64-release/darwin-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/darwin-x64.zip"}, + {"src": "android_release/zip_archives/android-arm-release/darwin-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/darwin-x64.zip"}, + {"src": "android_release_x64/zip_archives/android-x64-release/darwin-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/darwin-x64.zip"} + ] + }, + "ios-release": { + "steps": [ + { + "type": "rust", + "targets": ["aarch64-apple-ios"] + }, + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=release", "--gn-arg=shorebird_runtime=true"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_release" + } + ], + "compose_input": "ios-framework" + }, + "ios-release-ext": { + "steps": [ + { + "type": "rust", + "targets": ["aarch64-apple-ios"] + }, + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=release", "--darwin-extension-safe", "--xcode-symlinks", "--gn-arg=shorebird_runtime=true"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_release_extension_safe" + } + ], + "compose_input": "ios-framework" + }, + "ios-sim-x64": { + "steps": [ + { + "type": "rust", + "targets": ["x86_64-apple-ios"] + }, + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=debug", "--simulator"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_debug_sim" + } + ], + "compose_input": "ios-framework" + }, + "ios-sim-x64-ext": { + "steps": [ + { + "type": "rust", + "targets": ["x86_64-apple-ios"] + }, + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=debug", "--darwin-extension-safe", "--simulator"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_debug_sim_extension_safe" + } + ], + "compose_input": "ios-framework" + }, + "ios-sim-arm64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=debug", "--simulator", "--simulator-cpu=arm64"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_debug_sim_arm64" + } + ], + "compose_input": "ios-framework" + }, + "ios-sim-arm64-ext": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--ios", "--runtime-mode=debug", "--darwin-extension-safe", "--simulator", "--simulator-cpu=arm64"], + "ninja_targets": ["flutter/shell/platform/darwin/ios:flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins"], + "out_dir": "ios_debug_sim_arm64_extension_safe" + } + ], + "compose_input": "ios-framework" + }, + "mac-arm64": { + "steps": [ + { + "type": "rust", + "targets": ["aarch64-apple-darwin"] + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--mac-cpu=arm64", "--no-prebuilt-dart-sdk"], + "ninja_targets": ["dart_sdk"], + "out_dir": "host_release_arm64" + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--mac", "--mac-cpu=arm64"], + "ninja_targets": ["flutter/shell/platform/darwin/macos:zip_macos_flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins", "flutter/build/archives:artifacts"], + "out_dir": "mac_release_arm64" + } + ], + "compose_input": "macos-framework", + "artifacts": [ + {"src": "host_release_arm64/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-darwin-arm64.zip", "zip": true, "content_hash": true} + ] + }, + "mac-x64": { + "steps": [ + { + "type": "rust", + "targets": ["x86_64-apple-darwin"] + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--mac-cpu=x64", "--no-prebuilt-dart-sdk"], + "ninja_targets": ["dart_sdk"], + "out_dir": "host_release" + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--mac", "--mac-cpu=x64"], + "ninja_targets": ["flutter/shell/platform/darwin/macos:zip_macos_flutter_framework", "flutter/lib/snapshot:generate_snapshot_bins", "flutter/build/archives:artifacts"], + "out_dir": "mac_release" + } + ], + "compose_input": "macos-framework", + "artifacts": [ + {"src": "host_release/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-darwin-x64.zip", "zip": true, "content_hash": true}, + {"src": "engine_stamp.json", "dst": "flutter_infra_release/flutter/$engine/engine_stamp.json"} + ] + } +} diff --git a/shorebird/ci/shards/windows.json b/shorebird/ci/shards/windows.json new file mode 100644 index 0000000000000..5775e5d0a1ad0 --- /dev/null +++ b/shorebird/ci/shards/windows.json @@ -0,0 +1,66 @@ +{ + "android-arm64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=arm64", "--runtime-mode=release"], + "ninja_targets": ["archive_win_gen_snapshot"], + "out_dir": "android_release_arm64" + } + ], + "artifacts": [ + {"src": "zip_archives/android-arm64-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm64-release/windows-x64.zip"} + ] + }, + "android-arm32": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--android", "--runtime-mode=release"], + "ninja_targets": ["archive_win_gen_snapshot"], + "out_dir": "android_release" + } + ], + "artifacts": [ + {"src": "zip_archives/android-arm-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-arm-release/windows-x64.zip"} + ] + }, + "android-x64": { + "steps": [ + { + "type": "gn_ninja", + "gn_args": ["--android", "--android-cpu=x64", "--runtime-mode=release"], + "ninja_targets": ["archive_win_gen_snapshot"], + "out_dir": "android_release_x64" + } + ], + "artifacts": [ + {"src": "zip_archives/android-x64-release/windows-x64.zip", "dst": "flutter_infra_release/flutter/$engine/android-x64-release/windows-x64.zip"} + ] + }, + "host": { + "steps": [ + { + "type": "rust", + "targets": ["x86_64-pc-windows-msvc"] + }, + { + "type": "gn_ninja", + "gn_args": ["--runtime-mode=release", "--no-prebuilt-dart-sdk"], + "ninja_targets": ["dart_sdk", "flutter/build/archives:windows_flutter", "gen_snapshot", "windows", "flutter/build/archives:artifacts"], + "out_dir": "host_release" + }, + { + "type": "gn_ninja", + "gn_args": ["--no-prebuilt-dart-sdk"], + "ninja_targets": ["flutter/build/archives:artifacts"], + "out_dir": "host_debug" + } + ], + "artifacts": [ + {"src": "host_release/dart-sdk", "dst": "flutter_infra_release/flutter/$engine/dart-sdk-windows-x64.zip", "zip": true, "content_hash": true}, + {"src": "host_release/zip_archives/windows-x64-release/windows-x64-flutter.zip", "dst": "flutter_infra_release/flutter/$engine/windows-x64-release/windows-x64-flutter.zip"}, + {"src": "host_debug/zip_archives/windows-x64/artifacts.zip", "dst": "flutter_infra_release/flutter/$engine/windows-x64/artifacts.zip"} + ] + } +} diff --git a/shorebird/ci/win_build_and_upload.sh b/shorebird/ci/win_build_and_upload.sh new file mode 100755 index 0000000000000..bbda403ff2c67 --- /dev/null +++ b/shorebird/ci/win_build_and_upload.sh @@ -0,0 +1,32 @@ +#!/bin/bash -e + +# Usage: +# ./win_build_and_upload.sh flutter_root engine_hash +# +# This is the main entrypoint for building and uploading Windows engine artifacts. +# It is called from the _build_engine repository's CI scripts. + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 flutter_root engine_hash" + exit 1 +fi + +FLUTTER_ROOT=$1 +ENGINE_HASH=$2 +ENGINE_ROOT=$FLUTTER_ROOT/engine + +# Get the absolute path to the directory of this script. +SCRIPT_DIR=$(cd $(dirname $0) && pwd) + +echo "Building engine at $ENGINE_ROOT and uploading to gs://download.shorebird.dev" + +cd $SCRIPT_DIR + +# Run the setup script. +./internal/win_setup.sh + +# Then run the build. +./internal/win_build.sh $ENGINE_ROOT + +# Copy Shorebird engine artifacts to Google Cloud Storage. +./internal/win_upload.sh $ENGINE_ROOT $ENGINE_HASH diff --git a/shorebird/docs/BUILDING.md b/shorebird/docs/BUILDING.md new file mode 100644 index 0000000000000..dfd3c655a1324 --- /dev/null +++ b/shorebird/docs/BUILDING.md @@ -0,0 +1,44 @@ +# Building the Shorebird Engine Locally + +This document explains how to build the Shorebird engine locally for different platforms. + +All commands assume you are running from the `engine/src` directory. + +## Prerequisites + +- Follow the standard Flutter engine setup instructions +- Ensure you have the necessary toolchains installed for your target platform + +## macOS + +### iOS (arm64) + +```bash +./flutter/tools/gn --no-rbe --no-enable-unittests --runtime-mode=release --ios --gn-arg='shorebird_runtime=true' +ninja -C out/ios_release flutter/shell/platform/darwin/ios:flutter_framework flutter/lib/snapshot:generate_snapshot_bins +``` + +### Android arm64 + +```bash +./flutter/tools/gn --no-rbe --no-enable-unittests --android --android-cpu=arm64 --runtime-mode=release --gn-args='host_cpu="x64"' +ninja -C out/android_release_arm64 flutter/shell/platform/android:gen_snapshot +``` + +## Windows + +### Android arm64 + +```bash +./flutter/tools/gn --no-rbe --no-enable-unittests --android --android-cpu=arm64 --runtime-mode=release +ninja -C out/android_release_arm64 archive_win_gen_snapshot +``` + +## Linux + +### Android arm64 + +```bash +./flutter/tools/gn --no-rbe --no-enable-unittests --android --android-cpu=arm64 --runtime-mode=release +ninja -C out/android_release_arm64 default gen_snapshot +```