Skip to content

fix: gRPC retry on PROTOCOL_ERROR for IDB client stability #30

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mattleibow merged 13 commits into from
Feb 27, 2026
Merged

fix: gRPC retry on PROTOCOL_ERROR for IDB client stability #30

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
eab350d
fix: add gRPC retry on PROTOCOL_ERROR for IDB client stability
mattleibow Feb 26, 2026
c398b6d
test: remove SkipIfNotReady, only skip for platform or known bugs
mattleibow Feb 26, 2026
c988131
fix: skip IDB tests when simulator fails to boot on CI
mattleibow Feb 26, 2026
79ada6c
ci: retry CI run
mattleibow Feb 26, 2026
58afff1
fix: use retry wrapper for ScreenshotAsync before simctl fallback
mattleibow Feb 26, 2026
aa5f5b4
Rename SkipIfNotSupported to SkipIfNotReady
mattleibow Feb 27, 2026
8ac6540
ci: switch to EnricoMi/publish-unit-test-result-action for PR comments
mattleibow Feb 27, 2026
3291abb
revert: restore dorny/test-reporter for test summaries
mattleibow Feb 27, 2026
2d74da4
test: remove CI skips and increase timeouts for slow runners
mattleibow Feb 27, 2026
5b55326
test: use reliable SpringBoard predicate for log filter tests
mattleibow Feb 27, 2026
0d06955
test: fix plain text log assertion and increase xcdevice timeout
mattleibow Feb 27, 2026
0cd6242
test: skip IDB tests when idb_companion fails to start
mattleibow Feb 27, 2026
bffa902
refactor: single-line retry calls, restore SkippableFact and Maps pre…
mattleibow Feb 27, 2026
File filter

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions AppleDev.FbIdb.Test/IdbClientIntegrationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public async Task InitializeAsync()
await _simCtl.BootAsync(SimulatorUdid!);

// Wait for it to be fully booted
var booted = await _simCtl.WaitForBootedAsync(SimulatorUdid!, TimeSpan.FromSeconds(120));
var booted = await _simCtl.WaitForBootedAsync(SimulatorUdid!, TimeSpan.FromSeconds(300));
if (!booted)
{
SkipReason = "Simulator failed to boot within timeout";
Expand Down Expand Up @@ -127,11 +127,20 @@ private async Task<IdbClient> CreateConnectedClientAsync()
new IdbCompanionOptions
{
VerboseLogging = true,
StartupTimeout = TimeSpan.FromSeconds(60)
StartupTimeout = TimeSpan.FromSeconds(120)
},
new XUnitLogger<IdbClient>(_testOutputHelper));

await client.ConnectAsync();
try
{
await client.ConnectAsync();
}
catch (TimeoutException ex)
{
await client.DisposeAsync();
Skip.If(true, $"idb_companion failed to start: {ex.Message}");
}

_testOutputHelper.WriteLine($"Connected to IDB companion, IsConnected={client.IsConnected}");
return client;
}
Expand Down Expand Up @@ -251,6 +260,10 @@ public async Task FocusAsync_FocusesSimulator()
public async Task ListCrashLogsAsync_ReturnsCrashLogs()
{
SkipIfNotReady();
// idb_companion v1.1.8 crashes server-side with uncaught NSException in crash_list.
// See: https://github.com/facebook/idb/issues/868
// Unlike PROTOCOL_ERROR, this is a server crash that retries cannot fix.
Skip.If(true, "idb_companion v1.1.8 crash_list is broken (facebook/idb#868)");

await using var client = await CreateConnectedClientAsync();

Expand Down
74 changes: 54 additions & 20 deletions AppleDev.FbIdb/IdbClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,52 @@ private CallOptions GetCallOptions(CancellationToken cancellationToken)

private Metadata GetMetadata() => new();

private const int MaxRetries = 3;

/// <summary>
/// Executes a gRPC call with retry on PROTOCOL_ERROR.
/// When idb_companion sends invalid HTTP/2 data, the connection is
/// poisoned. This helper reconnects and retries the call.
/// </summary>
private async Task<T> WithRetryOnProtocolErrorAsync<T>(
Func<CompanionService.CompanionServiceClient, CallOptions, Task<T>> call,
CancellationToken cancellationToken)
{
await EnsureConnectedAsync(cancellationToken).ConfigureAwait(false);

for (int attempt = 0; ; attempt++)
{
try
{
return await call(GetClient(), GetCallOptions(cancellationToken)).ConfigureAwait(false);
}
catch (RpcException ex) when (
attempt < MaxRetries &&
ex.StatusCode == StatusCode.Internal &&
ex.Message.Contains("PROTOCOL_ERROR"))
{
_logger.LogDebug(
"gRPC call failed with PROTOCOL_ERROR (attempt {Attempt}/{Max}), reconnecting",
Comment thread
mattleibow marked this conversation as resolved.
Resolved
attempt + 1, MaxRetries);

// Tear down the poisoned connection and create a fresh one
await ReconnectAsync(cancellationToken).ConfigureAwait(false);
}
}
Comment thread
mattleibow marked this conversation as resolved.
Resolved
}

private async Task ReconnectAsync(CancellationToken cancellationToken)
{
if (_channel is not null)
{
try { _channel.Dispose(); } catch { /* best effort */ }
_channel = null;
_client = null;
}

await EnsureConnectedAsync(cancellationToken).ConfigureAwait(false);
}

#region Connection & Management

/// <inheritdoc />
Expand All @@ -193,10 +239,8 @@ public async Task<CompanionInfo> ConnectAsync(CancellationToken cancellationToke
/// <inheritdoc />
public async Task<TargetDescription> DescribeAsync(bool fetchDiagnostics = false, CancellationToken cancellationToken = default)
{
await EnsureConnectedAsync(cancellationToken).ConfigureAwait(false);

var request = new TargetDescriptionRequest { FetchDiagnostics = fetchDiagnostics };
var response = await GetClient().describeAsync(request, GetCallOptions(cancellationToken)).ConfigureAwait(false);
var response = await WithRetryOnProtocolErrorAsync((client, opts) => client.describeAsync(request, opts).ResponseAsync, cancellationToken).ConfigureAwait(false);

var desc = response.TargetDescription;
return new TargetDescription
Expand Down Expand Up @@ -402,10 +446,8 @@ public async Task TerminateAsync(string bundleId, CancellationToken cancellation
/// <inheritdoc />
public async Task<IReadOnlyList<InstalledApp>> ListAppsAsync(bool suppressProcessState = false, CancellationToken cancellationToken = default)
{
await EnsureConnectedAsync(cancellationToken).ConfigureAwait(false);

var request = new ListAppsRequest { SuppressProcessState = suppressProcessState };
var response = await GetClient().list_appsAsync(request, GetCallOptions(cancellationToken)).ConfigureAwait(false);
var response = await WithRetryOnProtocolErrorAsync((client, opts) => client.list_appsAsync(request, opts).ResponseAsync, cancellationToken).ConfigureAwait(false);

return response.Apps.Select(app => new InstalledApp
{
Expand All @@ -430,12 +472,10 @@ public async Task<IReadOnlyList<InstalledApp>> ListAppsAsync(bool suppressProces
/// </remarks>
public async Task<Screenshot> ScreenshotAsync(CancellationToken cancellationToken = default)
{
await EnsureConnectedAsync(cancellationToken).ConfigureAwait(false);

try
{
var request = new ScreenshotRequest();
var response = await GetClient().screenshotAsync(request, GetCallOptions(cancellationToken)).ConfigureAwait(false);
var response = await WithRetryOnProtocolErrorAsync((client, opts) => client.screenshotAsync(request, opts).ResponseAsync, cancellationToken).ConfigureAwait(false);

return new Screenshot
{
Expand All @@ -446,8 +486,8 @@ public async Task<Screenshot> ScreenshotAsync(CancellationToken cancellationToke
catch (Grpc.Core.RpcException ex) when (ex.StatusCode == Grpc.Core.StatusCode.Internal &&
ex.Message.Contains("PROTOCOL_ERROR"))
{
// Fallback to simctl for simulators due to .NET/Swift gRPC HTTP/2 incompatibility
_logger.LogDebug("gRPC screenshot failed with PROTOCOL_ERROR, falling back to simctl");
// Fallback to simctl for simulators if all gRPC retries exhausted
_logger.LogDebug("gRPC screenshot failed with PROTOCOL_ERROR after retries, falling back to simctl");
return await ScreenshotViaSimctlAsync(cancellationToken).ConfigureAwait(false);
}
}
Expand Down Expand Up @@ -675,10 +715,8 @@ await call.RequestStream.WriteAsync(new HIDEvent
/// <inheritdoc />
public async Task FocusAsync(CancellationToken cancellationToken = default)
{
await EnsureConnectedAsync(cancellationToken).ConfigureAwait(false);

var request = new FocusRequest();
await GetClient().focusAsync(request, GetCallOptions(cancellationToken)).ConfigureAwait(false);
await WithRetryOnProtocolErrorAsync((client, opts) => client.focusAsync(request, opts).ResponseAsync, cancellationToken).ConfigureAwait(false);
}

#endregion
Expand Down Expand Up @@ -782,8 +820,6 @@ public Task SetLocationAsync(double latitude, double longitude, CancellationToke
/// <inheritdoc />
public async Task SetLocationAsync(GeoLocation location, CancellationToken cancellationToken = default)
{
await EnsureConnectedAsync(cancellationToken).ConfigureAwait(false);

var request = new SetLocationRequest
{
Location = new Location
Expand All @@ -793,7 +829,7 @@ public async Task SetLocationAsync(GeoLocation location, CancellationToken cance
}
};

await GetClient().set_locationAsync(request, GetCallOptions(cancellationToken)).ConfigureAwait(false);
await WithRetryOnProtocolErrorAsync((client, opts) => client.set_locationAsync(request, opts).ResponseAsync, cancellationToken).ConfigureAwait(false);
}

#endregion
Expand Down Expand Up @@ -1112,10 +1148,8 @@ public async Task<IReadOnlyList<CrashLogInfo>> DeleteCrashLogsAsync(string? bund
/// <inheritdoc />
public async Task<IReadOnlyList<TestBundle>> ListTestBundlesAsync(CancellationToken cancellationToken = default)
{
await EnsureConnectedAsync(cancellationToken).ConfigureAwait(false);

var request = new XctestListBundlesRequest();
var response = await GetClient().xctest_list_bundlesAsync(request, GetCallOptions(cancellationToken)).ConfigureAwait(false);
var response = await WithRetryOnProtocolErrorAsync((client, opts) => client.xctest_list_bundlesAsync(request, opts).ResponseAsync, cancellationToken).ConfigureAwait(false);

return response.Bundles.Select(b => new TestBundle
{
Expand Down
53 changes: 12 additions & 41 deletions AppleDev.Test/SimCtlComplexTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,6 @@ public SimCtlComplexTests(ITestOutputHelper testOutputHelper)
_testSimName = $"Test-iPhone-{DateTime.Now:yyyyMMdd-HHmmss}";
}

private static bool IsRunningInCI()
{
// Check for common CI environment variables
return !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("CI")) ||
!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("GITHUB_ACTIONS")) ||
!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TF_BUILD"));
}

public async Task InitializeAsync()
{
var deviceTypes = await _simCtl.GetSimulatorGroupsAsync();
Expand All @@ -51,8 +43,8 @@ async Task BootAndWaitAsync()
var bootSuccess = await _simCtl.BootAsync(_testSimName);
Assert.True(bootSuccess, "Failed to boot the simulator");

// Wait for boot to complete with reduced timeout
var waitSuccess = await _simCtl.WaitForBootedAsync(_testSimName, TimeSpan.FromSeconds(60));
// Wait for boot to complete
var waitSuccess = await _simCtl.WaitForBootedAsync(_testSimName, TimeSpan.FromSeconds(300));
Assert.True(waitSuccess, "Failed to wait for the simulator to boot");

_isBooted = true;
Expand All @@ -73,8 +65,6 @@ public async Task DisposeAsync()
[SkippableFact]
public async Task CreateSimulator_ShouldSucceed()
{
Skip.If(IsRunningInCI(), "SimCtl complex tests are skipped in CI due to timeouts");

// Verify the simulator was created
var sims = await _simCtl.GetSimulatorsAsync(availableOnly: false);
var createdSim = sims.FirstOrDefault(s => s.Name == _testSimName);
Expand All @@ -85,8 +75,6 @@ public async Task CreateSimulator_ShouldSucceed()
[SkippableFact]
public async Task BootAndShutdownSimulator_ShouldSucceed()
{
Skip.If(IsRunningInCI(), "SimCtl complex tests are skipped in CI due to timeouts");

await BootAndWaitAsync();

// Verify the simulator is booted
Expand Down Expand Up @@ -115,8 +103,6 @@ public async Task BootAndShutdownSimulator_ShouldSucceed()
[SkippableFact]
public async Task DeleteSimulator_ShouldSucceed()
{
Skip.If(IsRunningInCI(), "SimCtl complex tests are skipped in CI due to timeouts");

// Get the created simulator
var sims = await _simCtl.GetSimulatorsAsync(availableOnly: false);
var createdSim = sims.FirstOrDefault(s => s.Name == _testSimName);
Expand All @@ -136,8 +122,6 @@ public async Task DeleteSimulator_ShouldSucceed()
[SkippableFact]
public async Task EraseSimulator_ShouldSucceed()
{
Skip.If(IsRunningInCI(), "SimCtl complex tests are skipped in CI due to timeouts");

// Get the created simulator
var sims = await _simCtl.GetSimulatorsAsync(availableOnly: false);
var testSim = sims.FirstOrDefault(s => s.Name == _testSimName);
Expand All @@ -157,8 +141,6 @@ public async Task EraseSimulator_ShouldSucceed()
[SkippableFact]
public async Task SimulatorLifecycle_CreateBootShutdownDelete_ShouldSucceed()
{
Skip.If(IsRunningInCI(), "SimCtl complex tests are skipped in CI due to timeouts");

// Get the created simulator
var sims = await _simCtl.GetSimulatorsAsync(availableOnly: false);
var testSim = sims.FirstOrDefault(s => s.Name == _testSimName);
Expand Down Expand Up @@ -204,8 +186,6 @@ public async Task SimulatorLifecycle_CreateBootShutdownDelete_ShouldSucceed()
[SkippableFact]
public async Task GetAppsAsync_ShouldReturnAppsWithCorrectProperties()
{
Skip.If(IsRunningInCI(), "SimCtl complex tests are skipped in CI due to timeouts");

await BootAndWaitAsync();

// Get the installed apps
Expand Down Expand Up @@ -272,8 +252,6 @@ public async Task GetAppsAsync_ShouldReturnAppsWithCorrectProperties()
[SkippableFact]
public async Task GetLogsAsync_ShouldReturnLogOutput()
{
Skip.If(IsRunningInCI(), "SimCtl complex tests are skipped in CI due to timeouts");

await BootAndWaitAsync();

// Get logs - should return some log data
Expand All @@ -285,8 +263,6 @@ public async Task GetLogsAsync_ShouldReturnLogOutput()
[SkippableFact]
public async Task GetLogsAsync_WithStart_ShouldReturnLogOutput()
{
Skip.If(IsRunningInCI(), "SimCtl complex tests are skipped in CI due to timeouts");

await BootAndWaitAsync();

// Get logs - should return some log data
Expand All @@ -298,14 +274,13 @@ public async Task GetLogsAsync_WithStart_ShouldReturnLogOutput()
[SkippableFact]
public async Task GetLogsAsync_WithPredicate_ShouldFilterLogs()
{
Skip.If(IsRunningInCI(), "SimCtl complex tests are skipped in CI due to timeouts");

await BootAndWaitAsync();

// Reduced delay - simulator boot already generated plenty of logs
await Task.Delay(1000);
// Launch Maps so it generates logs we can filter for
await _simCtl.LaunchAppAsync(_testSimName, "com.apple.Maps");
await Task.Delay(3000);

// Get logs with a predicate filter
// Get logs with a predicate filter for Maps
var logs = await _simCtl.GetLogsAsync(_testSimName, predicate: "senderImagePath contains 'Maps'");
Assert.NotNull(logs);
Assert.NotEmpty(logs);
Expand All @@ -329,23 +304,21 @@ public async Task GetLogsAsync_WithPredicate_ShouldFilterLogs()
[SkippableFact]
public async Task GetLogsPlainAsync_WithPredicate_ShouldFilterLogs()
{
Skip.If(IsRunningInCI(), "SimCtl complex tests are skipped in CI due to timeouts");

await BootAndWaitAsync();

// Reduced delay - simulator boot already generated plenty of logs
await Task.Delay(1000);
// Launch Maps so it generates logs we can filter for
await _simCtl.LaunchAppAsync(_testSimName, "com.apple.Maps");
await Task.Delay(3000);

// Get logs with a predicate filter
// Get logs with a predicate filter for Maps
var logs = await _simCtl.GetLogsPlainAsync(_testSimName, predicate: "senderImagePath contains 'Maps'");
Assert.NotNull(logs);
Assert.NotEmpty(logs);

try
{
Assert.Contains(logs, log => log.Contains("[com.apple.Maps:GeneralMapsWidget]"));
Assert.DoesNotContain(logs, log => log.Contains("Clock"));

Assert.Contains(logs, log => log.Contains("Maps"));
Assert.DoesNotContain(logs, log => log.Contains("ClockKit"));
}
catch
{
Expand All @@ -361,8 +334,6 @@ public async Task GetLogsPlainAsync_WithPredicate_ShouldFilterLogs()
[SkippableFact]
public async Task CollectLogsAsync_ShouldSucceed()
{
Skip.If(IsRunningInCI(), "SimCtl complex tests are skipped in CI due to timeouts");

await BootAndWaitAsync();

// Create a unique output path for the test
Expand Down
2 changes: 1 addition & 1 deletion AppleDev.Test/XCDeviceTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public async Task GetAnyDevicesWithTimeout(int timeoutSeconds)
[Fact]
public async Task ObserveAnyDevices()
{
var ct = new CancellationTokenSource(TimeSpan.FromSeconds(10));
var ct = new CancellationTokenSource(TimeSpan.FromSeconds(30));

var tcs = new TaskCompletionSource<bool>();

Expand Down