fix(core): keep the IPC connection while a half-written frame waits on a suspended host
Windows Modern Standby suspends the app while the Helper's Core keeps running, so a frame that was half written when the host stopped draining timed out and closed the connection, which ended the Core with it. A timed-out write on a half-written frame is now resumed for as long as the stall lasts, and only a hard error still closes. go-winio reports the expiry as its own ErrTimeout, so the check goes through Timeout(). Changelog: Keep the core running while Windows sleeps with the app suspended
This commit is contained in:
+11
-2
@@ -162,8 +162,17 @@ leaving a repo-wide policy as a comment reaches only the reader of that one file
|
||||
`logDeliveryError`, which writes to stderr and latches until a frame gets through or the next connection is installed.
|
||||
A write that fails without putting a byte on the wire — host backpressure hitting `ipcWriteTimeout`, or a payload above
|
||||
`maxIPCFrameSize` — drops that one frame and keeps the connection: the stream is still framed correctly, and tearing it
|
||||
down here ends the read loop, and with it the Core process. Only a half-written frame desynchronizes the stream, and
|
||||
that is the one case `send` closes on.
|
||||
down here ends the read loop, and with it the Core process. A half-written frame whose write merely timed out is
|
||||
resumed for as long as the stall lasts: Windows Modern Standby suspends the app while the Helper's Core keeps running,
|
||||
so the host can stop draining for hours and still come back, and go-winio reports the expiry as its own `ErrTimeout`
|
||||
rather than `os.ErrDeadlineExceeded`, which is why `send` checks `Timeout()`. The wait has no cap on purpose, and
|
||||
`send` holds `writeMu` throughout, so every other frame — method responses and the single batcher goroutine behind
|
||||
the event queues — waits behind the stalled one. Memory is bounded by the queues; what gives is delivery: the state
|
||||
queue fills and `enqueueState` starts dropping, which is the case the `UpdatingAction` sweep above exists for. Nothing
|
||||
on the Dart side restarts the Core over a stall: `CoreRpcClient` times each pending request out on its own and hands
|
||||
the caller `null` (a `no_response` exception for message methods), and the sweep clears core-scope updating state
|
||||
minutes later. Only a half-written frame that fails outright desynchronizes the stream, and that is the one case
|
||||
`send` closes on.
|
||||
- Core method handlers in `core/hub.go` are synchronous. Anything that must not block the dispatcher is spawned by
|
||||
`safeGo`/`safeGoDetached` in `core/method.go`, which recover; a bare `go` in a handler puts a panic outside every
|
||||
recovery and kills the process, which on Android is the whole application. The `//export` entry points in
|
||||
|
||||
+74
-5
@@ -28,6 +28,8 @@ type fakeConn struct {
|
||||
writeErrTimes int
|
||||
closed bool
|
||||
deadlines int
|
||||
deadlineErr error
|
||||
deadlineOK int
|
||||
}
|
||||
|
||||
func (fake *fakeConn) Read(p []byte) (int, error) {
|
||||
@@ -86,6 +88,9 @@ func (fake *fakeConn) SetWriteDeadline(time.Time) error {
|
||||
fake.mu.Lock()
|
||||
defer fake.mu.Unlock()
|
||||
fake.deadlines++
|
||||
if fake.deadlineErr != nil && fake.deadlines > fake.deadlineOK {
|
||||
return fake.deadlineErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -469,18 +474,82 @@ func TestSendResumesAFrameThatStalledOnTheWriteDeadline(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendDropsTheConnectionWhenAStalledFrameNeverDrains(t *testing.T) {
|
||||
fake := &fakeConn{writeErr: os.ErrDeadlineExceeded, writeErrAfter: 2}
|
||||
func TestSendWaitsOutAHostSuspendedForManyDeadlineWindows(t *testing.T) {
|
||||
const windows = 360
|
||||
fake := &fakeConn{writeErr: os.ErrDeadlineExceeded, writeErrAfter: 2, writeErrTimes: windows}
|
||||
previous := swapConn(fake)
|
||||
defer swapConn(previous)
|
||||
|
||||
send([]byte("{}"))
|
||||
|
||||
if fake.isClosed() {
|
||||
t.Fatal("a host asleep for an hour of deadline windows had its connection closed, and with it the Core")
|
||||
}
|
||||
frames := fake.frames(t)
|
||||
if len(frames) != 1 || string(frames[0]) != "{}" {
|
||||
t.Errorf("frames = %q, want the stalled frame finished once the host woke", frames)
|
||||
}
|
||||
if fake.deadlineCount() != 1+windows {
|
||||
t.Errorf("deadlines armed = %d, want one per stalled window plus the first", fake.deadlineCount())
|
||||
}
|
||||
}
|
||||
|
||||
type pipeTimeoutError struct{}
|
||||
|
||||
func (*pipeTimeoutError) Error() string { return "i/o timeout" }
|
||||
func (*pipeTimeoutError) Timeout() bool { return true }
|
||||
|
||||
func TestSendResumesAFrameThatStalledOnANamedPipeTimeout(t *testing.T) {
|
||||
fake := &fakeConn{writeErr: &pipeTimeoutError{}, writeErrAfter: 2, writeErrTimes: 1}
|
||||
previous := swapConn(fake)
|
||||
defer swapConn(previous)
|
||||
|
||||
send([]byte("{}"))
|
||||
|
||||
if fake.isClosed() {
|
||||
t.Fatal("a Windows pipe timeout on a half-written frame closed the connection instead of resuming it")
|
||||
}
|
||||
frames := fake.frames(t)
|
||||
if len(frames) != 1 || string(frames[0]) != "{}" {
|
||||
t.Errorf("frames = %q, want the stalled frame finished so the stream stays in sync", frames)
|
||||
}
|
||||
}
|
||||
|
||||
type pipeNonTimeoutError struct{}
|
||||
|
||||
func (*pipeNonTimeoutError) Error() string { return "pipe closed" }
|
||||
func (*pipeNonTimeoutError) Timeout() bool { return false }
|
||||
|
||||
func TestSendDropsTheConnectionWhenTimeoutReportsFalse(t *testing.T) {
|
||||
fake := &fakeConn{writeErr: &pipeNonTimeoutError{}, writeErrAfter: 2}
|
||||
previous := swapConn(fake)
|
||||
defer swapConn(previous)
|
||||
|
||||
send([]byte("{}"))
|
||||
|
||||
if !fake.isClosed() {
|
||||
t.Error("a frame that stalls past every retry leaves the stream desynchronized and must close the connection")
|
||||
t.Error("an error that implements Timeout() but reports false was treated as a stall instead of a dead host")
|
||||
}
|
||||
if fake.deadlineCount() != 1+ipcPartialFrameRetries {
|
||||
t.Errorf("deadlines armed = %d, want the retries bounded at %d", fake.deadlineCount(), ipcPartialFrameRetries)
|
||||
}
|
||||
|
||||
func TestSendDropsTheConnectionWhenAStalledFrameCannotRearmItsDeadline(t *testing.T) {
|
||||
fake := &fakeConn{
|
||||
writeErr: os.ErrDeadlineExceeded,
|
||||
writeErrAfter: 2,
|
||||
writeErrTimes: 1,
|
||||
deadlineErr: errors.New("pipe closed"),
|
||||
deadlineOK: 1,
|
||||
}
|
||||
previous := swapConn(fake)
|
||||
defer swapConn(previous)
|
||||
|
||||
send([]byte("{}"))
|
||||
|
||||
if !fake.isClosed() {
|
||||
t.Error("a half-written frame whose deadline could not be re-armed left the connection open; the stream is desynchronized")
|
||||
}
|
||||
if conn != nil {
|
||||
t.Error("conn still points at the dead connection")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+14
-9
@@ -26,9 +26,8 @@ var (
|
||||
)
|
||||
|
||||
const (
|
||||
maxIPCFrameSize = 64 * 1024 * 1024
|
||||
ipcWriteTimeout = 10 * time.Second
|
||||
ipcPartialFrameRetries = 6
|
||||
maxIPCFrameSize = 64 * 1024 * 1024
|
||||
ipcWriteTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
var deliveryFailureReported atomic.Bool
|
||||
@@ -73,7 +72,6 @@ func writeFrame(w io.Writer, data []byte) (int, error) {
|
||||
type resumingWriter struct {
|
||||
conn ipcConn
|
||||
written int
|
||||
stalls int
|
||||
}
|
||||
|
||||
func (writer *resumingWriter) Write(data []byte) (int, error) {
|
||||
@@ -91,17 +89,24 @@ func (writer *resumingWriter) Write(data []byte) (int, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// A half-written frame is only in sync while this process keeps writing it,
|
||||
// and a host the OS has suspended (Windows Modern Standby freezes the app but
|
||||
// not a service's child) drains again whenever it wakes, so a stall is waited
|
||||
// out for as long as it lasts. A dead host surfaces as a hard error instead.
|
||||
func (writer *resumingWriter) resume(err error) bool {
|
||||
if writer.written == 0 || writer.stalls >= ipcPartialFrameRetries {
|
||||
if writer.written == 0 || !isWriteTimeout(err) {
|
||||
return false
|
||||
}
|
||||
if !errors.Is(err, os.ErrDeadlineExceeded) {
|
||||
return false
|
||||
}
|
||||
writer.stalls++
|
||||
return writer.conn.SetWriteDeadline(time.Now().Add(ipcWriteTimeout)) == nil
|
||||
}
|
||||
|
||||
// go-winio reports an expired deadline as its own ErrTimeout, which does not
|
||||
// match os.ErrDeadlineExceeded; Timeout() is the contract every pipe shares.
|
||||
func isWriteTimeout(err error) bool {
|
||||
var timeout interface{ Timeout() bool }
|
||||
return errors.As(err, &timeout) && timeout.Timeout()
|
||||
}
|
||||
|
||||
func writeAll(w io.Writer, data []byte) (int, error) {
|
||||
written := 0
|
||||
for len(data) > 0 {
|
||||
|
||||
Reference in New Issue
Block a user