Skip to content

Commit 5c0226d

Browse files
murgatroid99murgatroid99
authored
Merge pull request #2760 from davidfiala/@grpc/[email protected]
Keepalive bugfixes and unify timers strategies between client and server
2 parents 52fe8e9 + 98cd87f commit 5c0226d

File tree

2 files changed

+237
-139
lines changed

2 files changed

+237
-139
lines changed

‎packages/grpc-js/src/server.ts‎

Lines changed: 186 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,14 @@ export class Server {
435435
);
436436
}
437437

438+
private keepaliveTrace(text: string): void {
439+
logging.trace(
440+
LogVerbosity.DEBUG,
441+
'keepalive',
442+
'(' + this.channelzRef.id + ') ' + text
443+
);
444+
}
445+
438446
addProtoService(): never {
439447
throw new Error('Not implemented. Use addService() instead');
440448
}
@@ -1376,8 +1384,7 @@ export class Server {
13761384

13771385
let connectionAgeTimer: NodeJS.Timeout | null = null;
13781386
let connectionAgeGraceTimer: NodeJS.Timeout | null = null;
1379-
let keeapliveTimeTimer: NodeJS.Timeout | null = null;
1380-
let keepaliveTimeoutTimer: NodeJS.Timeout | null = null;
1387+
let keepaliveTimer: NodeJS.Timeout | null = null;
13811388
let sessionClosedByServer = false;
13821389

13831390
const idleTimeoutObj = this.enableIdleTimeout(session);
@@ -1420,41 +1427,90 @@ export class Server {
14201427
connectionAgeTimer.unref?.();
14211428
}
14221429

1423-
if (this.keepaliveTimeMs < KEEPALIVE_MAX_TIME_MS) {
1424-
keeapliveTimeTimer = setInterval(() => {
1425-
keepaliveTimeoutTimer = setTimeout(() => {
1426-
sessionClosedByServer = true;
1427-
session.close();
1428-
}, this.keepaliveTimeoutMs);
1429-
keepaliveTimeoutTimer.unref?.();
1430+
const clearKeepaliveTimeout = () => {
1431+
if (keepaliveTimer) {
1432+
clearTimeout(keepaliveTimer);
1433+
keepaliveTimer = null;
1434+
}
1435+
};
14301436

1431-
try {
1432-
session.ping(
1433-
(err: Error | null, duration: number, payload: Buffer) => {
1434-
if (keepaliveTimeoutTimer) {
1435-
clearTimeout(keepaliveTimeoutTimer);
1436-
}
1437-
1438-
if (err) {
1439-
sessionClosedByServer = true;
1440-
this.trace(
1441-
'Connection dropped due to error of a ping frame ' +
1442-
err.message +
1443-
' return in ' +
1444-
duration
1445-
);
1446-
session.close();
1447-
}
1437+
const canSendPing = () => {
1438+
return (
1439+
!session.destroyed &&
1440+
this.keepaliveTimeMs < KEEPALIVE_MAX_TIME_MS &&
1441+
this.keepaliveTimeMs > 0
1442+
);
1443+
};
1444+
1445+
/* eslint-disable-next-line prefer-const */
1446+
let sendPing: () => void; // hoisted for use in maybeStartKeepalivePingTimer
1447+
1448+
const maybeStartKeepalivePingTimer = () => {
1449+
if (!canSendPing()) {
1450+
return;
1451+
}
1452+
this.keepaliveTrace(
1453+
'Starting keepalive timer for ' + this.keepaliveTimeMs + 'ms'
1454+
);
1455+
keepaliveTimer = setTimeout(() => {
1456+
clearKeepaliveTimeout();
1457+
sendPing();
1458+
}, this.keepaliveTimeMs);
1459+
keepaliveTimer.unref?.();
1460+
};
1461+
1462+
sendPing = () => {
1463+
if (!canSendPing()) {
1464+
return;
1465+
}
1466+
this.keepaliveTrace(
1467+
'Sending ping with timeout ' + this.keepaliveTimeoutMs + 'ms'
1468+
);
1469+
let pingSendError = '';
1470+
try {
1471+
const pingSentSuccessfully = session.ping(
1472+
(err: Error | null, duration: number, payload: Buffer) => {
1473+
clearKeepaliveTimeout();
1474+
if (err) {
1475+
this.keepaliveTrace('Ping failed with error: ' + err.message);
1476+
sessionClosedByServer = true;
1477+
session.close();
1478+
} else {
1479+
this.keepaliveTrace('Received ping response');
1480+
maybeStartKeepalivePingTimer();
14481481
}
1449-
);
1450-
} catch (e) {
1451-
clearTimeout(keepaliveTimeoutTimer);
1452-
// The ping can't be sent because the session is already closed
1453-
session.destroy();
1482+
}
1483+
);
1484+
if (!pingSentSuccessfully) {
1485+
pingSendError = 'Ping returned false';
14541486
}
1455-
}, this.keepaliveTimeMs);
1456-
keeapliveTimeTimer.unref?.();
1457-
}
1487+
} catch (e) {
1488+
// grpc/grpc-node#2139
1489+
pingSendError =
1490+
(e instanceof Error ? e.message : '') || 'Unknown error';
1491+
}
1492+
1493+
if (pingSendError) {
1494+
this.keepaliveTrace('Ping send failed: ' + pingSendError);
1495+
this.trace(
1496+
'Connection dropped due to ping send error: ' + pingSendError
1497+
);
1498+
sessionClosedByServer = true;
1499+
session.close();
1500+
return;
1501+
}
1502+
1503+
keepaliveTimer = setTimeout(() => {
1504+
clearKeepaliveTimeout();
1505+
this.keepaliveTrace('Ping timeout passed without response');
1506+
this.trace('Connection dropped by keepalive timeout');
1507+
sessionClosedByServer = true;
1508+
session.close();
1509+
}, this.keepaliveTimeoutMs);
1510+
keepaliveTimer.unref?.();
1511+
};
1512+
1513+
maybeStartKeepalivePingTimer();
14581514

14591515
session.on('close', () => {
14601516
if (!sessionClosedByServer) {
@@ -1471,12 +1527,7 @@ export class Server {
14711527
clearTimeout(connectionAgeGraceTimer);
14721528
}
14731529

1474-
if (keeapliveTimeTimer) {
1475-
clearInterval(keeapliveTimeTimer);
1476-
if (keepaliveTimeoutTimer) {
1477-
clearTimeout(keepaliveTimeoutTimer);
1478-
}
1479-
}
1530+
clearKeepaliveTimeout();
14801531

14811532
if (idleTimeoutObj !== null) {
14821533
clearTimeout(idleTimeoutObj.timeout);
@@ -1521,8 +1572,7 @@ export class Server {
15211572

15221573
let connectionAgeTimer: NodeJS.Timeout | null = null;
15231574
let connectionAgeGraceTimer: NodeJS.Timeout | null = null;
1524-
let keeapliveTimeTimer: NodeJS.Timeout | null = null;
1525-
let keepaliveTimeoutTimer: NodeJS.Timeout | null = null;
1575+
let keepaliveTimeout: NodeJS.Timeout | null = null;
15261576
let sessionClosedByServer = false;
15271577

15281578
const idleTimeoutObj = this.enableIdleTimeout(session);
@@ -1564,49 +1614,103 @@ export class Server {
15641614
connectionAgeTimer.unref?.();
15651615
}
15661616

1567-
if (this.keepaliveTimeMs < KEEPALIVE_MAX_TIME_MS) {
1568-
keeapliveTimeTimer = setInterval(() => {
1569-
keepaliveTimeoutTimer = setTimeout(() => {
1570-
sessionClosedByServer = true;
1571-
this.channelzTrace.addTrace(
1572-
'CT_INFO',
1573-
'Connection dropped by keepalive timeout from ' + clientAddress
1574-
);
1617+
const clearKeepaliveTimeout = () => {
1618+
if (keepaliveTimeout) {
1619+
clearTimeout(keepaliveTimeout);
1620+
keepaliveTimeout = null;
1621+
}
1622+
};
1623+
1624+
const canSendPing = () => {
1625+
return (
1626+
!session.destroyed &&
1627+
this.keepaliveTimeMs < KEEPALIVE_MAX_TIME_MS &&
1628+
this.keepaliveTimeMs > 0
1629+
);
1630+
};
15751631

1576-
session.close();
1577-
}, this.keepaliveTimeoutMs);
1578-
keepaliveTimeoutTimer.unref?.();
1632+
/* eslint-disable-next-line prefer-const */
1633+
let sendPing: () => void; // hoisted for use in maybeStartKeepalivePingTimer
15791634

1580-
try {
1581-
session.ping(
1582-
(err: Error | null, duration: number, payload: Buffer) => {
1583-
if (keepaliveTimeoutTimer) {
1584-
clearTimeout(keepaliveTimeoutTimer);
1585-
}
1586-
1587-
if (err) {
1588-
sessionClosedByServer = true;
1589-
this.channelzTrace.addTrace(
1590-
'CT_INFO',
1591-
'Connection dropped due to error of a ping frame ' +
1592-
err.message +
1593-
' return in ' +
1594-
duration
1595-
);
1596-
1597-
session.close();
1598-
}
1635+
const maybeStartKeepalivePingTimer = () => {
1636+
if (!canSendPing()) {
1637+
return;
1638+
}
1639+
this.keepaliveTrace(
1640+
'Starting keepalive timer for ' + this.keepaliveTimeMs + 'ms'
1641+
);
1642+
keepaliveTimeout = setTimeout(() => {
1643+
clearKeepaliveTimeout();
1644+
sendPing();
1645+
}, this.keepaliveTimeMs);
1646+
keepaliveTimeout.unref?.();
1647+
};
1648+
1649+
sendPing = () => {
1650+
if (!canSendPing()) {
1651+
return;
1652+
}
1653+
this.keepaliveTrace(
1654+
'Sending ping with timeout ' + this.keepaliveTimeoutMs + 'ms'
1655+
);
1656+
let pingSendError = '';
1657+
try {
1658+
const pingSentSuccessfully = session.ping(
1659+
(err: Error | null, duration: number, payload: Buffer) => {
1660+
clearKeepaliveTimeout();
1661+
if (err) {
1662+
this.keepaliveTrace('Ping failed with error: ' + err.message);
1663+
this.channelzTrace.addTrace(
1664+
'CT_INFO',
1665+
'Connection dropped due to error of a ping frame ' +
1666+
err.message +
1667+
' return in ' +
1668+
duration
1669+
);
1670+
sessionClosedByServer = true;
1671+
session.close();
1672+
} else {
1673+
this.keepaliveTrace('Received ping response');
1674+
maybeStartKeepalivePingTimer();
15991675
}
1600-
);
1601-
channelzSessionInfo.keepAlivesSent += 1;
1602-
} catch (e) {
1603-
clearTimeout(keepaliveTimeoutTimer);
1604-
// The ping can't be sent because the session is already closed
1605-
session.destroy();
1676+
}
1677+
);
1678+
if (!pingSentSuccessfully) {
1679+
pingSendError = 'Ping returned false';
16061680
}
1607-
}, this.keepaliveTimeMs);
1608-
keeapliveTimeTimer.unref?.();
1609-
}
1681+
} catch (e) {
1682+
// grpc/grpc-node#2139
1683+
pingSendError =
1684+
(e instanceof Error ? e.message : '') || 'Unknown error';
1685+
}
1686+
1687+
if (pingSendError) {
1688+
this.keepaliveTrace('Ping send failed: ' + pingSendError);
1689+
this.channelzTrace.addTrace(
1690+
'CT_INFO',
1691+
'Connection dropped due to ping send error: ' + pingSendError
1692+
);
1693+
sessionClosedByServer = true;
1694+
session.close();
1695+
return;
1696+
}
1697+
1698+
channelzSessionInfo.keepAlivesSent += 1;
1699+
1700+
keepaliveTimeout = setTimeout(() => {
1701+
clearKeepaliveTimeout();
1702+
this.keepaliveTrace('Ping timeout passed without response');
1703+
this.channelzTrace.addTrace(
1704+
'CT_INFO',
1705+
'Connection dropped by keepalive timeout from ' + clientAddress
1706+
);
1707+
sessionClosedByServer = true;
1708+
session.close();
1709+
}, this.keepaliveTimeoutMs);
1710+
keepaliveTimeout.unref?.();
1711+
};
1712+
1713+
maybeStartKeepalivePingTimer();
16101714

16111715
session.on('close', () => {
16121716
if (!sessionClosedByServer) {
@@ -1627,12 +1731,7 @@ export class Server {
16271731
clearTimeout(connectionAgeGraceTimer);
16281732
}
16291733

1630-
if (keeapliveTimeTimer) {
1631-
clearInterval(keeapliveTimeTimer);
1632-
if (keepaliveTimeoutTimer) {
1633-
clearTimeout(keepaliveTimeoutTimer);
1634-
}
1635-
}
1734+
clearKeepaliveTimeout();
16361735

16371736
if (idleTimeoutObj !== null) {
16381737
clearTimeout(idleTimeoutObj.timeout);

0 commit comments

Comments
 (0)