Skip to content

Commit 0fec7bb

Browse files
mohitkhullarmohitkhullar
committed
enforce max call time
Signed-off-by: mohitkhullar <[email protected]>
1 parent a45c6a1 commit 0fec7bb

File tree

2 files changed

+99
-30
lines changed

2 files changed

+99
-30
lines changed

‎cdb2api/cdb2api.c‎

Lines changed: 98 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,9 @@ static int cdb2_comdb2db_timeout_set_from_env = 0;
131131
static int CDB2_API_CALL_TIMEOUT = 120000; /* defaults to 2 minute */
132132
static int cdb2_api_call_timeout_set_from_env = 0;
133133

134+
static int CDB2_ENFORCE_API_CALL_TIMEOUT = 0;
135+
static int cdb2_enforce_api_call_timeout_set_from_env = 0;
136+
134137
static int CDB2_SOCKET_TIMEOUT = 5000;
135138
static int cdb2_socket_timeout_set_from_env = 0;
136139

@@ -856,6 +859,40 @@ static int is_sql_read(const char *sqlstr)
856859
#define HAVE_MSGHDR_MSG_CONTROL
857860
#endif
858861

862+
static int is_api_call_timedout(cdb2_hndl_tp *hndl)
863+
{
864+
struct timeval tv;
865+
gettimeofday(&tv, NULL);
866+
long long current_time = tv.tv_sec * 1000 + tv.tv_usec / 1000;
867+
if (hndl->max_call_time && (hndl->max_call_time < current_time)) {
868+
return 1;
869+
}
870+
return 0;
871+
}
872+
873+
static long long get_call_timeout(const cdb2_hndl_tp *hndl, long long timeout)
874+
{
875+
if (!hndl)
876+
return timeout;
877+
struct timeval tv;
878+
gettimeofday(&tv, NULL);
879+
long long current_time = tv.tv_sec * 1000 + tv.tv_usec / 1000;
880+
long long time_left = hndl->max_call_time - current_time;
881+
if (hndl->max_call_time && time_left <= 0)
882+
time_left = 1;
883+
if (time_left > 0 && (time_left < timeout))
884+
return time_left;
885+
return timeout;
886+
}
887+
888+
static void set_max_call_time(cdb2_hndl_tp *hndl)
889+
{
890+
struct timeval tv;
891+
gettimeofday(&tv, NULL);
892+
if (hndl)
893+
hndl->max_call_time = tv.tv_sec * 1000 + tv.tv_usec / 1000 + hndl->api_call_timeout;
894+
}
895+
859896
enum {
860897
PASSFD_SUCCESS = 0,
861898
PASSFD_RECVMSG = -1, /* error with recvmsg() */
@@ -995,8 +1032,7 @@ static int recv_fd_int(int sockfd, void *data, size_t nbytes, int *fd_recvd, int
9951032
static int recv_fd(const cdb2_hndl_tp *hndl, int sockfd, void *data, size_t nbytes, int *fd_recvd)
9961033
{
9971034
int rc, timeoutms = hndl ? hndl->sockpool_recv_timeoutms : CDB2_SOCKPOOL_RECV_TIMEOUTMS;
998-
if (hndl && hndl->api_call_timeout && (timeoutms > hndl->api_call_timeout))
999-
timeoutms = hndl->api_call_timeout;
1035+
timeoutms = get_call_timeout(hndl, timeoutms);
10001036
rc = recv_fd_int(sockfd, data, nbytes, fd_recvd, timeoutms);
10011037
if (rc != 0 && *fd_recvd != -1) {
10021038
int errno_save = errno;
@@ -1116,8 +1152,7 @@ static int send_fd_to(int sockfd, const void *data, size_t nbytes,
11161152
static int send_fd(const cdb2_hndl_tp *hndl, int sockfd, const void *data, size_t nbytes, int fd_to_send)
11171153
{
11181154
int timeoutms = hndl ? hndl->sockpool_send_timeoutms : CDB2_SOCKPOOL_SEND_TIMEOUTMS;
1119-
if (hndl && hndl->api_call_timeout && (timeoutms > hndl->api_call_timeout))
1120-
timeoutms = hndl->api_call_timeout;
1155+
timeoutms = get_call_timeout(hndl, timeoutms);
11211156
return send_fd_to(sockfd, data, nbytes, fd_to_send, timeoutms);
11221157
}
11231158

@@ -1591,6 +1626,8 @@ static void read_comdb2db_environment_cfg(cdb2_hndl_tp *hndl, const char *comdb2
15911626
&cdb2_sockpool_recv_timeoutms_set_from_env);
15921627
process_env_var_int("COMDB2_CONFIG_API_CALL_TIMEOUT", &CDB2_API_CALL_TIMEOUT,
15931628
&cdb2_api_call_timeout_set_from_env);
1629+
process_env_var_int("COMDB2_CONFIG_ENFORCE_API_CALL_TIMEOUT", &CDB2_ENFORCE_API_CALL_TIMEOUT,
1630+
&cdb2_enforce_api_call_timeout_set_from_env);
15941631
process_env_var_int("COMDB2_CONFIG_COMDB2DB_TIMEOUT", &COMDB2DB_TIMEOUT, &cdb2_comdb2db_timeout_set_from_env);
15951632
process_env_var_int("COMDB2_CONFIG_SOCKET_TIMEOUT", &CDB2_SOCKET_TIMEOUT, &cdb2_socket_timeout_set_from_env);
15961633
process_env_var_int("COMDB2_CONFIG_PROTOBUF_SIZE", &CDB2_PROTOBUF_SIZE, &cdb2_protobuf_size_set_from_env);
@@ -1841,6 +1878,9 @@ static void read_comdb2db_cfg(cdb2_hndl_tp *hndl, SBUF2 *s, const char *comdb2db
18411878
hndl->api_call_timeout = atoi(tok);
18421879
else if (tok)
18431880
CDB2_API_CALL_TIMEOUT = atoi(tok);
1881+
} else if (!cdb2_api_call_timeout_set_from_env && (strcasecmp("enforce_api_call_timeout", tok) == 0)) {
1882+
tok = strtok_r(NULL, " :,", &last);
1883+
CDB2_ENFORCE_API_CALL_TIMEOUT = value_on_off(tok, &err);
18441884
} else if (strcasecmp("auto_consume_timeout", tok) == 0) {
18451885
tok = strtok_r(NULL, " :,", &last);
18461886
if (tok)
@@ -2125,6 +2165,8 @@ static void set_cdb2_timeouts(cdb2_hndl_tp *hndl)
21252165
hndl->comdb2db_timeout = hndl->api_call_timeout;
21262166
if (hndl->socket_timeout > hndl->api_call_timeout)
21272167
hndl->socket_timeout = hndl->api_call_timeout;
2168+
2169+
set_max_call_time(hndl);
21282170
}
21292171

21302172
/* Read all available comdb2 configuration files.
@@ -3721,8 +3763,21 @@ static int cdb2portmux_get(cdb2_hndl_tp *hndl, const char *type,
37213763

37223764
debugprint("name %s\n", name);
37233765

3724-
fd = cdb2_tcpconnecth_to(hndl, remote_host, CDB2_PORTMUXPORT, 0,
3725-
hndl->connect_timeout);
3766+
int connect_timeout = hndl->connect_timeout;
3767+
3768+
if (CDB2_ENFORCE_API_CALL_TIMEOUT) {
3769+
#ifdef CDB2API_TEST
3770+
printf("RETRY with timeout %lld\n", get_call_timeout(hndl, connect_timeout));
3771+
#endif
3772+
if (is_api_call_timedout(hndl)) {
3773+
snprintf(hndl->errstr, sizeof(hndl->errstr), "%s:%d Timed out connecting to db\n", __func__, __LINE__);
3774+
port = -1;
3775+
goto after_callback;
3776+
}
3777+
connect_timeout = get_call_timeout(hndl, connect_timeout);
3778+
}
3779+
3780+
fd = cdb2_tcpconnecth_to(hndl, remote_host, CDB2_PORTMUXPORT, 0, connect_timeout);
37263781
if (fd < 0) {
37273782
debugprint("cdb2_tcpconnecth_to returns fd=%d'\n", fd);
37283783
if (errno == EINPROGRESS) {
@@ -3748,7 +3803,18 @@ static int cdb2portmux_get(cdb2_hndl_tp *hndl, const char *type,
37483803
port = -1;
37493804
goto after_callback;
37503805
}
3751-
sbuf2settimeout(ss, hndl->connect_timeout, hndl->connect_timeout);
3806+
if (CDB2_ENFORCE_API_CALL_TIMEOUT) {
3807+
#ifdef CDB2API_TEST
3808+
printf("RETRY with timeout %lld\n", get_call_timeout(hndl, connect_timeout));
3809+
#endif
3810+
if (is_api_call_timedout(hndl)) {
3811+
snprintf(hndl->errstr, sizeof(hndl->errstr), "%s:%d Timed out connecting to db\n", __func__, __LINE__);
3812+
port = -1;
3813+
goto after_callback;
3814+
}
3815+
connect_timeout = get_call_timeout(hndl, connect_timeout);
3816+
}
3817+
sbuf2settimeout(ss, connect_timeout, connect_timeout);
37523818
sbuf2printf(ss, "get %s\n", name);
37533819
sbuf2flush(ss);
37543820
res[0] = 0;
@@ -3937,6 +4003,18 @@ static int cdb2_read_record(cdb2_hndl_tp *hndl, uint8_t **buf, int *len, int *ty
39374003
goto after_callback;
39384004

39394005
retry:
4006+
if (CDB2_ENFORCE_API_CALL_TIMEOUT) {
4007+
int socket_timeout = get_call_timeout(hndl, hndl->socket_timeout);
4008+
#ifdef CDB2API_TEST
4009+
printf("GOT HEARTBEAT || Set timeout to %d\n", socket_timeout);
4010+
#endif
4011+
if (is_api_call_timedout(hndl)) {
4012+
snprintf(hndl->errstr, sizeof(hndl->errstr), "%s:%d Timed out reading response from the db\n", __func__,
4013+
__LINE__);
4014+
rc = -1;
4015+
}
4016+
sbuf2settimeout(sb, socket_timeout, socket_timeout);
4017+
}
39404018
b_read = sbuf2fread((char *)&hdr, 1, sizeof(hdr), sb);
39414019
debugprint("READ HDR b_read=%d, sizeof(hdr)=(%zu):\n", b_read, sizeof(hdr));
39424020

@@ -4101,6 +4179,10 @@ static int cdb2_read_record(cdb2_hndl_tp *hndl, uint8_t **buf, int *len, int *ty
41014179

41024180
rc = 0;
41034181
after_callback:
4182+
// reset here
4183+
if (CDB2_ENFORCE_API_CALL_TIMEOUT) {
4184+
sbuf2settimeout(sb, hndl->socket_timeout, hndl->socket_timeout);
4185+
}
41044186
while ((e = cdb2_next_callback(hndl, CDB2_AFTER_READ_RECORD, e)) != NULL) {
41054187
callbackrc =
41064188
cdb2_invoke_callback(hndl, e, 1, CDB2_RETURN_VALUE, (intptr_t)rc);
@@ -5987,10 +6069,8 @@ static int cdb2_run_statement_typed_int(cdb2_hndl_tp *hndl, const char *sql, int
59876069
hndl->retry_all = 1;
59886070
int run_last = 1;
59896071

5990-
time_t max_time =
5991-
time(NULL) + (hndl->api_call_timeout - hndl->connect_timeout) / 1000;
5992-
if (max_time < 0)
5993-
max_time = 0;
6072+
set_max_call_time(hndl);
6073+
59946074
retry_queries:
59956075
debugprint(
59966076
"retry_queries: hndl->host=%d (%s)\n", hndl->connected_host,
@@ -6018,18 +6098,10 @@ static int cdb2_run_statement_typed_int(cdb2_hndl_tp *hndl, const char *sql, int
60186098
cdb2_get_dbhosts(hndl);
60196099
}
60206100

6021-
int tmsec = 0;
6022-
6023-
// Add wait if we have already tried on all the nodes.
6024-
if (!hndl->sb && (retries_done > hndl->num_hosts)) {
6025-
tmsec = (retries_done - hndl->num_hosts) * 100;
6026-
}
6027-
60286101
if (hndl->sslerr != 0)
60296102
PRINT_AND_RETURN(CDB2ERR_CONNECT_ERROR);
60306103

6031-
if ((retries_done > 1) && ((retries_done > hndl->max_retries) ||
6032-
((time(NULL) + (tmsec / 1000)) >= max_time))) {
6104+
if ((retries_done > 1) && ((retries_done > hndl->max_retries) || (is_api_call_timedout(hndl)))) {
60336105
sprintf(hndl->errstr, "%s: Maximum number of retries done.", __func__);
60346106
if (is_hasql_commit) {
60356107
cleanup_query_list(hndl, &commit_query_list, __LINE__);
@@ -7906,17 +7978,13 @@ static int cdb2_get_dbhosts(cdb2_hndl_tp *hndl)
79067978
}
79077979
}
79087980

7909-
time_t max_time =
7910-
time(NULL) +
7911-
(hndl->api_call_timeout - (CDB2_POLL_TIMEOUT + hndl->connect_timeout)) /
7912-
1000;
7913-
if (max_time < 0)
7914-
max_time = 0;
7981+
if (!hndl->max_call_time)
7982+
set_max_call_time(hndl);
79157983

79167984
use_bmsd = cdb2_use_bmsd && (*cdb2_bmssuffix != '\0') && !hndl->num_shards; // cannot find shards via bmsd yet
79177985
retry:
79187986
if (rc) {
7919-
if (num_retry >= MAX_RETRIES || time(NULL) > max_time)
7987+
if (num_retry >= MAX_RETRIES || is_api_call_timedout(hndl))
79207988
goto after_callback;
79217989

79227990
num_retry++;
@@ -7934,7 +8002,7 @@ static int cdb2_get_dbhosts(cdb2_hndl_tp *hndl)
79348002
hndl, cdb2_default_cluster, comdb2db_name, comdb2db_num,
79358003
comdb2db_hosts[i], comdb2db_hosts, comdb2db_ports, &master,
79368004
&num_comdb2db_hosts, NULL);
7937-
if (rc == 0 || time(NULL) >= max_time) {
8005+
if (rc == 0 || is_api_call_timedout(hndl)) {
79388006
break;
79398007
}
79408008
}
@@ -7951,15 +8019,15 @@ static int cdb2_get_dbhosts(cdb2_hndl_tp *hndl)
79518019
hndl->hosts, &hndl->num_hosts, hndl->dbname, hndl->type, &hndl->dbnum,
79528020
&hndl->num_hosts_sameroom, num_retry, use_bmsd, hndl->shards, &hndl->num_shards,
79538021
&hndl->num_shards_sameroom);
7954-
if (rc == 0 || time(NULL) >= max_time) {
8022+
if (rc == 0 || is_api_call_timedout(hndl)) {
79558023
break;
79568024
} else if (use_bmsd) {
79578025
if (cdb2_comdb2db_fallback)
79588026
use_bmsd = 0;
79598027
goto retry;
79608028
}
79618029
}
7962-
if (rc == -1 && time(NULL) < max_time) {
8030+
if (rc == -1 && !is_api_call_timedout(hndl)) {
79638031
rc = comdb2db_get_dbhosts(hndl, comdb2db_name, comdb2db_num, comdb2db_hosts[master], comdb2db_ports[master],
79648032
hndl->hosts, &hndl->num_hosts, hndl->dbname, hndl->type, &hndl->dbnum,
79658033
&hndl->num_hosts_sameroom, num_retry, use_bmsd, hndl->shards, &hndl->num_shards,

‎cdb2api/cdb2api_hndl.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ struct cdb2_hndl {
168168
int is_hasql;
169169
int sent_client_info;
170170
void *user_arg;
171+
long long max_call_time;
171172
int api_call_timeout;
172173
int connect_timeout;
173174
int comdb2db_timeout;

0 commit comments

Comments
 (0)