Skip to content

Commit

Permalink
PG-1245 Rework telemetry file format
Browse files Browse the repository at this point in the history
  • Loading branch information
@artemgavrilov
artemgavrilov committed Dec 20, 2024
1 parent 6b3463d commit a10e8c6
Show file tree
Hide file tree
Showing 3 changed files with 105 additions and 165 deletions.
156 changes: 65 additions & 91 deletions percona_pg_telemetry.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -588,20 +588,18 @@ write_pg_settings(void)
SPITupleTable *tuptable;
int spi_result;
char *query = "SELECT name, unit, setting, reset_val, boot_val FROM pg_settings where vartype != 'string'";
char msg[2048] = {0};
char msg_json[4096] = {0};
size_t sz_json;
char buf[4096] = {0};
size_t buf_size = sizeof(buf);
FILE *fp;
int flags;

sz_json = sizeof(msg_json);

/* Open file in append mode. */
fp = json_file_open(ptss->dbtemp_filepath, "a+");

/* Construct and initiate the active extensions array block. */
construct_json_block(msg_json, sz_json, "", "settings", PT_JSON_ARRAY_START, &ptss->json_file_indent);
write_json_to_file(fp, msg_json);
construct_json_block(buf, buf_size, "settings", NULL, PT_JSON_NAMED_OBJECT_START, &ptss->json_file_indent);
write_json_to_file(fp, buf);

SetCurrentStatementStartTimestamp();
StartTransactionCommand();
Expand Down Expand Up @@ -630,47 +628,41 @@ write_pg_settings(void)
for (int row_count = 0; row_count < SPI_processed; row_count++)
{
char *null_value = "NULL";
char *value_str[PT_SETTINGS_COL_COUNT];


/* Construct and initiate the active extensions array block. */
construct_json_block(msg_json, sz_json, "setting", "", PT_JSON_BLOCK_ARRAY_VALUE, &ptss->json_file_indent);
write_json_to_file(fp, msg_json);
char *name = SPI_getvalue(tuptable->vals[row_count], tuptable->tupdesc, 1);

construct_json_block(buf, buf_size, name, NULL, PT_JSON_NAMED_OBJECT_START, &ptss->json_file_indent);
write_json_to_file(fp, buf);

/* Process the tuple as needed */
for (int col_count = 1; col_count <= tuptable->tupdesc->natts; col_count++)
for (int col_count = 2; col_count <= tuptable->tupdesc->natts; col_count++)
{
char *str = SPI_getvalue(tuptable->vals[row_count], tuptable->tupdesc, col_count);

value_str[col_count - 1] = (str == NULL || str[0] == '\0') ? null_value : str;
char *value = (str == NULL || str[0] == '\0') ? null_value : str;

flags = (col_count == tuptable->tupdesc->natts) ? (PT_JSON_BLOCK_SIMPLE | PT_JSON_BLOCK_LAST) : PT_JSON_BLOCK_SIMPLE;
flags = (col_count == tuptable->tupdesc->natts) ? (PT_JSON_KEY_VALUE | PT_JSON_LAST_ELEMENT) : PT_JSON_KEY_VALUE;

construct_json_block(msg_json, sz_json, NameStr(tuptable->tupdesc->attrs[col_count - 1].attname),
value_str[col_count - 1], flags, &ptss->json_file_indent);
write_json_to_file(fp, msg_json);
construct_json_block(buf, buf_size, NameStr(tuptable->tupdesc->attrs[col_count - 1].attname),
value, flags, &ptss->json_file_indent);
write_json_to_file(fp, buf);
}

/* Close the array */
construct_json_block(msg, sizeof(msg), "setting", "", PT_JSON_ARRAY_END | PT_JSON_BLOCK_LAST, &ptss->json_file_indent);
strcpy(msg_json, msg);

/* Close the extension block */
flags = (row_count == (SPI_processed - 1)) ? (PT_JSON_BLOCK_END | PT_JSON_BLOCK_LAST) : PT_JSON_BLOCK_END;
construct_json_block(msg, sizeof(msg), "setting", "", flags, &ptss->json_file_indent);
strlcat(msg_json, msg, sz_json);
flags = (row_count == (SPI_processed - 1)) ? (PT_JSON_OBJECT_END | PT_JSON_LAST_ELEMENT) : PT_JSON_OBJECT_END;
construct_json_block(buf, sizeof(buf), NULL, NULL, flags, &ptss->json_file_indent);

/* Write both to file. */
write_json_to_file(fp, msg_json);
write_json_to_file(fp, buf);
}
}

/* Close the array */
construct_json_block(msg, sizeof(msg), "settings", "", PT_JSON_ARRAY_END, &ptss->json_file_indent);
strcpy(msg_json, msg);
construct_json_block(buf, sizeof(buf), NULL, NULL, PT_JSON_OBJECT_END, &ptss->json_file_indent);

/* Write both to file. */
write_json_to_file(fp, msg_json);
write_json_to_file(fp, buf);

/* Clean up */
fclose(fp);
Expand Down Expand Up @@ -803,79 +795,61 @@ get_extensions_list(PTDatabaseInfo *dbinfo, MemoryContext cxt)
static bool
write_database_info(PTDatabaseInfo *dbinfo, List *extlist)
{
char msg[2048] = {0};
char msg_json[4096] = {0};
size_t sz_json;
char str[2048] = {0};
char buf[4096] = {0};
size_t buf_size = sizeof(buf);
FILE *fp;
ListCell *lc;
int flags;

sz_json = sizeof(msg_json);

/* Open file in append mode. */
fp = json_file_open(ptss->dbtemp_filepath, "a+");

if (ptss->first_db_entry)
{
/* Construct and initiate the active extensions array block. */
construct_json_block(msg_json, sz_json, "", "databases", PT_JSON_ARRAY_START, &ptss->json_file_indent);
write_json_to_file(fp, msg_json);
/* Start databases block. */
construct_json_block(buf, buf_size, "databases", NULL, PT_JSON_NAMED_OBJECT_START, &ptss->json_file_indent);
write_json_to_file(fp, buf);
}

/* Construct and initiate the active extensions array block. */
construct_json_block(msg_json, sz_json, "database", "value", PT_JSON_BLOCK_ARRAY_VALUE, &ptss->json_file_indent);
write_json_to_file(fp, msg_json);

/* Construct and write the database OID block. */
snprintf(msg, sizeof(msg), "%u", dbinfo->datid);
construct_json_block(msg_json, sz_json, "database_oid", msg, PT_JSON_BLOCK_SIMPLE, &ptss->json_file_indent);
write_json_to_file(fp, msg_json);
/* Start database block where database OID is key. */
snprintf(str, sizeof(str), "%u", dbinfo->datid);
construct_json_block(buf, buf_size, str, NULL, PT_JSON_NAMED_OBJECT_START, &ptss->json_file_indent);
write_json_to_file(fp, buf);

/* Construct and write the database size block. */
snprintf(msg, sizeof(msg), "%lu", dbinfo->datsize);
construct_json_block(msg_json, sz_json, "database_size", msg, PT_JSON_BLOCK_SIMPLE, &ptss->json_file_indent);
write_json_to_file(fp, msg_json);
/* Construct and write the database size. */
snprintf(str, sizeof(str), "%lu", dbinfo->datsize);
construct_json_block(buf, buf_size, "database", str, PT_JSON_KEY_VALUE, &ptss->json_file_indent);
write_json_to_file(fp, buf);

/* Construct and initiate the active extensions array block. */
construct_json_block(msg_json, sz_json, "active_extensions", "value", PT_JSON_BLOCK_ARRAY_VALUE, &ptss->json_file_indent);
write_json_to_file(fp, msg_json);
/* Start active extensions array block. */
construct_json_block(buf, buf_size, "active_extensions", NULL, PT_JSON_NAMED_ARRAY_START, &ptss->json_file_indent);
write_json_to_file(fp, buf);

/* Iterate through all extensions and those to the array. */
foreach(lc, extlist)
{
PTExtensionInfo *extinfo = lfirst(lc);

flags = (list_tail(extlist) == lc) ? (PT_JSON_BLOCK_SIMPLE | PT_JSON_BLOCK_LAST) : PT_JSON_BLOCK_SIMPLE;

construct_json_block(msg_json, sz_json, "extension_name", extinfo->extname, flags, &ptss->json_file_indent);
write_json_to_file(fp, msg_json);
flags = (list_tail(extlist) == lc) ? (PT_JSON_VALUE | PT_JSON_LAST_ELEMENT) : PT_JSON_VALUE;
construct_json_block(buf, buf_size, NULL, extinfo->extname, flags, &ptss->json_file_indent);
write_json_to_file(fp, buf);
}

/* Close the array and block and write to file */
construct_json_block(msg, sizeof(msg), "active_extensions", "active_extensions", PT_JSON_ARRAY_END | PT_JSON_BLOCK_END | PT_JSON_BLOCK_LAST, &ptss->json_file_indent);
strcpy(msg_json, msg);
write_json_to_file(fp, msg_json);

/* Close the array */
construct_json_block(msg, sizeof(msg), "database", "", PT_JSON_ARRAY_END | PT_JSON_BLOCK_LAST, &ptss->json_file_indent);
strcpy(msg_json, msg);
/* Close active extensions array */
construct_json_block(buf, buf_size, NULL, NULL, PT_JSON_ARRAY_END | PT_JSON_LAST_ELEMENT, &ptss->json_file_indent);
write_json_to_file(fp, buf);

/* Close the database block */
flags = (ptss->last_db_entry) ? (PT_JSON_BLOCK_END | PT_JSON_BLOCK_LAST) : PT_JSON_BLOCK_END;
construct_json_block(msg, sizeof(msg), "database", "", flags, &ptss->json_file_indent);
strlcat(msg_json, msg, sz_json);

/* Write both to file. */
write_json_to_file(fp, msg_json);
flags = (ptss->last_db_entry) ? (PT_JSON_OBJECT_END | PT_JSON_LAST_ELEMENT) : PT_JSON_OBJECT_END;
construct_json_block(buf, buf_size, NULL, NULL, flags, &ptss->json_file_indent);
write_json_to_file(fp, buf);

if (ptss->last_db_entry)
{
/* Close the array */
construct_json_block(msg, sizeof(msg), "databases", "", PT_JSON_ARRAY_END | PT_JSON_BLOCK_LAST, &ptss->json_file_indent);
strcpy(msg_json, msg);

/* Write both to file. */
write_json_to_file(fp, msg_json);
/* Close databases array */
construct_json_block(buf, buf_size, NULL, NULL, PT_JSON_OBJECT_END | PT_JSON_LAST_ELEMENT, &ptss->json_file_indent);
write_json_to_file(fp, buf);
}

/* Clean up */
Expand All @@ -895,9 +869,9 @@ percona_pg_telemetry_main(Datum main_arg)
ListCell *lc = NULL;
char json_pg_version[1024];
FILE *fp;
char msg[2048] = {0};
char msg_json[4096] = {0};
size_t sz_json = sizeof(msg_json);
char str[2048] = {0};
char buf[4096] = {0};
size_t buf_size = sizeof(buf);
bool first_time = true;

/* Save the version in a JSON escaped stirng just to be safe. */
Expand Down Expand Up @@ -979,27 +953,27 @@ percona_pg_telemetry_main(Datum main_arg)
/* Open file for writing. */
fp = json_file_open(ptss->dbtemp_filepath, "w");

construct_json_block(msg_json, sz_json, "", "", PT_JSON_BLOCK_START, &ptss->json_file_indent);
write_json_to_file(fp, msg_json);
construct_json_block(buf, buf_size, NULL, NULL, PT_JSON_OBJECT_START, &ptss->json_file_indent);
write_json_to_file(fp, buf);

/* Construct and write the database size block. */
pg_snprintf(msg, sizeof(msg), "%lu", GetSystemIdentifier());
construct_json_block(msg_json, sz_json, "db_instance_id", msg, PT_JSON_KEY_VALUE_PAIR, &ptss->json_file_indent);
write_json_to_file(fp, msg_json);
pg_snprintf(str, sizeof(str), "%lu", GetSystemIdentifier());
construct_json_block(buf, buf_size, "db_instance_id", str, PT_JSON_KEY_VALUE, &ptss->json_file_indent);
write_json_to_file(fp, buf);

/* Construct and initiate the active extensions array block. */
construct_json_block(msg_json, sz_json, "pillar_version", json_pg_version, PT_JSON_KEY_VALUE_PAIR, &ptss->json_file_indent);
write_json_to_file(fp, msg_json);
construct_json_block(buf, buf_size, "pillar_version", json_pg_version, PT_JSON_KEY_VALUE, &ptss->json_file_indent);
write_json_to_file(fp, buf);

/* Construct and initiate the active extensions array block. */
pg_snprintf(msg, sizeof(msg), "%ld", server_uptime());
construct_json_block(msg_json, sz_json, "uptime", msg, PT_JSON_KEY_VALUE_PAIR, &ptss->json_file_indent);
write_json_to_file(fp, msg_json);
pg_snprintf(str, sizeof(str), "%ld", server_uptime());
construct_json_block(buf, buf_size, "uptime", str, PT_JSON_KEY_VALUE, &ptss->json_file_indent);
write_json_to_file(fp, buf);

/* Construct and initiate the active extensions array block. */
pg_snprintf(temp_buff, sizeof(temp_buff), "%d", list_length(dblist));
construct_json_block(msg_json, sz_json, "databases_count", temp_buff, PT_JSON_KEY_VALUE_PAIR, &ptss->json_file_indent);
write_json_to_file(fp, msg_json);
construct_json_block(buf, buf_size, "databases_count", temp_buff, PT_JSON_KEY_VALUE, &ptss->json_file_indent);
write_json_to_file(fp, buf);

/* Let's close the file now so that processes may add their stuff. */
fclose(fp);
Expand Down Expand Up @@ -1037,8 +1011,8 @@ percona_pg_telemetry_main(Datum main_arg)

/* Open file, writing the closing bracket and close it. */
fp = json_file_open(ptss->dbtemp_filepath, "a+");
construct_json_block(msg_json, sz_json, "", "", PT_JSON_BLOCK_END | PT_JSON_BLOCK_LAST, &ptss->json_file_indent);
write_json_to_file(fp, msg_json);
construct_json_block(buf, buf_size, NULL, NULL, PT_JSON_OBJECT_END | PT_JSON_LAST_ELEMENT, &ptss->json_file_indent);
write_json_to_file(fp, buf);
fclose(fp);

/* Generate and save the filename */
Expand Down
81 changes: 29 additions & 52 deletions pt_json.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,98 +66,75 @@ json_fix_value(char *str)
* we don't expect to encounter that in extension names.
*/
char *
construct_json_block(char *msg_block, size_t msg_block_sz, char *key, char *raw_value, int flags, int *json_file_indent)
construct_json_block(char *buf, size_t buf_sz, char *key, char *raw_value, int flags, int *json_file_indent)
{
char *value = NULL;
char msg[2048] = {0};
char msg_json[2048] = {0};

char str[2048] = {0};
char fmt_str[2048] = {0};
char comma = (flags & PT_JSON_LAST_ELEMENT) ? '\0' : ',';
/* Make the string empty so that we can always concat. */
msg_block[0] = '\0';
buf[0] = '\0';

if (raw_value)
value = json_fix_value(raw_value);

if (flags & PT_JSON_BLOCK_START)
if (flags & PT_JSON_KEY)
snprintf(str, sizeof(str), "\"%s\": ", key);

if (flags & PT_JSON_OBJECT_START)
{
PT_FORMAT_JSON(msg_json, sizeof(msg_json), "{", (*json_file_indent));
strlcpy(msg_block, msg_json, msg_block_sz);
strlcat(str, "{", sizeof(str));
PT_FORMAT_JSON(fmt_str, sizeof(fmt_str), str, (*json_file_indent));
strlcat(buf, fmt_str, buf_sz);

(*json_file_indent)++;
}

if (flags & PT_JSON_KEY_VALUE_PAIR)
if (flags & PT_JSON_VALUE)
{
snprintf(msg, sizeof(msg), "\"%s\": \"%s\",", key, value);
PT_FORMAT_JSON(msg_json, sizeof(msg_json), msg, (*json_file_indent));
strlcat(msg_block, msg_json, msg_block_sz);
}
char v[2048] = {0};
snprintf(v, sizeof(v), "\"%s\"%c", value, comma);

if (flags & PT_JSON_BLOCK_KEY)
{
snprintf(msg, sizeof(msg), "\"key\": \"%s\",", key);
PT_FORMAT_JSON(msg_json, sizeof(msg_json), msg, (*json_file_indent));
strlcat(msg_block, msg_json, msg_block_sz);
}

if (flags & PT_JSON_BLOCK_VALUE)
{
snprintf(msg, sizeof(msg), "\"value\": \"%s\"", value);
PT_FORMAT_JSON(msg_json, sizeof(msg_json), msg, (*json_file_indent));
strlcat(msg_block, msg_json, msg_block_sz);
strlcat(str, v, sizeof(str));
PT_FORMAT_JSON(fmt_str, sizeof(fmt_str), str, (*json_file_indent));
strlcat(buf, fmt_str, buf_sz);
}

if (flags & PT_JSON_ARRAY_START)
{
if (value && value[0] != '\0')
snprintf(msg, sizeof(msg), "\"%s\": [", value);
else
snprintf(msg, sizeof(msg), "\"value\": [");

PT_FORMAT_JSON(msg_json, sizeof(msg_json), msg, (*json_file_indent));
strlcat(msg_block, msg_json, msg_block_sz);
strlcat(str, "[", sizeof(str));
PT_FORMAT_JSON(fmt_str, sizeof(fmt_str), str, (*json_file_indent));
strlcat(buf, fmt_str, buf_sz);

(*json_file_indent)++;
}

/* Value is not an array so we can close the block. */
if (flags & PT_JSON_ARRAY_END)
{
char closing[3] = {']', ',', '\0'};

if (flags & PT_JSON_BLOCK_LAST)
{
/* Let's remove the comma in case this is the last block. */
closing[1] = '\0';
}
char closing[3] = {']', comma, '\0'};

(*json_file_indent)--;

PT_FORMAT_JSON(msg_json, sizeof(msg_json), closing, (*json_file_indent));
strlcat(msg_block, msg_json, msg_block_sz);
PT_FORMAT_JSON(fmt_str, sizeof(fmt_str), closing, (*json_file_indent));
strlcat(buf, fmt_str, buf_sz);
}

/* Value is not an array so we can close the block. */
if (flags & PT_JSON_BLOCK_END)
if (flags & PT_JSON_OBJECT_END)
{
char closing[3] = {'}', ',', '\0'};

if (flags & PT_JSON_BLOCK_LAST)
{
/* Let's remove the comma in case this is the last block. */
closing[1] = '\0';
}
char closing[3] = {'}', comma, '\0'};

(*json_file_indent)--;

PT_FORMAT_JSON(msg_json, sizeof(msg_json), closing, (*json_file_indent));
strlcat(msg_block, msg_json, msg_block_sz);
PT_FORMAT_JSON(fmt_str, sizeof(fmt_str), closing, (*json_file_indent));
strlcat(buf, fmt_str, buf_sz);
}

if (value)
pfree(value);

return msg_block;
return buf;
}

/*
Expand Down
Loading

0 comments on commit a10e8c6

Please sign in to comment.