feat(mongodb-log-writer): add logRetentionGB configuration MONGOSH-1985

Author: gagik

packages/mongodb-log-writer/src/mongo-log-manager.spec.ts

@@ -86,6 +86,19 @@ describe('MongoLogManager', function () {
     }
   });
 
+  const getFilesState = async (paths: string[]) => {
+    return (
+      await Promise.all(
+        paths.map((path) =>
+          fs.stat(path).then(
+            () => 1,
+            () => 0
+          )
+        )
+      )
+    ).join('');
+  };
+
   it('cleans up least recent log files when requested', async function () {
     const manager = new MongoLogManager({
       directory,
@@ -106,21 +119,38 @@ describe('MongoLogManager', function () {
       paths.unshift(filename);
     }
 
-    const getFiles = async () => {
-      return (
-        await Promise.all(
-          paths.map((path) =>
-            fs.stat(path).then(
-              () => 1,
-              () => 0
-            )
-          )
-        )
-      ).join('');
-    };
-    expect(await getFiles()).to.equal('1111111111');
+    expect(await getFilesState(paths)).to.equal('1111111111');
+    await manager.cleanupOldLogFiles();
+    expect(await getFilesState(paths)).to.equal('0000011111');
+  });
+
+  it('cleans up least recent log files when requested with a storage limit', async function () {
+    const manager = new MongoLogManager({
+      directory,
+      retentionDays,
+      maxLogFileCount: 1000,
+      // 6 KB
+      logRetentionGB: 6 / 1024 / 1024,
+      onwarn,
+      onerror,
+    });
+
+    const paths: string[] = [];
+    const offset = Math.floor(Date.now() / 1000);
+
+    // Create 10 files of 1 KB each.
+    for (let i = 0; i < 10; i++) {
+      const filename = path.join(
+        directory,
+        ObjectId.createFromTime(offset - i).toHexString() + '_log'
+      );
+      await fs.writeFile(filename, '0'.repeat(1024));
+      paths.unshift(filename);
+    }
+
+    expect(await getFilesState(paths)).to.equal('1111111111');
     await manager.cleanupOldLogFiles();
-    expect(await getFiles()).to.equal('0000011111');
+    expect(await getFilesState(paths)).to.equal('0000111111');
   });
 
   it('cleaning up old log files is a no-op by default', async function () {

packages/mongodb-log-writer/src/mongo-log-manager.ts

@@ -17,6 +17,8 @@ interface MongoLogOptions {
   retentionDays: number;
   /** The maximal number of log files which are kept. */
   maxLogFileCount?: number;
+  /** The maximal GB of log files which are kept. */
+  logRetentionGB?: number;
   /** A handler for warnings related to a specific filesystem path. */
   onerror: (err: Error, path: string) => unknown | Promise<void>;
   /** A handler for errors related to a specific filesystem path. */
@@ -54,8 +56,15 @@ export class MongoLogManager {
     const leastRecentFileHeap = new Heap<{
       fileTimestamp: number;
       fullPath: string;
+      fileSize?: number;
     }>((a, b) => a.fileTimestamp - b.fileTimestamp);
 
+    const storageSizeLimit = this._options.logRetentionGB
+      ? this._options.logRetentionGB * 1024 * 1024 * 1024
+      : Infinity;
+    let usedStorageSize = this._options.logRetentionGB ? 0 : -Infinity;
+    // eslint-disable-next-line no-console
+
     for await (const dirent of dirHandle) {
       // Cap the overall time spent inside this function. Consider situations like
       // a large number of machines using a shared network-mounted $HOME directory
@@ -69,23 +78,53 @@ export class MongoLogManager {
       if (!id) continue;
       const fileTimestamp = +new ObjectId(id).getTimestamp();
       const fullPath = path.join(dir, dirent.name);
-      let toDelete: string | undefined;
+      let toDelete:
+        | {
+            fullPath: string;
+            /** If the file wasn't deleted right away and there is a
+             *  retention size limit, its size should be accounted */
+            fileSize?: number;
+          }
+        | undefined;
 
       // If the file is older than expected, delete it. If the file is recent,
       // add it to the list of seen files, and if that list is too large, remove
       // the least recent file we've seen so far.
       if (fileTimestamp < deletionCutoffTimestamp) {
-        toDelete = fullPath;
-      } else if (this._options.maxLogFileCount) {
-        leastRecentFileHeap.push({ fullPath, fileTimestamp });
-        if (leastRecentFileHeap.size() > this._options.maxLogFileCount) {
-          toDelete = leastRecentFileHeap.pop()?.fullPath;
+        toDelete = {
+          fullPath,
+        };
+      } else if (
+        this._options.logRetentionGB ||
+        this._options.maxLogFileCount
+      ) {
+        const fileSize = (await fs.stat(fullPath)).size;
+        if (this._options.logRetentionGB) {
+          usedStorageSize += fileSize;
+        }
+
+        leastRecentFileHeap.push({
+          fullPath,
+          fileTimestamp,
+          fileSize,
+        });
+
+        const reachedMaxStorageSize = usedStorageSize > storageSizeLimit;
+        const reachedMaxFileCount =
+          this._options.maxLogFileCount &&
+          leastRecentFileHeap.size() > this._options.maxLogFileCount;
+
+        if (reachedMaxStorageSize || reachedMaxFileCount) {
+          toDelete = leastRecentFileHeap.pop();
         }
       }
 
       if (!toDelete) continue;
       try {
-        await fs.unlink(toDelete);
+        await fs.unlink(toDelete.fullPath);
+        if (toDelete.fileSize) {
+          usedStorageSize -= toDelete.fileSize;
+        }
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
       } catch (err: any) {
         if (err?.code !== 'ENOENT') {