Merge remote-tracking branch 'origin/main' into MONGOSH-1914-parallel-log-cleanup-issue

# Conflicts:
#	packages/mongodb-log-writer/src/mongo-log-manager.ts

Author: alenakhineika

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@mongodb-js/devtools-connect",
-  "version": "3.5.0",
+  "version": "3.6.0",
   "description": "A connection establishment utility for MongoDB developer tools",
   "homepage": "https://github.com/mongodb-js/devtools-shared/tree/main/packages/devtools-connect",
   "repository": {
@@ -56,7 +56,7 @@
   "peerDependencies": {
     "@mongodb-js/oidc-plugin": "^1.1.0",
     "mongodb": "^6.9.0",
-    "mongodb-log-writer": "^2.2.0"
+    "mongodb-log-writer": "^2.3.0"
   },
   "devDependencies": {
     "@mongodb-js/oidc-plugin": "^1.1.0",
@@ -76,7 +76,7 @@
     "gen-esm-wrapper": "^1.1.0",
     "mocha": "^8.4.0",
     "mongodb": "^6.9.0",
-    "mongodb-log-writer": "^2.2.0",
+    "mongodb-log-writer": "^2.3.0",
     "nyc": "^15.1.0",
     "os-dns-native": "^1.2.0",
     "resolve-mongodb-srv": "^1.1.1",

packages/devtools-connect/package.json

@@ -1,7 +1,7 @@
 {
   "name": "mongodb-log-writer",
   "description": "A library for writing MongoDB logv2 messages",
-  "version": "2.2.0",
+  "version": "2.3.0",
   "author": {
     "name": "MongoDB Inc",
     "email": "[email protected]"

packages/mongodb-log-writer/package.json

@@ -32,6 +32,48 @@ describe('MongoLogManager', function () {
     sinon.restore();
   });
 
+  it('constructor throws with invalid prefixes', function () {
+    expect(() => {
+      new MongoLogManager({
+        directory,
+        retentionDays,
+        prefix: '%asdabs/',
+        onwarn,
+        onerror,
+      });
+    }).to.throw();
+
+    expect(() => {
+      new MongoLogManager({
+        directory,
+        retentionDays,
+        prefix: '$$$$',
+        onwarn,
+        onerror,
+      });
+    }).to.throw();
+
+    expect(() => {
+      new MongoLogManager({
+        directory,
+        retentionDays,
+        prefix: 'abc_',
+        onwarn,
+        onerror,
+      });
+    }).not.to.throw();
+
+    expect(() => {
+      new MongoLogManager({
+        directory,
+        retentionDays,
+        prefix: 'something',
+        onwarn,
+        onerror,
+      });
+    }).not.to.throw();
+  });
+
   it('allows creating and writing to log files', async function () {
     const manager = new MongoLogManager({
       directory,
@@ -62,6 +104,19 @@ describe('MongoLogManager', function () {
     expect(log[0].t.$date).to.be.a('string');
   });
 
+  it('can take a custom prefix for log files', async function () {
+    const manager = new MongoLogManager({
+      directory,
+      retentionDays,
+      prefix: 'custom_',
+      onwarn,
+      onerror,
+    });
+
+    const writer = await manager.createLogWriter();
+    expect(writer.logFilePath as string).to.match(/custom_/);
+  });
+
   it('cleans up old log files when requested', async function () {
     retentionDays = 0.000001; // 86.4 ms
     const manager = new MongoLogManager({
@@ -179,7 +234,57 @@ describe('MongoLogManager', function () {
     expect(leftoverFiles).deep.equals([faultyFile, ...validFiles.slice(3)]);
   });
 
-  it('cleans up least recent log files when over a storage limit', async function () {
+  it('cleanup only applies to files with the prefix, if set', async function () {
+    const manager = new MongoLogManager({
+      directory,
+      retentionDays,
+      maxLogFileCount: 7,
+      prefix: 'custom_',
+      onwarn,
+      onerror,
+    });
+
+    const paths: string[] = [];
+    const offset = Math.floor(Date.now() / 1000);
+
+    // Create 4 files: 2 with a different prefix and 2 with no prefix
+    for (let i = 1; i >= 0; i--) {
+      const withoutPrefix = path.join(
+        directory,
+        ObjectId.createFromTime(offset - i).toHexString() + '_log'
+      );
+      await fs.writeFile(withoutPrefix, '');
+      paths.push(withoutPrefix);
+
+      const withDifferentPrefix = path.join(
+        directory,
+        'different_' +
+          ObjectId.createFromTime(offset - i).toHexString() +
+          '_log'
+      );
+      await fs.writeFile(withDifferentPrefix, '');
+      paths.push(withDifferentPrefix);
+    }
+
+    // Create 10 files with the prefix
+    for (let i = 9; i >= 0; i--) {
+      const filename = path.join(
+        directory,
+        `custom_${ObjectId.createFromTime(offset - i).toHexString()}_log`
+      );
+      await fs.writeFile(filename, '');
+      paths.push(filename);
+    }
+
+    expect(await getFilesState(paths)).to.equal('11111111111111');
+    await manager.cleanupOldLogFiles();
+
+    // The first 4 files without the right prefix should still be there.
+    // The next (oldest) 3 files with the prefix should be deleted.
+    expect(await getFilesState(paths)).to.equal('11110001111111');
+  });
+
+  it('cleans up least recent log files when requested with a storage limit', async function () {
     const manager = new MongoLogManager({
       directory,
       retentionDays,

packages/mongodb-log-writer/src/mongo-log-manager.spec.ts

@@ -19,6 +19,8 @@ interface MongoLogOptions {
   maxLogFileCount?: number;
   /** The maximal size of log files which are kept. */
   retentionGB?: number;
+  /** Prefix to use for the log files */
+  prefix?: string;
   /** A handler for errors related to a specific filesystem path. */
   onerror: (err: Error, path: string) => unknown | Promise<void>;
   /** A handler for warnings related to a specific filesystem path. */
@@ -34,6 +36,13 @@ export class MongoLogManager {
   _options: MongoLogOptions;
 
   constructor(options: MongoLogOptions) {
+    if (options.prefix) {
+      if (!/^[a-z0-9_]+$/i.test(options.prefix)) {
+        throw new Error(
+          'Prefix must only contain letters, numbers, and underscores'
+        );
+      }
+    }
     this._options = options;
   }
 
@@ -48,6 +57,10 @@ export class MongoLogManager {
     }
   }
 
+  private get prefix() {
+    return this._options.prefix ?? '';
+  }
+
   /** Clean up log files older than `retentionDays`. */
   async cleanupOldLogFiles(maxDurationMs = 5_000, remainingRetries = 1): Promise<void> {
     const dir = this._options.directory;
@@ -81,8 +94,11 @@ export class MongoLogManager {
         if (Date.now() - deletionStartTimestamp > maxDurationMs) break;
 
         if (!dirent.isFile()) continue;
-        const { id } =
-          /^(?<id>[a-f0-9]{24})_log(\.gz)?$/i.exec(dirent.name)?.groups ?? {};
+        const logRegExp = new RegExp(
+          `^${this.prefix}(?<id>[a-f0-9]{24})_log(\\.gz)?$`,
+          'i'
+        );
+        const { id } = logRegExp.exec(dirent.name)?.groups ?? {};
         if (!id) continue;
 
         const fileTimestamp = +new ObjectId(id).getTimestamp();
@@ -121,21 +137,6 @@ export class MongoLogManager {
           usedStorageSize -= toDelete.fileSize ?? 0;
         }
       }
-
-      if (this._options.retentionGB) {
-        const storageSizeLimit = this._options.retentionGB * 1024 * 1024 * 1024;
-  
-        for (const file of leastRecentFileHeap) {
-          if (Date.now() - deletionStartTimestamp > maxDurationMs) break;
-  
-          if (usedStorageSize <= storageSizeLimit) break;
-  
-          if (!file.fileSize) continue;
-  
-          await this.deleteFile(file.fullPath);
-          usedStorageSize -= file.fileSize;
-        }
-      }
     } catch (statErr: any) {
       // Multiple processes may attempt to clean up log files in parallel.
       // A situation can arise where one process tries to read a file
@@ -146,6 +147,21 @@ export class MongoLogManager {
         await this.cleanupOldLogFiles(maxDurationMs - Date.now() - deletionStartTimestamp, remainingRetries - 1);
       }
     }
+
+    if (this._options.retentionGB) {
+      const storageSizeLimit = this._options.retentionGB * 1024 * 1024 * 1024;
+
+      for (const file of leastRecentFileHeap) {
+        if (Date.now() - deletionStartTimestamp > maxDurationMs) break;
+
+        if (usedStorageSize <= storageSizeLimit) break;
+
+        if (!file.fileSize) continue;
+
+        await this.deleteFile(file.fullPath);
+        usedStorageSize -= file.fileSize;
+      }
+    }
   }
 
   /** Create a MongoLogWriter stream for a new log file. */
@@ -154,7 +170,7 @@ export class MongoLogManager {
     const doGzip = !!this._options.gzip;
     const logFilePath = path.join(
       this._options.directory,
-      `${logId}_log${doGzip ? '.gz' : ''}`
+      `${this.prefix}${logId}_log${doGzip ? '.gz' : ''}`
     );
 
     let originalTarget: Writable;