fix: JavaScript heap out of memory errors in browser tests (#569)

* Initial plan

* Initial analysis of memory heap issue in browser tests

Co-authored-by: kobenguyent <[email protected]>

* Fix memory heap overflow issues in browser tests

- Replace synchronous fs.readFileSync with streaming in get-file.js
- Add 10MB file size limit to prevent memory overload
- Add memory cleanup for worker processes with bounded arrays
- Enhance safe-serialize with memory limits for arrays and objects
- Add comprehensive test coverage for memory fixes
- Maintain backward compatibility with existing functionality

Co-authored-by: kobenguyent <[email protected]>

---------

Co-authored-by: copilot-swe-agent[bot] <[email protected]>
Co-authored-by: kobenguyent <[email protected]>

Author: Copilot

lib/api/get-file.js

@@ -1,7 +1,56 @@
-const fs = require('fs').promises;
+const fs = require('fs');
+const path = require('path');
 
-module.exports = (req, res) => {
-  if (!req.file) return;
-  if (!req.file.startsWith(global.codecept_dir)) return; // not a codecept file
-  res.send(fs.readFileSync(req.file));
+// Maximum file size to prevent memory issues (10MB)
+const MAX_FILE_SIZE = 10 * 1024 * 1024;
+
+module.exports = async (req, res) => {
+  if (!req.file) {
+    return res.status(400).json({ error: 'File parameter is required' });
+  }
+  
+  if (!req.file.startsWith(global.codecept_dir)) {
+    return res.status(403).json({ error: 'Access denied. File must be within codecept directory' });
+  }
+  
+  try {
+    // Check if file exists and get stats
+    const stats = await fs.promises.stat(req.file);
+    
+    // Check file size to prevent memory issues
+    if (stats.size > MAX_FILE_SIZE) {
+      return res.status(413).json({ 
+        error: 'File too large',
+        message: `File size (${stats.size} bytes) exceeds maximum allowed size (${MAX_FILE_SIZE} bytes)`
+      });
+    }
+    
+    // Use streaming for better memory management
+    const readStream = fs.createReadStream(req.file);
+    
+    // Set appropriate headers
+    res.setHeader('Content-Type', 'text/plain; charset=utf-8');
+    res.setHeader('Content-Length', stats.size);
+    
+    // Handle stream errors
+    readStream.on('error', (error) => {
+      console.error('Error reading file:', error);
+      if (!res.headersSent) {
+        res.status(500).json({ 
+          error: 'Failed to read file',
+          message: error.message
+        });
+      }
+    });
+    
+    // Pipe the file directly to response to avoid loading into memory
+    readStream.pipe(res);
+    
+  } catch (error) {
+    console.error('Error accessing file:', error);
+    res.status(404).json({ 
+      error: 'File not found',
+      message: error.message
+    });
+  }
 };

lib/model/codeceptjs-run-workers/index.js

@@ -17,6 +17,9 @@ const stats = {
   tests: 0,
 };
 
+// Memory management constants
+const MAX_ERRORS = 1000;          // Maximum errors to keep in memory
+const MAX_FINISHED_TESTS = 500;   // Maximum finished tests to keep in memory
 
 let numberOfWorkersClosed = 0;
 const hasFailures = () => stats.failures || errors.length;
@@ -74,8 +77,15 @@ module.exports = function (workers, config, options, codecept, container) {
 
       switch (message.event) {
       case event.test.failed:
-        finishedTests.push(repackTest(message.data));
-        output.test.failed(repackTest(message.data));
+        const failedTest = repackTest(message.data);
+        finishedTests.push(failedTest);
+        
+        // Prevent memory buildup by limiting finished tests
+        if (finishedTests.length > MAX_FINISHED_TESTS) {
+          finishedTests.shift(); // Remove oldest test
+        }
+        
+        output.test.failed(failedTest);
         break;
       case event.test.passed: output.test.passed(repackTest(message.data)); break;
       case event.suite.before: output.suite.started(message.data); break;
@@ -86,6 +96,12 @@ module.exports = function (workers, config, options, codecept, container) {
 
     worker.on('error', (err) => {
       errors.push(err);
+      
+      // Prevent memory buildup by limiting error array size
+      if (errors.length > MAX_ERRORS) {
+        errors.shift(); // Remove oldest error
+      }
+      
       // eslint-disable-next-line no-console
       console.error(err);
     });
@@ -102,6 +118,9 @@ module.exports = function (workers, config, options, codecept, container) {
           }
           // eslint-disable-next-line no-console
           console.log('FInished Successfully');
+          
+          // Clean up memory after test run completion
+          cleanupWorkerMemory();
           // process.exit(0);
         });
       }
@@ -162,6 +181,23 @@ function repackTest(test) {
 //   return Object.assign(new Suite(suite.title), suite);
 // }
 
+/**
+ * Clean up memory after test run completion to prevent memory leaks
+ */
+function cleanupWorkerMemory() {
+  // Clear arrays
+  finishedTests.length = 0;
+  errors.length = 0;
+  
+  // Reset counters
+  numberOfWorkersClosed = 0;
+  
+  // Force garbage collection if available (for debugging/testing)
+  if (global.gc) {
+    global.gc();
+  }
+}
+
 function simplifyObject(object, remove = {}) {
   const defaultRemove = {
     objects: true,

lib/utils/safe-serialize.js

@@ -8,14 +8,20 @@
  * @param {*} obj - The object to serialize safely
  * @param {number} maxDepth - Maximum recursion depth (default: 50)
  * @param {WeakSet} seen - Internally used to track visited objects
+ * @param {number} maxProperties - Maximum properties per object (default: 100)
  * @returns {*} Safe copy of the object
  */
-function safeSerialize(obj, maxDepth = 50, seen = new WeakSet()) {
-  // Handle primitive types and null
-  if (obj === null || typeof obj !== 'object') {
+function safeSerialize(obj, maxDepth = 50, seen = new WeakSet(), maxProperties = 100) {
+  // Handle primitive types and null (but not functions)
+  if (obj === null || (typeof obj !== 'object' && typeof obj !== 'function')) {
     return obj;
   }
 
+  // Handle Functions explicitly (they are not objects but need special handling)
+  if (typeof obj === 'function') {
+    return '[Function: ' + (obj.name || 'anonymous') + ']';
+  }
+  
   // Check depth limit
   if (maxDepth <= 0) {
     return '[Object: max depth reached]';
@@ -32,15 +38,21 @@ function safeSerialize(obj, maxDepth = 50, seen = new WeakSet()) {
   try {
     // Handle arrays
     if (Array.isArray(obj)) {
-      return obj.map(item => safeSerialize(item, maxDepth - 1, seen));
+      // Limit array size to prevent memory issues
+      if (obj.length > 1000) {
+        const truncated = obj.slice(0, 1000).map(item => safeSerialize(item, maxDepth - 1, seen, maxProperties));
+        truncated.push(`[Array truncated - original length: ${obj.length}]`);
+        return truncated;
+      }
+      return obj.map(item => safeSerialize(item, maxDepth - 1, seen, maxProperties));
     }
 
     // Handle Error objects specially
     if (obj instanceof Error) {
       return {
         name: obj.name,
         message: obj.message,
-        stack: obj.stack,
+        stack: obj.stack ? obj.stack.substring(0, 2000) : undefined, // Limit stack trace length
         code: obj.code,
         type: 'Error'
       };
@@ -58,17 +70,27 @@ function safeSerialize(obj, maxDepth = 50, seen = new WeakSet()) {
 
     // Handle Buffer objects
     if (typeof Buffer !== 'undefined' && Buffer.isBuffer && Buffer.isBuffer(obj)) {
-      return '[Buffer]';
+      return `[Buffer: ${obj.length} bytes]`;
     }
 
     // Handle plain objects
     const result = {};
+    let propertyCount = 0;
+    
     for (const key in obj) {
       if (Object.prototype.hasOwnProperty.call(obj, key)) {
+        // Limit properties to prevent memory issues
+        if (propertyCount >= maxProperties) {
+          result['[...truncated]'] = `Object had ${Object.keys(obj).length} properties, showing first ${maxProperties}`;
+          break;
+        }
+        
         try {
-          result[key] = safeSerialize(obj[key], maxDepth - 1, seen);
+          result[key] = safeSerialize(obj[key], maxDepth - 1, seen, maxProperties);
+          propertyCount++;
         } catch (err) {
           result[key] = '[Serialization Error: ' + err.message + ']';
+          propertyCount++;
         }
       }
     }