Merge pull request #789 from particle-iot/feature/sc-133461/allow-caching-downloaded-os-files

implement cache limit validation

Author: hugomontero

settings.js

@@ -41,7 +41,8 @@ let settings = {
 		'tinker': true
 	},
 
-	tachyonMeta: 'https://tachyon-ci.particle.io/meta'
+	tachyonMeta: 'https://tachyon-ci.particle.io/meta',
+	tachyonCacheLimitGB: 10
 };
 
 function envValue(varName, defaultValue) {

src/cmd/download-tachyon-package.js

@@ -36,7 +36,7 @@ module.exports = class DownloadTachyonPackageCommand extends CLICommandBase {
 		const answer = await this.ui.prompt(question);
 		return answer.version;
 	}
-	async download ({ region, version }) {
+	async download ({ region, version, alwaysCleanCache = false }) {
 		// prompt for region and version if not provided
 		if (!region) {
 			region = await this._selectRegion();
@@ -52,7 +52,7 @@ module.exports = class DownloadTachyonPackageCommand extends CLICommandBase {
 		}
 		const { artifact_url: url, sha256_checksum: expectedChecksum } = build.artifacts[0];
 		const outputFileName = url.replace(/.*\//, '');
-		const filePath = await manager.download({ url, outputFileName, expectedChecksum });
+		const filePath = await manager.download({ url, outputFileName, expectedChecksum, options: { alwaysCleanCache } });
 		this.ui.write(`Downloaded package to: ${filePath}`);
 
 		return filePath;

src/cmd/setup-tachyon.js

@@ -31,27 +31,7 @@ module.exports = class SetupTachyonCommands extends CLICommandBase {
 
 	async setup({ skip_flashing_os: skipFlashingOs, version, load_config: loadConfig, save_config: saveConfig }) {
 		try {
-			this.ui.write(`
-===========================================================
-              Particle Tachyon Setup Command
-===========================================================
-
-Welcome to the Particle Tachyon setup! This interactive command:
-
-- Flashes your Tachyon device
-- Configures it (password, WiFi credentials etc...)
-- Connects it to the internet and the Particle Cloud!
-
-**What you'll need:**
-
-1. Your Tachyon device
-2. The Tachyon battery
-3. A USB-C cable
-
-**Important:**
-- This tool requires you to be logged into your Particle account.
-- For more details, check out the documentation at: https://part.cl/setup-tachyon`);
-
+			this._showWelcomeMessage();
 			this._formatAndDisplaySteps("Okay—first up! Checking if you're logged in...", 1);
 
 			await this._verifyLogin();
@@ -66,6 +46,7 @@ Welcome to the Particle Tachyon setup! This interactive command:
 			}
 
 			let config = { systemPassword: null, wifi: null, sshPublicKey: null };
+			let alwaysCleanCache = false;
 
 			if ( !loadConfig ) {
 				config = await this._runStepWithTiming(
@@ -81,6 +62,7 @@ Welcome to the Particle Tachyon setup! This interactive command:
 					0
 				);
 			} else {
+				alwaysCleanCache = true;
 				this.ui.write(
 					`${os.EOL}${os.EOL}Skipping to Step 3 - Using configuration file: ` + loadConfig + `${os.EOL}`
 				);
@@ -100,7 +82,7 @@ Welcome to the Particle Tachyon setup! This interactive command:
         `if it's interrupted. If you have to kill the CLI, it will pick up where it left. You can also${os.EOL}` +
         "just let it run in the background. We'll wait for you to be ready when its time to flash the device.",
 				4,
-				() => this._download({ region, version })
+				() => this._download({ region, version, alwaysCleanCache })
 			);
 
 			const registrationCode = await this._runStepWithTiming(
@@ -168,6 +150,29 @@ Welcome to the Particle Tachyon setup! This interactive command:
 		}
 	}
 
+	async _showWelcomeMessage() {
+		this.ui.write(`
+===================================================================================
+			  Particle Tachyon Setup Command
+===================================================================================
+
+Welcome to the Particle Tachyon setup! This interactive command:
+
+- Flashes your Tachyon device
+- Configures it (password, WiFi credentials etc...)
+- Connects it to the internet and the Particle Cloud!
+
+**What you'll need:**
+
+1. Your Tachyon device
+2. The Tachyon battery
+3. A USB-C cable
+
+**Important:**
+- This tool requires you to be logged into your Particle account.
+- For more details, check out the documentation at: https://part.cl/setup-tachyon`);
+	}
+
 	async _runStepWithTiming(stepDesc, stepNumber, asyncTask, minDuration = 2000) {
 		this._formatAndDisplaySteps(stepDesc, stepNumber);
 
@@ -189,7 +194,7 @@ Welcome to the Particle Tachyon setup! This interactive command:
 
 	async _formatAndDisplaySteps(text, step) {
 		// Display the formatted step
-		this.ui.write(`${os.EOL}===========================================================${os.EOL}`);
+		this.ui.write(`${os.EOL}===================================================================================${os.EOL}`);
 		this.ui.write(`Step ${step}:${os.EOL}`);
 		this.ui.write(`${text}${os.EOL}`);
 	}
@@ -325,7 +330,7 @@ Welcome to the Particle Tachyon setup! This interactive command:
 		return { systemPassword, wifi, sshPublicKey };
 	}
 
-	async _download({ region, version }) {
+	async _download({ region, version, alwaysCleanCache }) {
 
 		//before downloading a file, we need to check if 'version' is a local file or directory
 		//if it is a local file or directory, we need to return the path to the file
@@ -344,7 +349,7 @@ Welcome to the Particle Tachyon setup! This interactive command:
 		const outputFileName = url.replace(/.*\//, '');
 		const expectedChecksum = artifact.sha256_checksum;
 
-		return manager.download({ url, outputFileName, expectedChecksum });
+		return manager.download({ url, outputFileName, expectedChecksum, options: { alwaysCleanCache } });
 	}
 
 	async _getSystemPassword() {

src/lib/download-manager.js

@@ -54,11 +54,13 @@ class DownloadManager {
 
 	async download({ url, outputFileName, expectedChecksum, options = {} }) {
 		// Check cache
+		const { alwaysCleanCache = false } = options;
 		const cachedFile = await this._getCachedFile(outputFileName, expectedChecksum);
 		if (cachedFile) {
 			this.ui.write(`Using cached file: ${cachedFile}`);
 			return cachedFile;
 		}
+		await this._validateCacheLimit({ url, currentFileName: outputFileName, alwaysCleanCache });
 		const filePath = await this._downloadFile(url, outputFileName, options);
 		// Validate checksum after download completes
 		// Validate checksum after download completes
@@ -96,6 +98,73 @@ class DownloadManager {
 		}
 	}
 
+	async _validateCacheLimit({ url, currentFileName, alwaysCleanCache = false }) {
+		const maxCacheSizeGB = parseFloat(settings.tachyonCacheLimitGB);
+		const fileHeaders = await fetch(url, { method: 'HEAD' });
+
+		const contentLength = parseInt(fileHeaders.headers.get('content-length') || '0', 10);
+		// get the size of the download directory
+		const downloadDirStats = await this.getDownloadedFilesStats(currentFileName);
+		const totalSizeGB = (downloadDirStats.totalSize + contentLength) / (1024 * 1024 * 1024); // convert to GB
+		const downloadedCacheSizeGB = downloadDirStats.totalSize / (1024 * 1024 * 1024); // convert to GB
+		const shouldCleanCache = await this._shouldCleanCache({
+			downloadedCacheSizeGB,
+			alwaysCleanCache,
+			maxCacheSizeGB,
+			totalSizeGB,
+			downloadDirStats
+		});
+		if (shouldCleanCache) {
+			await this._freeCacheSpace(downloadDirStats);
+		}
+		return;
+	}
+
+	async getDownloadedFilesStats(currentFile) {
+		const files = await fs.readdir(this.downloadDir);
+		// use just the zip files and progress files
+		const packageFiles = files.filter(file => file.endsWith('.zip') || file.endsWith('.progress'));
+		// exclude the current file from the list
+		const filteredFiles = currentFile ? packageFiles.filter(file => file !== currentFile && file !== `${currentFile}.progress`) : packageFiles;
+		// stat to get size, and mtime to sort by date modified
+		const fileStats = await Promise.all(filteredFiles.map(async (file) => {
+			const filePath = path.join(this.downloadDir, file);
+			const stats = await fs.stat(filePath);
+			return { filePath, size: stats.size, mtime: stats.mtime };
+		}));
+		// sort files by date modified
+		const sortedFileStats = fileStats.sort((a, b) => a.mtime - b.mtime);
+
+		return {
+			totalSize: sortedFileStats.reduce((sum, file) => sum + file.size, 0),
+			fileStats
+		};
+	}
+
+	async _shouldCleanCache({ downloadedCacheSizeGB, alwaysCleanCache, maxCacheSizeGB, totalSizeGB, downloadDirStats }) {
+		if (maxCacheSizeGB === 0 || alwaysCleanCache) {
+			return true;
+		}
+		if (maxCacheSizeGB === -1) {
+			return false;
+		}
+
+		if (totalSizeGB < maxCacheSizeGB || downloadDirStats.fileStats.length === 0) {
+			return false;
+		}
+		const question = {
+			type: 'confirm',
+			name: 'cleanCache',
+			message: `Do you want to delete previously downloaded versions to free up ${downloadedCacheSizeGB.toFixed(1)} GB of space?`,
+			default: true
+		};
+		const answer = await this.ui.prompt(question);
+		if (!answer.cleanCache) {
+			this.ui.write('Cache cleanup skipped. Remove files manually to free up space.');
+		}
+		return answer.cleanCache;
+	}
+
 	async _attemptDownload(url, outputFileName, progressFilePath, finalFilePath, timeout) {
 		const progressBar = this.ui.createProgressBar();
 		let downloadedBytes = 0;
@@ -118,7 +187,6 @@ class DownloadManager {
 			}
 			await this._streamToFile(response.body, progressFilePath, progressBar, downloadedBytes, timeout, controller);
 			fs.renameSync(progressFilePath, finalFilePath);
-			this.ui.write(`Download completed: ${finalFilePath}`);
 			return finalFilePath;
 		} finally {
 			if (progressBar) {
@@ -219,6 +287,14 @@ class DownloadManager {
 			throw error;
 		}
 	}
+
+	async _freeCacheSpace(downloadDirStats) {
+		const { fileStats } = downloadDirStats;
+		for (const file of fileStats) {
+			const { filePath } = file;
+			await fs.remove(filePath);
+		}
+	}
 }
 
 module.exports = DownloadManager;

src/lib/download-manager.test.js

@@ -18,6 +18,7 @@ describe('DownloadManager', () => {
 
 	afterEach(async () => {
 		sinon.restore();
+		nock.cleanAll();
 		process.env = originalEnv;
 		await fs.remove(path.join(PATH_TMP_DIR, '.particle/downloads'));
 	});
@@ -90,6 +91,9 @@ describe('DownloadManager', () => {
 			const fileContent = 'This is a test file.';
 
 			// Mock the HTTP response
+			nock(url)
+				.head(`/${outputFileName}`)
+				.reply(200);
 			nock(url)
 				.get(`/${outputFileName}`)
 				.reply(200, fileContent);
@@ -113,6 +117,9 @@ describe('DownloadManager', () => {
 			fs.writeFileSync(tempFilePath, initialContent);
 
 			// Mock the HTTP response with a range
+			nock(url)
+				.head(`/${outputFileName}`)
+				.reply(200);
 			nock(url, { reqheaders: { Range: 'bytes=10-' } })
 				.get(`/${outputFileName}`)
 				.reply(206, remainingContent);
@@ -130,6 +137,9 @@ describe('DownloadManager', () => {
 			let error;
 
 			// Mock the HTTP response to simulate a failure
+			nock(url)
+				.head(`/${outputFileName}`)
+				.reply(200);
 			nock(url)
 				.get(`/${outputFileName}`)
 				.reply(500);
@@ -155,6 +165,9 @@ describe('DownloadManager', () => {
 			let error;
 
 			// Mock the HTTP response
+			nock(url)
+				.head(`/${outputFileName}`)
+				.reply(200);
 			nock(url)
 				.get(`/${outputFileName}`)
 				.reply(200, fileContent);
@@ -177,6 +190,9 @@ describe('DownloadManager', () => {
 			const checksum = 'f29bc64a9d3732b4b9035125fdb3285f5b6455778edca72414671e0ca3b2e0de';
 
 			// Mock the HTTP response
+			nock(url)
+				.head(`/${outputFileName}`)
+				.reply(200);
 			nock(url)
 				.get(`/${outputFileName}`)
 				.reply(200, fileContent);