From aa9852d5fe3544124eb4af60ca523d55ef0f8e8b Mon Sep 17 00:00:00 2001 From: microbit-carlos Date: Thu, 26 Oct 2017 16:49:19 +0100 Subject: [PATCH 1/3] Refactor hex code into individual module To keep all things related to the Intel Hex format and the MicroPython details contained in their own modules. There is no functional changes, just the refactor. --- editor.html | 4 +- hexlify.js | 398 ++++++++++++++++++++++++++++++++++++++ python-main.js | 156 +++------------ tests.html | 1 + tests/spec/python-spec.js | 12 +- 5 files changed, 436 insertions(+), 135 deletions(-) create mode 100644 hexlify.js diff --git a/editor.html b/editor.html index 372fb44e..5dbcff88 100644 --- a/editor.html +++ b/editor.html @@ -270,11 +270,13 @@

{{ title }}

- + + diff --git a/hexlify.js b/hexlify.js new file mode 100644 index 00000000..8dc0ca87 --- /dev/null +++ b/hexlify.js @@ -0,0 +1,398 @@ +/** + * This file contains two modules, one generic tool for manipulating data in the + * Intel Hex format, other specific to manipulating micro:bit MicroPython hex + * data. + * + * TODO: Extra tests! Only the existing tests have been updated for the original + * refactor into these modules. Need more for the extra functions. + */ + + +/** + * Generic data manipulation of the Intel Hex format. + * https://en.wikipedia.org/wiki/Intel_HEX + * This module currently only contains the basic functionality required by the + * purpose of the upyhex module below. + */ +var ihexlify = (function() { + 'use strict'; + + /** Start character for each record (line) of the hex. */ + var START_CODE_STR = ':'; + + /** Record types. */ + var RECORD_TYPE_DATA = 0; + var RECORD_END_OF_FILE = 1; + var RECORD_EXT_SEGMENT_ADDR = 2; + var RECORD_START_SEGMENT_ADDR = 3; + var RECORD_EXT_LINEAR_ADDR = 4; + var RECORD_START_LINEAR_ADDR = 5; + + /** Position of the record fields in a Intel Hex string. */ + var STR_IND_START_CODE = 0; + var STR_IND_BYTE_CNT = 1; + var STR_IND_ADDR = 3; + var STR_IND_RECORD_TYPE = 7; + var STR_IND_DATA = 9; + + /** Position of the record fields in a Intel Hex byte array. */ + var BYTE_IND_BYTE_CNT = 0; + var BYTE_IND_ADDR_HIGH = 1; + var BYTE_IND_ADDR_LOW = 2; + var BYTE_IND_RECORD_TYPE = 3; + var BYTE_IND_DATA = 4; + + /** Sizes in bytes and characters for each record structure field. */ + var SIZE_C_START_CODE = START_CODE_STR.length; + var SIZE_B_BYTE_CNT = 1; + var SIZE_C_BYTE_CNT = SIZE_B_BYTE_CNT * 2; + var SIZE_B_ADDR = 2; + var SIZE_C_ADDR = SIZE_B_ADDR * 2; + var SIZE_B_RECORD_TYPE = 1; + var SIZE_C_RECORD_TYPE = SIZE_B_RECORD_TYPE * 2; + var SIZE_B_CHECKSUM = 1; + var SIZE_C_CHECKSUM = SIZE_B_CHECKSUM * 2; + + /** Bytes between START_CODE and the data fields. */ + var SIZE_B_PRE_DATA = SIZE_B_BYTE_CNT + SIZE_B_ADDR + SIZE_B_RECORD_TYPE; + + /** Size, in bytes and characters, of fields after the data field. */ + var SIZE_B_POS_DATA = SIZE_B_CHECKSUM; + var SIZE_C_POS_DATA = SIZE_C_CHECKSUM; + + /** + * Converts each byte in a an array into a single concatenated hex string. + * @param {Uint8Array|Object[]} byteArray - Bytes to convert into hex str. + * @return {string} The hex string form from the input data. + */ + function bytesToHexStr(byteArray) { + var result = ''; + for (var i = 0; i < byteArray.length; ++i) { + if (byteArray[i] < 16) { + result += '0'; + } + result += byteArray[i].toString(16); + } + return result.toUpperCase(); + } + + /** + * Converts an string of hex data (not intel hex) into an array of bytes. + * @param {string} hexStr - String of plain hex data. + * @return {Uint8Array} Array of bytes with the data. + */ + function hexStrToBytes(hexStr) { + // String has to have an even number of characters + if (hexStr.length % 2 !== 0) { + throw new RangeError('Hex str to parse has odd number of chars.'); + } + var result = new Uint8Array(hexStr.length / 2); + for (var i = 0; i < result.length; i++) { + result[i] = parseInt(hexStr.substr(i * 2, 2), 16); + } + return result; + } + + /** + * Calculates checksum by taking the LSB of the 2's complement (negative) of + * the sum of all bytes in the array, or up to the delimiter. + * @param {Uint8Array|Object[]} byteArray - Array with single byte per item. + * @param {number} [byteCnt] - Optional length of array items to checksum. + * @return {number} Intel Hex checksum for a record. + */ + function checksum(byteArray, byteCnt) { + var checksum = 0; + byteCnt = byteCnt || byteArray.length; + for (var i = 0; i < byteCnt; ++i) { + checksum += byteArray[i]; + } + return (-checksum) & 0xff; + } + + /** + * Validates if the input string is a valid single Intel Hex record. + * TODO: This is a very basic check for start character and hex value, could + * be expanded to validate valid fields like byte count matches data + * size, or the checksum at the end. + * @param {string} intelHexRecord - String containing a single record (line) + * of Intel Hex. + * @return {boolean} True if the string is a valid Intel Hex record. + */ + function isRecordValid(intelHexRecord) { + if (intelHexRecord[0] !== START_CODE_STR) { + return false; + } + return intelHexRecord.slice(1).match(/^[a-f0-9]/i) !== null; + } + + /** + * Extracts the Record Type field (as a string) from a Intel Hex record. + * @param {string} intelHexRecord - Intel Hex string. + * @return {?string} Hex string with the Record Type. + */ + function getRecordTypeHexStr(intelHexRecord) { + if (!isRecordValid(intelHexRecord)) return null; + return intelHexRecord.substr(STR_IND_RECORD_TYPE, SIZE_C_RECORD_TYPE); + } + + /** + * Extracts the Record Type field (as a number) from a Intel Hex record. + * @param {string} intelHexRecord - Intel Hex string. + * @return {?number} Number representing the Record Type. + */ + function getRecordType(intelHexRecord) { + var typeStr = getRecordTypeHexStr(intelHexRecord); + var typeNumber = parseInt(typeStr, 16); + if (isNaN(typeNumber)) { + typeNumber = null; + } + return typeNumber; + } + + /** + * Extracts the Data field (as a string) from a single Intel Hex record. + * @param {string} intelHexRecord - Intel Hex string. + * @return {?string} Hex string with the Record Type. + */ + function getRecordDataHexStr(intelHexRecord) { + if (!isRecordValid(intelHexRecord)) return null; + return intelHexRecord.slice(STR_IND_DATA, -(SIZE_C_POS_DATA)); + } + + /** + * Converts a byte array into a string of Intel Hex. The start address and + * amount of bytes per record are also required. + * @param {number} addr - Start address for the data. + * @param {number} byteCnt - Number of bytes per Intel Hex record (line). + * @param {Uint8Array|Object[]} byteArray - Data in an array of bytes. + * @return {string} Intel Hex string. + */ + function bytesToIntelHexStr(addr, byteCnt, byteArray) { + // Byte count has to be a number that fits in 2 hex digits + if (!(byteCnt <= 0xFF)) { + throw new RangeError('Invalid Byte Count'); + } + // Array size needs to fit all the record fields in addition to the data + var chunk = (new Uint8Array( + SIZE_B_PRE_DATA + byteCnt + SIZE_B_POS_DATA)).fill(0xFF); + var output = []; + for (var i = 0; i < byteArray.length; i += byteCnt, addr += byteCnt) { + // Fill beginning of record structure + chunk[BYTE_IND_BYTE_CNT] = byteCnt; + chunk[BYTE_IND_ADDR_HIGH] = (addr >> 8) & 0xff; // MSB 16-bit addr + chunk[BYTE_IND_ADDR_LOW] = addr & 0xff; // LSB 16-bit addr + chunk[BYTE_IND_RECORD_TYPE] = RECORD_TYPE_DATA; // record type + // Add the input data + for (var j = 0; j < byteCnt; ++j) { + chunk[BYTE_IND_DATA + j] = byteArray[i + j]; + } + // Calculate and add checksum as last byte + chunk[chunk.length-1] = checksum(chunk, SIZE_B_PRE_DATA + byteCnt); + // Form record into string format and add it to output + output.push(START_CODE_STR + bytesToHexStr(chunk)); + } + return output.join('\n'); + } + + /** + * Converts a string of Intel Hex into a byte array with only the data. + * TODO: This function only supports converting data records, should be + * expanded to deal with the other record types, specially those that + * change the addressing. + * @param {string} intelHexStr - String with Intel Hex records. + * @return {Uint8Array[]} Array of "byte arrays" with only the data. An + * array item per Intel Hex record, each being a Uint8Array with the + * data. + */ + function intelHexStrToBytes(intelHexStr) { + var lines = intelHexStr.trimRight().split(/\r?\n/); + if (lines.length <= 0) { + return ''; + } + var dataBytes = []; + for (var i = 0; i < lines.length; i++) { + var recordType = getRecordType(lines[i]); + if (recordType !== RECORD_TYPE_DATA) { + throw new Error('A record in line ' + i + ' of the Intel Hex ' + + 'string is not of the "data" type'); + } + var dataOnlyStr = getRecordDataHexStr(lines[i]); + dataBytes.push(hexStrToBytes(dataOnlyStr)); + } + // TODO: Append all Uint8Array from dataBytes into a single Uint8Array + return dataBytes; + } + + /** + * Creates an Intel Hex Extended Linear Address record for a given address. + * @param {number} addr - Address to create the record. + * @return {string} Intel Hex Extended Linear Address record. + */ + function createExtLinearAddrRecord(addr) { + // The size of the data field for a Extended Linear Address is 2 bytes + var byteCnt = 2; + var recordBytes = + new Uint8Array(SIZE_B_PRE_DATA + byteCnt + SIZE_B_POS_DATA); + // Fill beginning of record structure + recordBytes[BYTE_IND_BYTE_CNT] = byteCnt; + recordBytes[BYTE_IND_ADDR_HIGH] = 0; // address field ignored + recordBytes[BYTE_IND_ADDR_LOW] = 0; // typically 00 00 + recordBytes[BYTE_IND_RECORD_TYPE] = RECORD_EXT_LINEAR_ADDR; + // Only take the 2 MSB of the Big Ending 4 bytes address + recordBytes[BYTE_IND_DATA] = (addr >> 24) & 0xff; + recordBytes[BYTE_IND_DATA + 1] = (addr >> 16) & 0xff; + // Calculate and add checksum as last byte + recordBytes[recordBytes.length - 1] = + checksum(recordBytes, SIZE_B_PRE_DATA + byteCnt); + return START_CODE_STR + bytesToHexStr(recordBytes); + } + + return { + bytesToIntelHexStr: bytesToIntelHexStr, + intelHexStrToBytes: intelHexStrToBytes, + createExtLinearAddrRecord: createExtLinearAddrRecord + }; +}()); + + +/** + * Module to add and remove Python scripts into and from a MicroPython hex. + */ +var upyhex = (function() { + 'use strict'; + + /** Start of user script marked by "MP" + 2 bytes for the script length. */ + var USER_SCRIPT_START_BYTE_0 = 77; // 'M' + var USER_SCRIPT_START_BYTE_1 = 80; // 'P' + + /** User script located at specific flash address. */ + var USER_SCRIPT_START_ADDR = 0x3e000; + + /** User script header size. */ + var USER_SCRIPT_HEADER_CHARS = 4; + + /** When user script added. */ + var USER_SCRIPT_MAX_LENGTH = 8192; + + /** Number of data bytes per Intel Hex record (line). */ + var INTEL_HEX_BYTE_CNT = 16; + + /** + * String placed inside the MicroPython hex string to indicate where to + * paste the Python Code + * */ + var HEX_INSERTION_POINT = ":::::::::::::::::::::::::::::::::::::::::::"; + + /** + * Converts a user Python script into the Intel Hex format with the header + * and the address expected by MicroPython and configured for the micro:bit + * hex file format. + * @param {string} pyStr - Python code to convert. + * @return {string} Code in Intel Hex string formatted for the micro:bit. + */ + function pyStrToIntelHex(pyStr) { + // Add header to the script size + var dataLength = USER_SCRIPT_HEADER_CHARS + pyStr.length; + // Add padding for the data field size in the Intel Hex records + dataLength += INTEL_HEX_BYTE_CNT - (dataLength % INTEL_HEX_BYTE_CNT); + // Check the data block fits in the allocated flash area + if (dataLength > USER_SCRIPT_MAX_LENGTH) { + throw new RangeError('Too long'); + } + // The user script has to start with "MP" marker + script length + var data = new Uint8Array(dataLength); + data[0] = USER_SCRIPT_START_BYTE_0; + data[1] = USER_SCRIPT_START_BYTE_1; + data[2] = pyStr.length & 0xff; + data[3] = (pyStr.length >> 8) & 0xff; + for (var i = 0; i < pyStr.length; ++i) { + data[4 + i] = pyStr.charCodeAt(i); + } + // Convert to Intel Hex format + return ihexlify.bytesToIntelHexStr( + USER_SCRIPT_START_ADDR, INTEL_HEX_BYTE_CNT, data); + } + + /** + * Converts a byte array into a string of characters. + * TODO: This currently only deals with single byte characters, so needs to + * be expanded to support UTF-8 characters longer than 1 byte. + * @param {Uint8Array|Object[]} byteArray - Array of bytes to convert. + * @return {string} String output from the conversion. + */ + function bytesToStr(byteArray) { + var result = []; + for (var i = 0; i < byteArray.length; i++) { + result.push(String.fromCharCode(byteArray[i])); + } + return result.join(''); + } + + /** + * Takes a block if Intel Hex records, extracts the data, discards the + * headers required by MicroPython and converts the rest into a string. + * TODO: This function takes advantage of the way the return value from + * ihexlify.intelHexStrToBytes() is formatted (an array of Uint8Array) + * so will need to be completely rewritten if that function is updated + * to return a single Uint8Array with all the data. + * @param {string} intelHexStr - Intel Hex block to scan for the code. + * @return {strin} The Python code. + */ + function intelHexToPyStr(intelHexStr) { + // Convert the Intel Hex block into an array of Uint8Arrays, one item + // per record + var output = ihexlify.intelHexStrToBytes(intelHexStr); + // Convert the bytes from each record into a string + for (var i = 0; i < output.length; i++) { + output[i] = bytesToStr(output[i]); + } + // Discard the header from the beginning, and clean the null terminator + output[0] = output[0].slice(4); + var last = output.length - 1; + output[last] = output[last].replace(/\0/g, ''); + return output.join(''); + } + + /** + * Parses through an Intel Hex string to find the Python code at the + * allocated address and extracts it. + * @param {string} intelHexStr - Intel Hex block to scan for the code. + * @return {string} Python code. + */ + function extractPyStrFromIntelHex(intelHexStr) { + var hex_lines = intelHexStr.trimRight().split(/\r?\n/); + var extAddrRecord = + ihexlify.createExtLinearAddrRecord(USER_SCRIPT_START_ADDR); + var start_line = hex_lines.lastIndexOf(extAddrRecord); + if (start_line > 0) { + var lines = hex_lines.slice(start_line + 1, -2); + var blob = lines.join('\n'); + if (blob === '') { + return ''; + } else { + return intelHexToPyStr(blob); + } + } else { + return ''; + } + } + + /** + * Converts the Python code into the Intel Hex format expected by + * MicroPython and injects it into a Intel Hex string containing a marker. + * @param {string} intelHexStr - Intel Hex block to inject the code. + * @param {string} pyStr - Python code string. + * @return {string} Intel Hex string with the Python code injected. + */ + function injectPyStrIntoIntelHex(intelHexStr, pyStr) { + var pyCodeIntelHex = pyStrToIntelHex(pyStr); + return intelHexStr.replace(HEX_INSERTION_POINT, pyCodeIntelHex); + } + + return { + pyStrToIntelHex: pyStrToIntelHex, + intelHexToPyStr: intelHexToPyStr, + extractPyStrFromIntelHex: extractPyStrFromIntelHex, + injectPyStrIntoIntelHex: injectPyStrIntoIntelHex, + }; +}()); diff --git a/python-main.js b/python-main.js index 2d18be8f..7c7cd0a6 100644 --- a/python-main.js +++ b/python-main.js @@ -19,7 +19,7 @@ function pythonEditor(id) { var ACE = ace.edit(id); // The editor is in the tag with the referenced id. ACE.setOptions({ enableSnippets: true // Enable code snippets. - }) + }); ACE.setTheme("ace/theme/kr_theme"); // Make it look nice. ACE.getSession().setMode("ace/mode/python"); // We're editing Python. ACE.getSession().setTabSize(4); // Tab=4 spaces. @@ -40,7 +40,7 @@ function pythonEditor(id) { // Give the editor user input focus. editor.focus = function() { ACE.focus(); - } + }; // Set a handler function to be run if code in the editor changes. editor.on_change = function(handler) { @@ -62,110 +62,10 @@ function pythonEditor(id) { } }; - /* - Turn a Python script into Intel HEX format to be concatenated at the - end of the MicroPython firmware.hex. A simple header is added to the - script. - - - takes a Python script as a string - - returns hexlified string, with newlines between lines - */ - editor.hexlify = function(script) { - function hexlify(ar) { - var result = ''; - for (var i = 0; i < ar.length; ++i) { - if (ar[i] < 16) { - result += '0'; - } - result += ar[i].toString(16); - } - return result; - } - // add header, pad to multiple of 16 bytes - data = new Uint8Array(4 + script.length + (16 - (4 + script.length) % 16)); - data[0] = 77; // 'M' - data[1] = 80; // 'P' - data[2] = script.length & 0xff; - data[3] = (script.length >> 8) & 0xff; - for (var i = 0; i < script.length; ++i) { - data[4 + i] = script.charCodeAt(i); - } - // check data.length < 0x2000 - if(data.length > 8192) { - throw new RangeError('Too long'); - } - // convert to .hex format - var addr = 0x3e000; // magic start address in flash - var chunk = new Uint8Array(5 + 16); - var output = []; - for (var i = 0; i < data.length; i += 16, addr += 16) { - chunk[0] = 16; // length of data section - chunk[1] = (addr >> 8) & 0xff; // high byte of 16-bit addr - chunk[2] = addr & 0xff; // low byte of 16-bit addr - chunk[3] = 0; // type (data) - for (var j = 0; j < 16; ++j) { - chunk[4 + j] = data[i + j]; - } - var checksum = 0; - for (var j = 0; j < 4 + 16; ++j) { - checksum += chunk[j]; - } - chunk[4 + 16] = (-checksum) & 0xff; - output.push(':' + hexlify(chunk).toUpperCase()) - } - return output.join('\n'); - }; - // Generates a hex file containing the user's Python from the firmware. editor.getHexFile = function(firmware) { - var hexlified_python = this.hexlify(this.getCode()); - var insertion_point = ":::::::::::::::::::::::::::::::::::::::::::"; - return firmware.replace(insertion_point, hexlified_python); - } - - // Takes a hex blob and turns it into a decoded string. - editor.unhexlify = function(data) { - - var hex2str = function(str) { - var result = ''; - for (var i=0, l=str.length; i 0) { - var output = []; - for (var i=0; i 0) { - var lines = hex_lines.slice(start_line + 1, -5); - var blob = lines.join('\n'); - if (blob=='') { - return ''; - } else { - return this.unhexlify(blob); - } - } else { - return ''; - } - } + return upyhex.injectPyStrIntoIntelHex(firmware, this.getCode()); + }; // Given a password and some plaintext, will return an encrypted version. editor.encrypt = function(password, plaintext) { @@ -185,7 +85,7 @@ function pythonEditor(id) { output.putBytes(salt); output.putBuffer(cipher.output); return encodeURIComponent(btoa(output.getBytes())); - } + }; // Given a password and cyphertext will return the decrypted plaintext. editor.decrypt = function(password, cyphertext) { @@ -204,10 +104,10 @@ function pythonEditor(id) { decipher.update(input); var result = decipher.finish(); return decipher.output.getBytes(); - } + }; return editor; -}; +} /* The following code contains the various functions that connect the behaviour of @@ -267,7 +167,7 @@ function web_editor(config) { if (workspace && continueZooming) { Blockly.getMainWorkspace().zoomCenter(1); } - }; + } // Sets up the zoom-out functionality. function zoomOut() { @@ -286,7 +186,7 @@ function web_editor(config) { if (workspace && continueZooming) { Blockly.getMainWorkspace().zoomCenter(-1); } - }; + } // Checks for feature flags in the config object and shows/hides UI // elements as required. @@ -300,7 +200,7 @@ function web_editor(config) { if(config.flags.share) { $("#command-share").removeClass('hidden'); } - }; + } // This function is called to initialise the editor. It sets things up so // the user sees their code or, in the case of a new program, uses some @@ -319,7 +219,7 @@ function web_editor(config) { } vex.open({ content: Mustache.render(template, context) - }) + }); $('#button-decrypt-link').click(function() { var password = $('#passphrase').val(); setName(EDITOR.decrypt(password, message.n)); @@ -335,7 +235,7 @@ function web_editor(config) { EDITOR.focus(); } else { // If there's no name, default to something sensible. - setName("microbit") + setName("microbit"); // If there's no description, default to something sensible. setDescription("A MicroPython script"); // A sane default starting point for a new script. @@ -344,7 +244,7 @@ function web_editor(config) { EDITOR.ACE.gotoLine(EDITOR.ACE.session.getLength()); // If configured as experimental update editor background to indicate it if(config.flags.experimental) { - EDITOR.ACE.renderer.scroller.style.backgroundImage = "url('static/img/experimental.png')" + EDITOR.ACE.renderer.scroller.style.backgroundImage = "url('static/img/experimental.png')"; } // Configure the zoom related buttons. $("#zoom-in").click(function (e) { @@ -466,18 +366,19 @@ function web_editor(config) { setDescription(config.translate.drop.python); reader.onload = function(e) { EDITOR.setCode(e.target.result); - } + }; reader.readAsText(f); EDITOR.ACE.gotoLine(EDITOR.ACE.session.getLength()); } else if (ext == 'hex') { setName(f.name.replace('.hex', '')); setDescription(config.translate.drop.hex); reader.onload = function(e) { - var code = EDITOR.extractScript(e.target.result); + var code = upyhex.extractPyStrFromIntelHex( + e.target.result); if(code.length < 8192) { EDITOR.setCode(code); } - } + }; reader.readAsText(f); EDITOR.ACE.gotoLine(EDITOR.ACE.session.getLength()); } @@ -487,7 +388,7 @@ function web_editor(config) { return false; }); } - }) + }); $('.load-toggle').on('click', function(e) { $('.load-drag-target').toggle(); $('.load-form').toggle(); @@ -550,7 +451,7 @@ function web_editor(config) { } // Set editor to current state of blocks. EDITOR.setCode(Blockly.Python.workspaceToCode(workspace)); - }; + } } // This function describes what to do when the snippets button is clicked. @@ -572,9 +473,9 @@ function web_editor(config) { var name = render(text); var trigger = name.split(' - ')[0]; return config.translate.code_snippets[trigger]; - } + }; } - } + }; vex.open({ content: Mustache.render(template, context), afterOpen: function(vexContent) { @@ -594,7 +495,7 @@ function web_editor(config) { Mustache.parse(template); vex.open({ content: Mustache.render(template, config.translate.share) - }) + }); $('#passphrase').focus(); $('#button-create-link').click(function() { var password = $('#passphrase').val(); @@ -642,18 +543,18 @@ function web_editor(config) { setDescription(config.translate.drop.python); reader.onload = function(e) { EDITOR.setCode(e.target.result); - } + }; reader.readAsText(file); EDITOR.ACE.gotoLine(EDITOR.ACE.session.getLength()); } else if (ext == 'hex') { setName(file.name.replace('.hex', '')); setDescription(config.translate.drop.hex); reader.onload = function(e) { - var code = EDITOR.extractScript(e.target.result); + var code = upyhex.extractPyStrFromIntelHex(e.target.result); if (code.length < 8192) { EDITOR.setCode(code); } - } + }; reader.readAsText(file); EDITOR.ACE.gotoLine(EDITOR.ACE.session.getLength()); } @@ -735,7 +636,7 @@ function web_editor(config) { Mustache.parse(template); var context = config.translate.messagebar; var messagebar = $('#messagebar'); - messagebar.html(Mustache.render(template, context)) + messagebar.html(Mustache.render(template, context)); messagebar.show(); $('#messagebar-link').attr('href', window.location.href.replace(VERSION, data.latest)); @@ -746,11 +647,10 @@ function web_editor(config) { }); } - var qs = get_qs_context() + var qs = get_qs_context(); var migration = get_migration(); setupFeatureFlags(); setupEditor(qs, migration); checkVersion(qs); setupButtons(); -}; - +} diff --git a/tests.html b/tests.html index aab7ca77..1b83dbc2 100644 --- a/tests.html +++ b/tests.html @@ -16,6 +16,7 @@ + diff --git a/tests/spec/python-spec.js b/tests/spec/python-spec.js index 9f3c561b..f9eacb0f 100644 --- a/tests/spec/python-spec.js +++ b/tests/spec/python-spec.js @@ -143,7 +143,7 @@ describe("An editor for MicroPython on the BBC micro:bit:", function() { }); it("The editor converts text into Intel's hex format.", function() { - var hexified = editor.hexlify('display.scroll("Hello")'); + var hexified = upyhex.pyStrToIntelHex('display.scroll("Hello")'); var expected = ':10E000004D501700646973706C61792E7363726F81\n' + ':10E010006C6C282248656C6C6F222900000000009F'; expect(hexified).toEqual(expected); @@ -151,13 +151,13 @@ describe("An editor for MicroPython on the BBC micro:bit:", function() { it("The editor complains if the Python script is greater than 8k in length.", function() { var hex_fail = function() { - var result = editor.hexlify(new Array(8189).join('a')); + var result = upyhex.pyStrToIntelHex(new Array(8189).join('a')); } expect(hex_fail).toThrowError(RangeError, 'Too long'); }); it("The editor is fine if the Python script is 8k in length.", function() { - var hexified = editor.hexlify(new Array(8188).join('a')); + var hexified = upyhex.pyStrToIntelHex(new Array(8188).join('a')); expect(hexified).not.toBe(null); }); @@ -189,7 +189,7 @@ describe("An editor for MicroPython on the BBC micro:bit:", function() { it("The editor converts from Intel's hex format to text", function() { var raw_hex = ":10E000004D501700646973706C61792E7363726F81\n" + ":10E010006C6C282248656C6C6F222900000000009F\n"; - var result = editor.unhexlify(raw_hex); + var result = upyhex.intelHexToPyStr(raw_hex); var expected = 'display.scroll("Hello")'; expect(result).toEqual(expected); }); @@ -204,7 +204,7 @@ describe("An editor for MicroPython on the BBC micro:bit:", function() { ":10E010006C6C282248656C6C6F222900000000009F\n" + ":04000005000153EDB6\n" + ":00000001FF"; - var result = editor.extractScript(raw_hex); + var result = upyhex.extractPyStrFromIntelHex(raw_hex); var expected = 'display.scroll("Hello")'; expect(result).toEqual(expected); }); @@ -217,7 +217,7 @@ describe("An editor for MicroPython on the BBC micro:bit:", function() { ":10E010006C6C282248656C6C6F222900000000009F\n" + ":04000005000153EDB6\n" + ":00000001FF"; - var result = editor.extractScript(raw_hex); + var result = upyhex.extractPyStrFromIntelHex(raw_hex); var expected = ''; expect(result).toEqual(expected); }); From 9e6047cbbbf53e0fc378aeb3c0bb19fa8421d4bc Mon Sep 17 00:00:00 2001 From: microbit-carlos Date: Tue, 8 Jan 2019 18:31:05 +0000 Subject: [PATCH 2/3] Replace Intel Hex manipulation code with nrf-intel-hex library. --- editor.html | 1 + hexlify.js | 393 ++-------- static/js/intel-hex.browser.js | 1249 ++++++++++++++++++++++++++++++++ tests.html | 1 + tests/spec/python-spec.js | 41 +- 5 files changed, 1332 insertions(+), 353 deletions(-) create mode 100644 static/js/intel-hex.browser.js diff --git a/editor.html b/editor.html index 5dbcff88..2ada46a4 100644 --- a/editor.html +++ b/editor.html @@ -253,6 +253,7 @@

{{ title }}

+ diff --git a/hexlify.js b/hexlify.js index 8dc0ca87..56801516 100644 --- a/hexlify.js +++ b/hexlify.js @@ -1,316 +1,47 @@ -/** - * This file contains two modules, one generic tool for manipulating data in the - * Intel Hex format, other specific to manipulating micro:bit MicroPython hex - * data. - * - * TODO: Extra tests! Only the existing tests have been updated for the original - * refactor into these modules. Need more for the extra functions. - */ - - -/** - * Generic data manipulation of the Intel Hex format. - * https://en.wikipedia.org/wiki/Intel_HEX - * This module currently only contains the basic functionality required by the - * purpose of the upyhex module below. - */ -var ihexlify = (function() { - 'use strict'; - - /** Start character for each record (line) of the hex. */ - var START_CODE_STR = ':'; - - /** Record types. */ - var RECORD_TYPE_DATA = 0; - var RECORD_END_OF_FILE = 1; - var RECORD_EXT_SEGMENT_ADDR = 2; - var RECORD_START_SEGMENT_ADDR = 3; - var RECORD_EXT_LINEAR_ADDR = 4; - var RECORD_START_LINEAR_ADDR = 5; - - /** Position of the record fields in a Intel Hex string. */ - var STR_IND_START_CODE = 0; - var STR_IND_BYTE_CNT = 1; - var STR_IND_ADDR = 3; - var STR_IND_RECORD_TYPE = 7; - var STR_IND_DATA = 9; - - /** Position of the record fields in a Intel Hex byte array. */ - var BYTE_IND_BYTE_CNT = 0; - var BYTE_IND_ADDR_HIGH = 1; - var BYTE_IND_ADDR_LOW = 2; - var BYTE_IND_RECORD_TYPE = 3; - var BYTE_IND_DATA = 4; - - /** Sizes in bytes and characters for each record structure field. */ - var SIZE_C_START_CODE = START_CODE_STR.length; - var SIZE_B_BYTE_CNT = 1; - var SIZE_C_BYTE_CNT = SIZE_B_BYTE_CNT * 2; - var SIZE_B_ADDR = 2; - var SIZE_C_ADDR = SIZE_B_ADDR * 2; - var SIZE_B_RECORD_TYPE = 1; - var SIZE_C_RECORD_TYPE = SIZE_B_RECORD_TYPE * 2; - var SIZE_B_CHECKSUM = 1; - var SIZE_C_CHECKSUM = SIZE_B_CHECKSUM * 2; - - /** Bytes between START_CODE and the data fields. */ - var SIZE_B_PRE_DATA = SIZE_B_BYTE_CNT + SIZE_B_ADDR + SIZE_B_RECORD_TYPE; - - /** Size, in bytes and characters, of fields after the data field. */ - var SIZE_B_POS_DATA = SIZE_B_CHECKSUM; - var SIZE_C_POS_DATA = SIZE_C_CHECKSUM; - - /** - * Converts each byte in a an array into a single concatenated hex string. - * @param {Uint8Array|Object[]} byteArray - Bytes to convert into hex str. - * @return {string} The hex string form from the input data. - */ - function bytesToHexStr(byteArray) { - var result = ''; - for (var i = 0; i < byteArray.length; ++i) { - if (byteArray[i] < 16) { - result += '0'; - } - result += byteArray[i].toString(16); - } - return result.toUpperCase(); - } - - /** - * Converts an string of hex data (not intel hex) into an array of bytes. - * @param {string} hexStr - String of plain hex data. - * @return {Uint8Array} Array of bytes with the data. - */ - function hexStrToBytes(hexStr) { - // String has to have an even number of characters - if (hexStr.length % 2 !== 0) { - throw new RangeError('Hex str to parse has odd number of chars.'); - } - var result = new Uint8Array(hexStr.length / 2); - for (var i = 0; i < result.length; i++) { - result[i] = parseInt(hexStr.substr(i * 2, 2), 16); - } - return result; - } - - /** - * Calculates checksum by taking the LSB of the 2's complement (negative) of - * the sum of all bytes in the array, or up to the delimiter. - * @param {Uint8Array|Object[]} byteArray - Array with single byte per item. - * @param {number} [byteCnt] - Optional length of array items to checksum. - * @return {number} Intel Hex checksum for a record. - */ - function checksum(byteArray, byteCnt) { - var checksum = 0; - byteCnt = byteCnt || byteArray.length; - for (var i = 0; i < byteCnt; ++i) { - checksum += byteArray[i]; - } - return (-checksum) & 0xff; - } - - /** - * Validates if the input string is a valid single Intel Hex record. - * TODO: This is a very basic check for start character and hex value, could - * be expanded to validate valid fields like byte count matches data - * size, or the checksum at the end. - * @param {string} intelHexRecord - String containing a single record (line) - * of Intel Hex. - * @return {boolean} True if the string is a valid Intel Hex record. - */ - function isRecordValid(intelHexRecord) { - if (intelHexRecord[0] !== START_CODE_STR) { - return false; - } - return intelHexRecord.slice(1).match(/^[a-f0-9]/i) !== null; - } - - /** - * Extracts the Record Type field (as a string) from a Intel Hex record. - * @param {string} intelHexRecord - Intel Hex string. - * @return {?string} Hex string with the Record Type. - */ - function getRecordTypeHexStr(intelHexRecord) { - if (!isRecordValid(intelHexRecord)) return null; - return intelHexRecord.substr(STR_IND_RECORD_TYPE, SIZE_C_RECORD_TYPE); - } - - /** - * Extracts the Record Type field (as a number) from a Intel Hex record. - * @param {string} intelHexRecord - Intel Hex string. - * @return {?number} Number representing the Record Type. - */ - function getRecordType(intelHexRecord) { - var typeStr = getRecordTypeHexStr(intelHexRecord); - var typeNumber = parseInt(typeStr, 16); - if (isNaN(typeNumber)) { - typeNumber = null; - } - return typeNumber; - } - - /** - * Extracts the Data field (as a string) from a single Intel Hex record. - * @param {string} intelHexRecord - Intel Hex string. - * @return {?string} Hex string with the Record Type. - */ - function getRecordDataHexStr(intelHexRecord) { - if (!isRecordValid(intelHexRecord)) return null; - return intelHexRecord.slice(STR_IND_DATA, -(SIZE_C_POS_DATA)); - } - - /** - * Converts a byte array into a string of Intel Hex. The start address and - * amount of bytes per record are also required. - * @param {number} addr - Start address for the data. - * @param {number} byteCnt - Number of bytes per Intel Hex record (line). - * @param {Uint8Array|Object[]} byteArray - Data in an array of bytes. - * @return {string} Intel Hex string. - */ - function bytesToIntelHexStr(addr, byteCnt, byteArray) { - // Byte count has to be a number that fits in 2 hex digits - if (!(byteCnt <= 0xFF)) { - throw new RangeError('Invalid Byte Count'); - } - // Array size needs to fit all the record fields in addition to the data - var chunk = (new Uint8Array( - SIZE_B_PRE_DATA + byteCnt + SIZE_B_POS_DATA)).fill(0xFF); - var output = []; - for (var i = 0; i < byteArray.length; i += byteCnt, addr += byteCnt) { - // Fill beginning of record structure - chunk[BYTE_IND_BYTE_CNT] = byteCnt; - chunk[BYTE_IND_ADDR_HIGH] = (addr >> 8) & 0xff; // MSB 16-bit addr - chunk[BYTE_IND_ADDR_LOW] = addr & 0xff; // LSB 16-bit addr - chunk[BYTE_IND_RECORD_TYPE] = RECORD_TYPE_DATA; // record type - // Add the input data - for (var j = 0; j < byteCnt; ++j) { - chunk[BYTE_IND_DATA + j] = byteArray[i + j]; - } - // Calculate and add checksum as last byte - chunk[chunk.length-1] = checksum(chunk, SIZE_B_PRE_DATA + byteCnt); - // Form record into string format and add it to output - output.push(START_CODE_STR + bytesToHexStr(chunk)); - } - return output.join('\n'); - } - - /** - * Converts a string of Intel Hex into a byte array with only the data. - * TODO: This function only supports converting data records, should be - * expanded to deal with the other record types, specially those that - * change the addressing. - * @param {string} intelHexStr - String with Intel Hex records. - * @return {Uint8Array[]} Array of "byte arrays" with only the data. An - * array item per Intel Hex record, each being a Uint8Array with the - * data. - */ - function intelHexStrToBytes(intelHexStr) { - var lines = intelHexStr.trimRight().split(/\r?\n/); - if (lines.length <= 0) { - return ''; - } - var dataBytes = []; - for (var i = 0; i < lines.length; i++) { - var recordType = getRecordType(lines[i]); - if (recordType !== RECORD_TYPE_DATA) { - throw new Error('A record in line ' + i + ' of the Intel Hex ' + - 'string is not of the "data" type'); - } - var dataOnlyStr = getRecordDataHexStr(lines[i]); - dataBytes.push(hexStrToBytes(dataOnlyStr)); - } - // TODO: Append all Uint8Array from dataBytes into a single Uint8Array - return dataBytes; - } - - /** - * Creates an Intel Hex Extended Linear Address record for a given address. - * @param {number} addr - Address to create the record. - * @return {string} Intel Hex Extended Linear Address record. - */ - function createExtLinearAddrRecord(addr) { - // The size of the data field for a Extended Linear Address is 2 bytes - var byteCnt = 2; - var recordBytes = - new Uint8Array(SIZE_B_PRE_DATA + byteCnt + SIZE_B_POS_DATA); - // Fill beginning of record structure - recordBytes[BYTE_IND_BYTE_CNT] = byteCnt; - recordBytes[BYTE_IND_ADDR_HIGH] = 0; // address field ignored - recordBytes[BYTE_IND_ADDR_LOW] = 0; // typically 00 00 - recordBytes[BYTE_IND_RECORD_TYPE] = RECORD_EXT_LINEAR_ADDR; - // Only take the 2 MSB of the Big Ending 4 bytes address - recordBytes[BYTE_IND_DATA] = (addr >> 24) & 0xff; - recordBytes[BYTE_IND_DATA + 1] = (addr >> 16) & 0xff; - // Calculate and add checksum as last byte - recordBytes[recordBytes.length - 1] = - checksum(recordBytes, SIZE_B_PRE_DATA + byteCnt); - return START_CODE_STR + bytesToHexStr(recordBytes); - } - - return { - bytesToIntelHexStr: bytesToIntelHexStr, - intelHexStrToBytes: intelHexStrToBytes, - createExtLinearAddrRecord: createExtLinearAddrRecord - }; -}()); - - /** * Module to add and remove Python scripts into and from a MicroPython hex. */ var upyhex = (function() { 'use strict'; - /** Start of user script marked by "MP" + 2 bytes for the script length. */ - var USER_SCRIPT_START_BYTE_0 = 77; // 'M' - var USER_SCRIPT_START_BYTE_1 = 80; // 'P' - /** User script located at specific flash address. */ - var USER_SCRIPT_START_ADDR = 0x3e000; - - /** User script header size. */ - var USER_SCRIPT_HEADER_CHARS = 4; + var USER_CODE_START_ADDR = 0x3e000; + var USER_CODE_LEN = 8 * 1024; + var USER_CODE_END_ADDR = USER_CODE_START_ADDR + USER_CODE_LEN; - /** When user script added. */ - var USER_SCRIPT_MAX_LENGTH = 8192; + /** User code header */ + var USER_CODE_HEADER_SIZE = 4; + var USER_CODE_HEADER_START_B0_INDEX = 0; + var USER_CODE_HEADER_START_B1_INDEX = 1; + var USER_CODE_HEADER_LEN_LSB_INDEX = 2; + var USER_CODE_HEADER_LEN_MSB_INDEX = 3; /** Number of data bytes per Intel Hex record (line). */ var INTEL_HEX_BYTE_CNT = 16; + /** Start of user script marked by "MP" + 2 bytes for the script length. */ + var USER_CODE_HEADER_START_B0 = 77; // 'M' + var USER_CODE_HEADER_START_B1 = 80; // 'P' + /** * String placed inside the MicroPython hex string to indicate where to * paste the Python Code * */ - var HEX_INSERTION_POINT = ":::::::::::::::::::::::::::::::::::::::::::"; + var HEX_INSERTION_POINT = ":::::::::::::::::::::::::::::::::::::::::::\n"; /** - * Converts a user Python script into the Intel Hex format with the header - * and the address expected by MicroPython and configured for the micro:bit - * hex file format. - * @param {string} pyStr - Python code to convert. - * @return {string} Code in Intel Hex string formatted for the micro:bit. + * Converts a string into a byte array of characters. + * TODO: Update to encode to UTF-8 correctly. + * @param {Uint8Array|Object[]} byteArray - Array of bytes to convert. + * @return {string} String output from the conversion. */ - function pyStrToIntelHex(pyStr) { - // Add header to the script size - var dataLength = USER_SCRIPT_HEADER_CHARS + pyStr.length; - // Add padding for the data field size in the Intel Hex records - dataLength += INTEL_HEX_BYTE_CNT - (dataLength % INTEL_HEX_BYTE_CNT); - // Check the data block fits in the allocated flash area - if (dataLength > USER_SCRIPT_MAX_LENGTH) { - throw new RangeError('Too long'); + function strToBytes(str) { + var data = new Uint8Array(str.length); + for (var i = 0; i < str.length; i++) { + // TODO: This will only keep the LSB from the UTF-16 code points + data[i] = str.charCodeAt(i); } - // The user script has to start with "MP" marker + script length - var data = new Uint8Array(dataLength); - data[0] = USER_SCRIPT_START_BYTE_0; - data[1] = USER_SCRIPT_START_BYTE_1; - data[2] = pyStr.length & 0xff; - data[3] = (pyStr.length >> 8) & 0xff; - for (var i = 0; i < pyStr.length; ++i) { - data[4 + i] = pyStr.charCodeAt(i); - } - // Convert to Intel Hex format - return ihexlify.bytesToIntelHexStr( - USER_SCRIPT_START_ADDR, INTEL_HEX_BYTE_CNT, data); + return data; } /** @@ -329,28 +60,12 @@ var upyhex = (function() { } /** - * Takes a block if Intel Hex records, extracts the data, discards the - * headers required by MicroPython and converts the rest into a string. - * TODO: This function takes advantage of the way the return value from - * ihexlify.intelHexStrToBytes() is formatted (an array of Uint8Array) - * so will need to be completely rewritten if that function is updated - * to return a single Uint8Array with all the data. - * @param {string} intelHexStr - Intel Hex block to scan for the code. - * @return {strin} The Python code. + * Removes the old insertion line the input Intel Hex string contains it. + * @param {string} intelHexStr String with the intel hex lines. + * @return {string} The Intel Hex string without insertion line. */ - function intelHexToPyStr(intelHexStr) { - // Convert the Intel Hex block into an array of Uint8Arrays, one item - // per record - var output = ihexlify.intelHexStrToBytes(intelHexStr); - // Convert the bytes from each record into a string - for (var i = 0; i < output.length; i++) { - output[i] = bytesToStr(output[i]); - } - // Discard the header from the beginning, and clean the null terminator - output[0] = output[0].slice(4); - var last = output.length - 1; - output[last] = output[last].replace(/\0/g, ''); - return output.join(''); + function cleanseOldHexFormat(intelHexStr) { + return intelHexStr.replace(HEX_INSERTION_POINT, ''); } /** @@ -360,21 +75,20 @@ var upyhex = (function() { * @return {string} Python code. */ function extractPyStrFromIntelHex(intelHexStr) { - var hex_lines = intelHexStr.trimRight().split(/\r?\n/); - var extAddrRecord = - ihexlify.createExtLinearAddrRecord(USER_SCRIPT_START_ADDR); - var start_line = hex_lines.lastIndexOf(extAddrRecord); - if (start_line > 0) { - var lines = hex_lines.slice(start_line + 1, -2); - var blob = lines.join('\n'); - if (blob === '') { - return ''; - } else { - return intelHexToPyStr(blob); + var pyCodeStr = ''; + var hexFileMemMap = MemoryMap.fromHex(intelHexStr); + // Check that the known flash location has user code + if (hexFileMemMap.has(USER_CODE_START_ADDR)) { + var pyCodeMemMap = hexFileMemMap.slice(USER_CODE_START_ADDR, USER_CODE_LEN); + var codeBytes = pyCodeMemMap.get(USER_CODE_START_ADDR); + if ((codeBytes[USER_CODE_HEADER_START_B0_INDEX] === USER_CODE_HEADER_START_B0) && + (codeBytes[USER_CODE_HEADER_START_B1_INDEX] === USER_CODE_HEADER_START_B1)) { + pyCodeStr = bytesToStr(codeBytes.slice(USER_CODE_HEADER_SIZE)); + // Clean null terminators at the end + pyCodeStr = pyCodeStr.replace(/\0/g, ''); } - } else { - return ''; } + return pyCodeStr; } /** @@ -385,13 +99,30 @@ var upyhex = (function() { * @return {string} Intel Hex string with the Python code injected. */ function injectPyStrIntoIntelHex(intelHexStr, pyStr) { - var pyCodeIntelHex = pyStrToIntelHex(pyStr); - return intelHexStr.replace(HEX_INSERTION_POINT, pyCodeIntelHex); + var codeBytes = strToBytes(pyStr); + var blockLength = USER_CODE_HEADER_SIZE + codeBytes.length; + // Check the data block fits in the allocated flash area + if (blockLength > USER_CODE_LEN) { + throw new RangeError('Too long'); + } + // Older DAPLink versions need the last line to be padded + blockLength += INTEL_HEX_BYTE_CNT - (blockLength % INTEL_HEX_BYTE_CNT); + // The user script block has to start with "MP" marker + script length + var blockBytes = new Uint8Array(blockLength); + blockBytes[0] = USER_CODE_HEADER_START_B0; + blockBytes[1] = USER_CODE_HEADER_START_B1; + blockBytes[2] = codeBytes.length & 0xff; + blockBytes[3] = (codeBytes.length >> 8) & 0xff; + blockBytes.set(codeBytes, USER_CODE_HEADER_SIZE); + // Convert to Intel Hex format + intelHexStr = cleanseOldHexFormat(intelHexStr); + var intelHexMap = MemoryMap.fromHex(intelHexStr); + intelHexMap.set(USER_CODE_START_ADDR, blockBytes); + // Older versions of DAPLink need the file to end in a new line + return intelHexMap.asHexString() + '\n'; } return { - pyStrToIntelHex: pyStrToIntelHex, - intelHexToPyStr: intelHexToPyStr, extractPyStrFromIntelHex: extractPyStrFromIntelHex, injectPyStrIntoIntelHex: injectPyStrIntoIntelHex, }; diff --git a/static/js/intel-hex.browser.js b/static/js/intel-hex.browser.js new file mode 100644 index 00000000..b36d1d0b --- /dev/null +++ b/static/js/intel-hex.browser.js @@ -0,0 +1,1249 @@ +(function (global, factory) { + typeof exports === 'object' && typeof module !== 'undefined' ? module.exports = factory() : + typeof define === 'function' && define.amd ? define(factory) : + (global.MemoryMap = factory()); +}(this, (function () { 'use strict'; + +var _slicedToArray = function () { function sliceIterator(arr, i) { var _arr = []; var _n = true; var _d = false; var _e = undefined; try { for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"]) _i["return"](); } finally { if (_d) throw _e; } } return _arr; } return function (arr, i) { if (Array.isArray(arr)) { return arr; } else if (Symbol.iterator in Object(arr)) { return sliceIterator(arr, i); } else { throw new TypeError("Invalid attempt to destructure non-iterable instance"); } }; }(); + +var _typeof = typeof Symbol === "function" && typeof Symbol.iterator === "symbol" ? function (obj) { return typeof obj; } : function (obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; }; + +var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +/** + * Parser/writer for the "Intel hex" format. + */ + +/* + * A regexp that matches lines in a .hex file. + * + * One hexadecimal character is matched by "[0-9A-Fa-f]". + * Two hex characters are matched by "[0-9A-Fa-f]{2}" + * Eight or more hex characters are matched by "[0-9A-Fa-f]{8,}" + * A capture group of two hex characters is "([0-9A-Fa-f]{2})" + * + * Record mark : + * 8 or more hex chars ([0-9A-Fa-f]{8,}) + * Checksum ([0-9A-Fa-f]{2}) + * Optional newline (?:\r\n|\r|\n|) + */ +var hexLineRegexp = /:([0-9A-Fa-f]{8,})([0-9A-Fa-f]{2})(?:\r\n|\r|\n|)/g; + +// Takes a Uint8Array as input, +// Returns an integer in the 0-255 range. +function checksum(bytes) { + return -bytes.reduce(function (sum, v) { + return sum + v; + }, 0) & 0xFF; +} + +// Takes two Uint8Arrays as input, +// Returns an integer in the 0-255 range. +function checksumTwo(array1, array2) { + var partial1 = array1.reduce(function (sum, v) { + return sum + v; + }, 0); + var partial2 = array2.reduce(function (sum, v) { + return sum + v; + }, 0); + return -(partial1 + partial2) & 0xFF; +} + +// Trivial utility. Converts a number to hex and pads with zeroes up to 2 characters. +function hexpad(number) { + return number.toString(16).toUpperCase().padStart(2, '0'); +} + +// Polyfill as per https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/isInteger +Number.isInteger = Number.isInteger || function (value) { + return typeof value === 'number' && isFinite(value) && Math.floor(value) === value; +}; + +/** + * @class MemoryMap + * + * Represents the contents of a memory layout, with main focus into (possibly sparse) blocks of data. + *
+ * A {@linkcode MemoryMap} acts as a subclass of + * {@linkcode https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Map|Map}. + * In every entry of it, the key is the starting address of a data block (an integer number), + * and the value is the Uint8Array with the data for that block. + *
+ * The main rationale for this is that a .hex file can contain a single block of contiguous + * data starting at memory address 0 (and it's the common case for simple .hex files), + * but complex files with several non-contiguous data blocks are also possible, thus + * the need for a data structure on top of the Uint8Arrays. + *
+ * In order to parse .hex files, use the {@linkcode MemoryMap.fromHex} static factory + * method. In order to write .hex files, create a new {@linkcode MemoryMap} and call + * its {@linkcode MemoryMap.asHexString} method. + * + * @extends Map + * @example + * import MemoryMap from 'nrf-intel-hex'; + * + * let memMap1 = new MemoryMap(); + * let memMap2 = new MemoryMap([[0, new Uint8Array(1,2,3,4)]]); + * let memMap3 = new MemoryMap({0: new Uint8Array(1,2,3,4)}); + * let memMap4 = new MemoryMap({0xCF0: new Uint8Array(1,2,3,4)}); + */ + +var MemoryMap = function () { + /** + * @param {Iterable} blocks The initial value for the memory blocks inside this + * MemoryMap. All keys must be numeric, and all values must be instances of + * Uint8Array. Optionally it can also be a plain Object with + * only numeric keys. + */ + function MemoryMap(blocks) { + _classCallCheck(this, MemoryMap); + + this._blocks = new Map(); + + if (blocks && typeof blocks[Symbol.iterator] === 'function') { + var _iteratorNormalCompletion = true; + var _didIteratorError = false; + var _iteratorError = undefined; + + try { + for (var _iterator = blocks[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true) { + var tuple = _step.value; + + if (!(tuple instanceof Array) || tuple.length !== 2) { + throw new Error('First parameter to MemoryMap constructor must be an iterable of [addr, bytes] or undefined'); + } + this.set(tuple[0], tuple[1]); + } + } catch (err) { + _didIteratorError = true; + _iteratorError = err; + } finally { + try { + if (!_iteratorNormalCompletion && _iterator.return) { + _iterator.return(); + } + } finally { + if (_didIteratorError) { + throw _iteratorError; + } + } + } + } else if ((typeof blocks === 'undefined' ? 'undefined' : _typeof(blocks)) === 'object') { + // Try iterating through the object's keys + var addrs = Object.keys(blocks); + var _iteratorNormalCompletion2 = true; + var _didIteratorError2 = false; + var _iteratorError2 = undefined; + + try { + for (var _iterator2 = addrs[Symbol.iterator](), _step2; !(_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done); _iteratorNormalCompletion2 = true) { + var addr = _step2.value; + + this.set(parseInt(addr), blocks[addr]); + } + } catch (err) { + _didIteratorError2 = true; + _iteratorError2 = err; + } finally { + try { + if (!_iteratorNormalCompletion2 && _iterator2.return) { + _iterator2.return(); + } + } finally { + if (_didIteratorError2) { + throw _iteratorError2; + } + } + } + } else if (blocks !== undefined && blocks !== null) { + throw new Error('First parameter to MemoryMap constructor must be an iterable of [addr, bytes] or undefined'); + } + } + + _createClass(MemoryMap, [{ + key: 'set', + value: function set(addr, value) { + if (!Number.isInteger(addr)) { + throw new Error('Address passed to MemoryMap is not an integer'); + } + if (addr < 0) { + throw new Error('Address passed to MemoryMap is negative'); + } + if (!(value instanceof Uint8Array)) { + throw new Error('Bytes passed to MemoryMap are not an Uint8Array'); + } + return this._blocks.set(addr, value); + } + // Delegate the following to the 'this._blocks' Map: + + }, { + key: 'get', + value: function get(addr) { + return this._blocks.get(addr); + } + }, { + key: 'clear', + value: function clear() { + return this._blocks.clear(); + } + }, { + key: 'delete', + value: function _delete(addr) { + return this._blocks.delete(addr); + } + }, { + key: 'entries', + value: function entries() { + return this._blocks.entries(); + } + }, { + key: 'forEach', + value: function forEach(callback, that) { + return this._blocks.forEach(callback, that); + } + }, { + key: 'has', + value: function has(addr) { + return this._blocks.has(addr); + } + }, { + key: 'keys', + value: function keys() { + return this._blocks.keys(); + } + }, { + key: 'values', + value: function values() { + return this._blocks.values(); + } + }, { + key: Symbol.iterator, + value: function value() { + return this._blocks[Symbol.iterator](); + } + + /** + * Parses a string containing data formatted in "Intel HEX" format, and + * returns an instance of {@linkcode MemoryMap}. + *
+ * The insertion order of keys in the {@linkcode MemoryMap} is guaranteed to be strictly + * ascending. In other words, when iterating through the {@linkcode MemoryMap}, the addresses + * will be ordered in ascending order. + *
+ * The parser has an opinionated behaviour, and will throw a descriptive error if it + * encounters some malformed input. Check the project's + * {@link https://github.com/NordicSemiconductor/nrf-intel-hex#Features|README file} for details. + *
+ * If maxBlockSize is given, any contiguous data block larger than that will + * be split in several blocks. + * + * @param {String} hexText The contents of a .hex file. + * @param {Number} [maxBlockSize=Infinity] Maximum size of the returned Uint8Arrays. + * + * @return {MemoryMap} + * + * @example + * import MemoryMap from 'nrf-intel-hex'; + * + * let intelHexString = + * ":100000000102030405060708090A0B0C0D0E0F1068\n" + + * ":00000001FF"; + * + * let memMap = MemoryMap.fromHex(intelHexString); + * + * for (let [address, dataBlock] of memMap) { + * console.log('Data block at ', address, ', bytes: ', dataBlock); + * } + */ + + }, { + key: 'join', + + + /** + * Returns a new instance of {@linkcode MemoryMap}, containing + * the same data, but concatenating together those memory blocks that are adjacent. + *
+ * The insertion order of keys in the {@linkcode MemoryMap} is guaranteed to be strictly + * ascending. In other words, when iterating through the {@linkcode MemoryMap}, the addresses + * will be ordered in ascending order. + *
+ * If maxBlockSize is given, blocks will be concatenated together only + * until the joined block reaches this size in bytes. This means that the output + * {@linkcode MemoryMap} might have more entries than the input one. + *
+ * If there is any overlap between blocks, an error will be thrown. + *
+ * The returned {@linkcode MemoryMap} will use newly allocated memory. + * + * @param {Number} [maxBlockSize=Infinity] Maximum size of the Uint8Arrays in the + * returned {@linkcode MemoryMap}. + * + * @return {MemoryMap} + */ + value: function join() { + var maxBlockSize = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : Infinity; + + + // First pass, create a Map of address→length of contiguous blocks + var sortedKeys = Array.from(this.keys()).sort(function (a, b) { + return a - b; + }); + var blockSizes = new Map(); + var lastBlockAddr = -1; + var lastBlockEndAddr = -1; + + for (var i = 0, l = sortedKeys.length; i < l; i++) { + var blockAddr = sortedKeys[i]; + var blockLength = this.get(sortedKeys[i]).length; + + if (lastBlockEndAddr === blockAddr && lastBlockEndAddr - lastBlockAddr < maxBlockSize) { + // Grow when the previous end address equals the current, + // and we don't go over the maximum block size. + blockSizes.set(lastBlockAddr, blockSizes.get(lastBlockAddr) + blockLength); + lastBlockEndAddr += blockLength; + } else if (lastBlockEndAddr <= blockAddr) { + // Else mark a new block. + blockSizes.set(blockAddr, blockLength); + lastBlockAddr = blockAddr; + lastBlockEndAddr = blockAddr + blockLength; + } else { + throw new Error('Overlapping data around address 0x' + blockAddr.toString(16)); + } + } + + // Second pass: allocate memory for the contiguous blocks and copy data around. + var mergedBlocks = new MemoryMap(); + var mergingBlock = void 0; + var mergingBlockAddr = -1; + for (var _i = 0, _l = sortedKeys.length; _i < _l; _i++) { + var _blockAddr = sortedKeys[_i]; + if (blockSizes.has(_blockAddr)) { + mergingBlock = new Uint8Array(blockSizes.get(_blockAddr)); + mergedBlocks.set(_blockAddr, mergingBlock); + mergingBlockAddr = _blockAddr; + } + mergingBlock.set(this.get(_blockAddr), _blockAddr - mergingBlockAddr); + } + + return mergedBlocks; + } + + /** + * Given a {@link https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Map|Map} + * of {@linkcode MemoryMap}s, indexed by a alphanumeric ID, + * returns a Map of address to tuples (Arrayss of length 2) of the form + * (id, Uint8Array)s. + *
+ * The scenario for using this is having several {@linkcode MemoryMap}s, from several calls to + * {@link module:nrf-intel-hex~hexToArrays|hexToArrays}, each having a different identifier. + * This function locates where those memory block sets overlap, and returns a Map + * containing addresses as keys, and arrays as values. Each array will contain 1 or more + * (id, Uint8Array) tuples: the identifier of the memory block set that has + * data in that region, and the data itself. When memory block sets overlap, there will + * be more than one tuple. + *
+ * The Uint8Arrays in the output are + * {@link https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/TypedArray/subarray|subarrays} + * of the input data; new memory is not allocated for them. + *
+ * The insertion order of keys in the output Map is guaranteed to be strictly + * ascending. In other words, when iterating through the Map, the addresses + * will be ordered in ascending order. + *
+ * When two blocks overlap, the corresponding array of tuples will have the tuples ordered + * in the insertion order of the input Map of block sets. + *
+ * + * @param {Map.MemoryMap} memoryMaps The input memory block sets + * + * @example + * import MemoryMap from 'nrf-intel-hex'; + * + * let memMap1 = MemoryMap.fromHex( hexdata1 ); + * let memMap2 = MemoryMap.fromHex( hexdata2 ); + * let memMap3 = MemoryMap.fromHex( hexdata3 ); + * + * let maps = new Map([ + * ['file A', blocks1], + * ['file B', blocks2], + * ['file C', blocks3] + * ]); + * + * let overlappings = MemoryMap.overlapMemoryMaps(maps); + * + * for (let [address, tuples] of overlappings) { + * // if 'tuples' has length > 1, there is an overlap starting at 'address' + * + * for (let [address, tuples] of overlappings) { + * let [id, bytes] = tuple; + * // 'id' in this example is either 'file A', 'file B' or 'file C' + * } + * } + * @return {Map.Array} The map of possibly overlapping memory blocks + */ + + }, { + key: 'paginate', + + + /** + * Returns a new instance of {@linkcode MemoryMap}, where: + * + *
    + *
  • Each key (the start address of each Uint8Array) is a multiple of + * pageSize
    • + *
  • The size of each Uint8Array is exactly pageSize
    • + *
  • Bytes from the input map to bytes in the output
    • + *
  • Bytes not in the input are replaced by a padding value
    • + *
+ *
+ * The scenario is wanting to prepare pages of bytes for a write operation, where the write + * operation affects a whole page/sector at once. + *
+ * The insertion order of keys in the output {@linkcode MemoryMap} is guaranteed + * to be strictly ascending. In other words, when iterating through the + * {@linkcode MemoryMap}, the addresses will be ordered in ascending order. + *
+ * The Uint8Arrays in the output will be newly allocated. + *
+ * + * @param {Number} [pageSize=1024] The size of the output pages, in bytes + * @param {Number} [pad=0xFF] The byte value to use for padding + * @return {MemoryMap} + */ + value: function paginate() { + var pageSize = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 1024; + var pad = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 0xFF; + + if (pageSize <= 0) { + throw new Error('Page size must be greater than zero'); + } + var outPages = new MemoryMap(); + var page = void 0; + + var sortedKeys = Array.from(this.keys()).sort(function (a, b) { + return a - b; + }); + + for (var i = 0, l = sortedKeys.length; i < l; i++) { + var blockAddr = sortedKeys[i]; + var block = this.get(blockAddr); + var blockLength = block.length; + var blockEnd = blockAddr + blockLength; + + for (var pageAddr = blockAddr - blockAddr % pageSize; pageAddr < blockEnd; pageAddr += pageSize) { + page = outPages.get(pageAddr); + if (!page) { + page = new Uint8Array(pageSize); + page.fill(pad); + outPages.set(pageAddr, page); + } + + var offset = pageAddr - blockAddr; + var subBlock = void 0; + if (offset <= 0) { + // First page which intersects the block + subBlock = block.subarray(0, Math.min(pageSize + offset, blockLength)); + page.set(subBlock, -offset); + } else { + // Any other page which intersects the block + subBlock = block.subarray(offset, offset + Math.min(pageSize, blockLength - offset)); + page.set(subBlock, 0); + } + } + } + + return outPages; + } + + /** + * Locates the Uint8Array which contains the given offset, + * and returns the four bytes held at that offset, as a 32-bit unsigned integer. + * + *
+ * Behaviour is similar to {@linkcode https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/DataView/getUint32|DataView.prototype.getUint32}, + * except that this operates over a {@linkcode MemoryMap} instead of + * over an ArrayBuffer, and that this may return undefined if + * the address is not entirely contained within one of the Uint8Arrays. + *
+ * + * @param {Number} offset The memory offset to read the data + * @param {Boolean} [littleEndian=false] Whether to fetch the 4 bytes as a little- or big-endian integer + * @return {Number|undefined} An unsigned 32-bit integer number + */ + + }, { + key: 'getUint32', + value: function getUint32(offset, littleEndian) { + var keys = Array.from(this.keys()); + + for (var i = 0, l = keys.length; i < l; i++) { + var blockAddr = keys[i]; + var block = this.get(blockAddr); + var blockLength = block.length; + var blockEnd = blockAddr + blockLength; + + if (blockAddr <= offset && offset + 4 <= blockEnd) { + return new DataView(block.buffer, offset - blockAddr, 4).getUint32(0, littleEndian); + } + } + return; + } + + /** + * Returns a String of text representing a .hex file. + *
+ * The writer has an opinionated behaviour. Check the project's + * {@link https://github.com/NordicSemiconductor/nrf-intel-hex#Features|README file} for details. + * + * @param {Number} [lineSize=16] Maximum number of bytes to be encoded in each data record. + * Must have a value between 1 and 255, as per the specification. + * + * @return {String} String of text with the .hex representation of the input binary data + * + * @example + * import MemoryMap from 'nrf-intel-hex'; + * + * let memMap = new MemoryMap(); + * let bytes = new Uint8Array(....); + * memMap.set(0x0FF80000, bytes); // The block with 'bytes' will start at offset 0x0FF80000 + * + * let string = memMap.asHexString(); + */ + + }, { + key: 'asHexString', + value: function asHexString() { + var lineSize = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 16; + + var lowAddress = 0; // 16 least significant bits of the current addr + var highAddress = -1 << 16; // 16 most significant bits of the current addr + var records = []; + if (lineSize <= 0) { + throw new Error('Size of record must be greater than zero'); + } else if (lineSize > 255) { + throw new Error('Size of record must be less than 256'); + } + + // Placeholders + var offsetRecord = new Uint8Array(6); + var recordHeader = new Uint8Array(4); + + var sortedKeys = Array.from(this.keys()).sort(function (a, b) { + return a - b; + }); + for (var i = 0, l = sortedKeys.length; i < l; i++) { + var blockAddr = sortedKeys[i]; + var block = this.get(blockAddr); + + // Sanity checks + if (!(block instanceof Uint8Array)) { + throw new Error('Block at offset ' + blockAddr + ' is not an Uint8Array'); + } + if (blockAddr < 0) { + throw new Error('Block at offset ' + blockAddr + ' has a negative thus invalid address'); + } + var blockSize = block.length; + if (!blockSize) { + continue; + } // Skip zero-length blocks + + + if (blockAddr > highAddress + 0xFFFF) { + // Insert a new 0x04 record to jump to a new 64KiB block + + // Round up the least significant 16 bits - no bitmasks because they trigger + // base-2 negative numbers, whereas subtracting the modulo maintains precision + highAddress = blockAddr - blockAddr % 0x10000; + lowAddress = 0; + + offsetRecord[0] = 2; // Length + offsetRecord[1] = 0; // Load offset, high byte + offsetRecord[2] = 0; // Load offset, low byte + offsetRecord[3] = 4; // Record type + offsetRecord[4] = highAddress >> 24; // new address offset, high byte + offsetRecord[5] = highAddress >> 16; // new address offset, low byte + + records.push(':' + Array.prototype.map.call(offsetRecord, hexpad).join('') + hexpad(checksum(offsetRecord))); + } + + if (blockAddr < highAddress + lowAddress) { + throw new Error('Block starting at 0x' + blockAddr.toString(16) + ' overlaps with a previous block.'); + } + + lowAddress = blockAddr % 0x10000; + var blockOffset = 0; + var blockEnd = blockAddr + blockSize; + if (blockEnd > 0xFFFFFFFF) { + throw new Error('Data cannot be over 0xFFFFFFFF'); + } + + // Loop for every 64KiB memory segment that spans this block + while (highAddress + lowAddress < blockEnd) { + + if (lowAddress > 0xFFFF) { + // Insert a new 0x04 record to jump to a new 64KiB block + highAddress += 1 << 16; // Increase by one + lowAddress = 0; + + offsetRecord[0] = 2; // Length + offsetRecord[1] = 0; // Load offset, high byte + offsetRecord[2] = 0; // Load offset, low byte + offsetRecord[3] = 4; // Record type + offsetRecord[4] = highAddress >> 24; // new address offset, high byte + offsetRecord[5] = highAddress >> 16; // new address offset, low byte + + records.push(':' + Array.prototype.map.call(offsetRecord, hexpad).join('') + hexpad(checksum(offsetRecord))); + } + + var recordSize = -1; + // Loop for every record for that spans the current 64KiB memory segment + while (lowAddress < 0x10000 && recordSize) { + recordSize = Math.min(lineSize, // Normal case + blockEnd - highAddress - lowAddress, // End of block + 0x10000 - lowAddress // End of low addresses + ); + + if (recordSize) { + + recordHeader[0] = recordSize; // Length + recordHeader[1] = lowAddress >> 8; // Load offset, high byte + recordHeader[2] = lowAddress; // Load offset, low byte + recordHeader[3] = 0; // Record type + + var subBlock = block.subarray(blockOffset, blockOffset + recordSize); // Data bytes for this record + + records.push(':' + Array.prototype.map.call(recordHeader, hexpad).join('') + Array.prototype.map.call(subBlock, hexpad).join('') + hexpad(checksumTwo(recordHeader, subBlock))); + + blockOffset += recordSize; + lowAddress += recordSize; + } + } + } + } + + records.push(':00000001FF'); // EOF record + + return records.join('\n'); + } + + /** + * Performs a deep copy of the current {@linkcode MemoryMap}, returning a new one + * with exactly the same contents, but allocating new memory for each of its + * Uint8Arrays. + * + * @return {MemoryMap} + */ + + }, { + key: 'clone', + value: function clone() { + var cloned = new MemoryMap(); + + var _iteratorNormalCompletion3 = true; + var _didIteratorError3 = false; + var _iteratorError3 = undefined; + + try { + for (var _iterator3 = this[Symbol.iterator](), _step3; !(_iteratorNormalCompletion3 = (_step3 = _iterator3.next()).done); _iteratorNormalCompletion3 = true) { + var _step3$value = _slicedToArray(_step3.value, 2), + addr = _step3$value[0], + value = _step3$value[1]; + + cloned.set(addr, new Uint8Array(value)); + } + } catch (err) { + _didIteratorError3 = true; + _iteratorError3 = err; + } finally { + try { + if (!_iteratorNormalCompletion3 && _iterator3.return) { + _iterator3.return(); + } + } finally { + if (_didIteratorError3) { + throw _iteratorError3; + } + } + } + + return cloned; + } + + /** + * Given one Uint8Array, looks through its contents and returns a new + * {@linkcode MemoryMap}, stripping away those regions where there are only + * padding bytes. + *
+ * The start of the input Uint8Array is assumed to be offset zero for the output. + *
+ * The use case here is dumping memory from a working device and try to see the + * "interesting" memory regions it has. This assumes that there is a constant, + * predefined padding byte value being used in the "non-interesting" regions. + * In other words: this will work as long as the dump comes from a flash memory + * which has been previously erased (thus 0xFFs for padding), or from a + * previously blanked HDD (thus 0x00s for padding). + *
+ * This method uses subarray on the input data, and thus does not allocate memory + * for the Uint8Arrays. + * + * @param {Uint8Array} bytes The input data + * @param {Number} [padByte=0xFF] The value of the byte assumed to be used as padding + * @param {Number} [minPadLength=64] The minimum number of consecutive pad bytes to + * be considered actual padding + * + * @return {MemoryMap} + */ + + }, { + key: 'slice', + + + /** + * Returns a new instance of {@linkcode MemoryMap}, containing only data between + * the addresses address and address + length. + * Behaviour is similar to {@linkcode https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array/slice|Array.prototype.slice}, + * in that the return value is a portion of the current {@linkcode MemoryMap}. + * + *
+ * The returned {@linkcode MemoryMap} might be empty. + * + *
+ * Internally, this uses subarray, so new memory is not allocated. + * + * @param {Number} address The start address of the slice + * @param {Number} length The length of memory map to slice out + * @return {MemoryMap} + */ + value: function slice(address) { + var length = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : Infinity; + + if (length < 0) { + throw new Error('Length of the slice cannot be negative'); + } + + var sliced = new MemoryMap(); + + var _iteratorNormalCompletion4 = true; + var _didIteratorError4 = false; + var _iteratorError4 = undefined; + + try { + for (var _iterator4 = this[Symbol.iterator](), _step4; !(_iteratorNormalCompletion4 = (_step4 = _iterator4.next()).done); _iteratorNormalCompletion4 = true) { + var _step4$value = _slicedToArray(_step4.value, 2), + blockAddr = _step4$value[0], + block = _step4$value[1]; + + var blockLength = block.length; + + if (blockAddr + blockLength >= address && blockAddr < address + length) { + var sliceStart = Math.max(address, blockAddr); + var sliceEnd = Math.min(address + length, blockAddr + blockLength); + var sliceLength = sliceEnd - sliceStart; + var relativeSliceStart = sliceStart - blockAddr; + + if (sliceLength > 0) { + sliced.set(sliceStart, block.subarray(relativeSliceStart, relativeSliceStart + sliceLength)); + } + } + } + } catch (err) { + _didIteratorError4 = true; + _iteratorError4 = err; + } finally { + try { + if (!_iteratorNormalCompletion4 && _iterator4.return) { + _iterator4.return(); + } + } finally { + if (_didIteratorError4) { + throw _iteratorError4; + } + } + } + + return sliced; + } + + /** + * Returns a new instance of {@linkcode https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/DataView/getUint32|Uint8Array}, containing only data between + * the addresses address and address + length. Any byte without a value + * in the input {@linkcode MemoryMap} will have a value of padByte. + * + *
+ * This method allocates new memory. + * + * @param {Number} address The start address of the slice + * @param {Number} length The length of memory map to slice out + * @param {Number} [padByte=0xFF] The value of the byte assumed to be used as padding + * @return {MemoryMap} + */ + + }, { + key: 'slicePad', + value: function slicePad(address, length) { + var padByte = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 0xFF; + + if (length < 0) { + throw new Error('Length of the slice cannot be negative'); + } + + var out = new Uint8Array(length).fill(padByte); + + var _iteratorNormalCompletion5 = true; + var _didIteratorError5 = false; + var _iteratorError5 = undefined; + + try { + for (var _iterator5 = this[Symbol.iterator](), _step5; !(_iteratorNormalCompletion5 = (_step5 = _iterator5.next()).done); _iteratorNormalCompletion5 = true) { + var _step5$value = _slicedToArray(_step5.value, 2), + blockAddr = _step5$value[0], + block = _step5$value[1]; + + var blockLength = block.length; + + if (blockAddr + blockLength >= address && blockAddr < address + length) { + var sliceStart = Math.max(address, blockAddr); + var sliceEnd = Math.min(address + length, blockAddr + blockLength); + var sliceLength = sliceEnd - sliceStart; + var relativeSliceStart = sliceStart - blockAddr; + + if (sliceLength > 0) { + out.set(block.subarray(relativeSliceStart, relativeSliceStart + sliceLength), sliceStart - address); + } + } + } + } catch (err) { + _didIteratorError5 = true; + _iteratorError5 = err; + } finally { + try { + if (!_iteratorNormalCompletion5 && _iterator5.return) { + _iterator5.return(); + } + } finally { + if (_didIteratorError5) { + throw _iteratorError5; + } + } + } + + return out; + } + + /** + * Checks whether the current memory map contains the one given as a parameter. + * + *
+ * "Contains" means that all the offsets that have a byte value in the given + * memory map have a value in the current memory map, and that the byte values + * are the same. + * + *
+ * An empty memory map is always contained in any other memory map. + * + *
+ * Returns boolean true if the memory map is contained, false + * otherwise. + * + * @param {MemoryMap} memMap The memory map to check + * @return {Boolean} + */ + + }, { + key: 'contains', + value: function contains(memMap) { + var _iteratorNormalCompletion6 = true; + var _didIteratorError6 = false; + var _iteratorError6 = undefined; + + try { + for (var _iterator6 = memMap[Symbol.iterator](), _step6; !(_iteratorNormalCompletion6 = (_step6 = _iterator6.next()).done); _iteratorNormalCompletion6 = true) { + var _step6$value = _slicedToArray(_step6.value, 2), + blockAddr = _step6$value[0], + block = _step6$value[1]; + + var blockLength = block.length; + + var slice = this.slice(blockAddr, blockLength).join().get(blockAddr); + + if (!slice || slice.length !== blockLength) { + return false; + } + + for (var i in block) { + if (block[i] !== slice[i]) { + return false; + } + } + } + } catch (err) { + _didIteratorError6 = true; + _iteratorError6 = err; + } finally { + try { + if (!_iteratorNormalCompletion6 && _iterator6.return) { + _iterator6.return(); + } + } finally { + if (_didIteratorError6) { + throw _iteratorError6; + } + } + } + + return true; + } + }, { + key: 'size', + get: function get() { + return this._blocks.size; + } + }], [{ + key: 'fromHex', + value: function fromHex(hexText) { + var maxBlockSize = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : Infinity; + + var blocks = new MemoryMap(); + + var lastCharacterParsed = 0; + var matchResult = void 0; + var recordCount = 0; + + // Upper Linear Base Address, the 16 most significant bits (2 bytes) of + // the current 32-bit (4-byte) address + // In practice this is a offset that is summed to the "load offset" of the + // data records + var ulba = 0; + + hexLineRegexp.lastIndex = 0; // Reset the regexp, if not it would skip content when called twice + + while ((matchResult = hexLineRegexp.exec(hexText)) !== null) { + recordCount++; + + // By default, a regexp loop ignores gaps between matches, but + // we want to be aware of them. + if (lastCharacterParsed !== matchResult.index) { + throw new Error('Malformed hex file: Could not parse between characters ' + lastCharacterParsed + ' and ' + matchResult.index + ' ("' + hexText.substring(lastCharacterParsed, Math.min(matchResult.index, lastCharacterParsed + 16)).trim() + '")'); + } + lastCharacterParsed = hexLineRegexp.lastIndex; + + // Give pretty names to the match's capture groups + + var _matchResult = matchResult, + _matchResult2 = _slicedToArray(_matchResult, 3), + recordStr = _matchResult2[1], + recordChecksum = _matchResult2[2]; + + // String to Uint8Array - https://stackoverflow.com/questions/43131242/how-to-convert-a-hexademical-string-of-data-to-an-arraybuffer-in-javascript + + + var recordBytes = new Uint8Array(recordStr.match(/[\da-f]{2}/gi).map(function (h) { + return parseInt(h, 16); + })); + + var recordLength = recordBytes[0]; + if (recordLength + 4 !== recordBytes.length) { + throw new Error('Mismatched record length at record ' + recordCount + ' (' + matchResult[0].trim() + '), expected ' + recordLength + ' data bytes but actual length is ' + (recordBytes.length - 4)); + } + + var cs = checksum(recordBytes); + if (parseInt(recordChecksum, 16) !== cs) { + throw new Error('Checksum failed at record ' + recordCount + ' (' + matchResult[0].trim() + '), should be ' + cs.toString(16)); + } + + var offset = (recordBytes[1] << 8) + recordBytes[2]; + var recordType = recordBytes[3]; + var data = recordBytes.subarray(4); + + if (recordType === 0) { + // Data record, contains data + // Create a new block, at (upper linear base address + offset) + if (blocks.has(ulba + offset)) { + throw new Error('Duplicated data at record ' + recordCount + ' (' + matchResult[0].trim() + ')'); + } + if (offset + data.length > 0x10000) { + throw new Error('Data at record ' + recordCount + ' (' + matchResult[0].trim() + ') wraps over 0xFFFF. This would trigger ambiguous behaviour. Please restructure your data so that for every record the data offset plus the data length do not exceed 0xFFFF.'); + } + + blocks.set(ulba + offset, data); + } else { + + // All non-data records must have a data offset of zero + if (offset !== 0) { + throw new Error('Record ' + recordCount + ' (' + matchResult[0].trim() + ') must have 0000 as data offset.'); + } + + switch (recordType) { + case 1: + // EOF + if (lastCharacterParsed !== hexText.length) { + // This record should be at the very end of the string + throw new Error('There is data after an EOF record at record ' + recordCount); + } + + return blocks.join(maxBlockSize); + + case 2: + // Extended Segment Address Record + // Sets the 16 most significant bits of the 20-bit Segment Base + // Address for the subsequent data. + ulba = (data[0] << 8) + data[1] << 4; + break; + + case 3: + // Start Segment Address Record + // Do nothing. Record type 3 only applies to 16-bit Intel CPUs, + // where it should reset the program counter (CS+IP CPU registers) + break; + + case 4: + // Extended Linear Address Record + // Sets the 16 most significant (upper) bits of the 32-bit Linear Address + // for the subsequent data + ulba = (data[0] << 8) + data[1] << 16; + break; + + case 5: + // Start Linear Address Record + // Do nothing. Record type 5 only applies to 32-bit Intel CPUs, + // where it should reset the program counter (EIP CPU register) + // It might have meaning for other CPU architectures + // (see http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.faqs/ka9903.html ) + // but will be ignored nonetheless. + break; + default: + throw new Error('Invalid record type 0x' + hexpad(recordType) + ' at record ' + recordCount + ' (should be between 0x00 and 0x05)'); + } + } + } + + if (recordCount) { + throw new Error('No EOF record at end of file'); + } else { + throw new Error('Malformed .hex file, could not parse any registers'); + } + } + }, { + key: 'overlapMemoryMaps', + value: function overlapMemoryMaps(memoryMaps) { + // First pass: create a list of addresses where any block starts or ends. + var cuts = new Set(); + var _iteratorNormalCompletion7 = true; + var _didIteratorError7 = false; + var _iteratorError7 = undefined; + + try { + for (var _iterator7 = memoryMaps[Symbol.iterator](), _step7; !(_iteratorNormalCompletion7 = (_step7 = _iterator7.next()).done); _iteratorNormalCompletion7 = true) { + var _step7$value = _slicedToArray(_step7.value, 2), + blocks = _step7$value[1]; + + var _iteratorNormalCompletion8 = true; + var _didIteratorError8 = false; + var _iteratorError8 = undefined; + + try { + for (var _iterator8 = blocks[Symbol.iterator](), _step8; !(_iteratorNormalCompletion8 = (_step8 = _iterator8.next()).done); _iteratorNormalCompletion8 = true) { + var _step8$value = _slicedToArray(_step8.value, 2), + address = _step8$value[0], + block = _step8$value[1]; + + cuts.add(address); + cuts.add(address + block.length); + } + } catch (err) { + _didIteratorError8 = true; + _iteratorError8 = err; + } finally { + try { + if (!_iteratorNormalCompletion8 && _iterator8.return) { + _iterator8.return(); + } + } finally { + if (_didIteratorError8) { + throw _iteratorError8; + } + } + } + } + } catch (err) { + _didIteratorError7 = true; + _iteratorError7 = err; + } finally { + try { + if (!_iteratorNormalCompletion7 && _iterator7.return) { + _iterator7.return(); + } + } finally { + if (_didIteratorError7) { + throw _iteratorError7; + } + } + } + + var orderedCuts = Array.from(cuts.values()).sort(function (a, b) { + return a - b; + }); + var overlaps = new Map(); + + // Second pass: iterate through the cuts, get slices of every intersecting blockset + + var _loop = function _loop(i, l) { + var cut = orderedCuts[i]; + var nextCut = orderedCuts[i + 1]; + var tuples = []; + + var _iteratorNormalCompletion9 = true; + var _didIteratorError9 = false; + var _iteratorError9 = undefined; + + try { + for (var _iterator9 = memoryMaps[Symbol.iterator](), _step9; !(_iteratorNormalCompletion9 = (_step9 = _iterator9.next()).done); _iteratorNormalCompletion9 = true) { + var _step9$value = _slicedToArray(_step9.value, 2), + setId = _step9$value[0], + blocks = _step9$value[1]; + + // Find the block with the highest address that is equal or lower to + // the current cut (if any) + var blockAddr = Array.from(blocks.keys()).reduce(function (acc, val) { + if (val > cut) { + return acc; + } + return Math.max(acc, val); + }, -1); + + if (blockAddr !== -1) { + var block = blocks.get(blockAddr); + var subBlockStart = cut - blockAddr; + var subBlockEnd = nextCut - blockAddr; + + if (subBlockStart < block.length) { + tuples.push([setId, block.subarray(subBlockStart, subBlockEnd)]); + } + } + } + } catch (err) { + _didIteratorError9 = true; + _iteratorError9 = err; + } finally { + try { + if (!_iteratorNormalCompletion9 && _iterator9.return) { + _iterator9.return(); + } + } finally { + if (_didIteratorError9) { + throw _iteratorError9; + } + } + } + + if (tuples.length) { + overlaps.set(cut, tuples); + } + }; + + for (var i = 0, l = orderedCuts.length - 1; i < l; i++) { + _loop(i, l); + } + + return overlaps; + } + + /** + * Given the output of the {@linkcode MemoryMap.overlapMemoryMaps|overlapMemoryMaps} + * (a Map of address to an Array of (id, Uint8Array) tuples), + * returns a {@linkcode MemoryMap}. This discards the IDs in the process. + *
+ * The output Map contains as many entries as the input one (using the same addresses + * as keys), but the value for each entry will be the Uint8Array of the last + * tuple for each address in the input data. + *
+ * The scenario is wanting to join together several parsed .hex files, not worrying about + * their overlaps. + *
+ * + * @param {Map.Array} overlaps The (possibly overlapping) input memory blocks + * @return {MemoryMap} The flattened memory blocks + */ + + }, { + key: 'flattenOverlaps', + value: function flattenOverlaps(overlaps) { + return new MemoryMap(Array.from(overlaps.entries()).map(function (_ref) { + var _ref2 = _slicedToArray(_ref, 2), + address = _ref2[0], + tuples = _ref2[1]; + + return [address, tuples[tuples.length - 1][1]]; + })); + } + }, { + key: 'fromPaddedUint8Array', + value: function fromPaddedUint8Array(bytes) { + var padByte = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 0xFF; + var minPadLength = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 64; + + + if (!(bytes instanceof Uint8Array)) { + throw new Error('Bytes passed to fromPaddedUint8Array are not an Uint8Array'); + } + + // The algorithm used is naïve and checks every byte. + // An obvious optimization would be to implement Boyer-Moore + // (see https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_string_search_algorithm ) + // or otherwise start skipping up to minPadLength bytes when going through a non-pad + // byte. + // Anyway, we could expect a lot of cases where there is a majority of pad bytes, + // and the algorithm should check most of them anyway, so the perf gain is questionable. + + var memMap = new MemoryMap(); + var consecutivePads = 0; + var lastNonPad = -1; + var firstNonPad = 0; + var skippingBytes = false; + var l = bytes.length; + + for (var addr = 0; addr < l; addr++) { + var byte = bytes[addr]; + + if (byte === padByte) { + consecutivePads++; + if (consecutivePads >= minPadLength) { + // Edge case: ignore writing a zero-length block when skipping + // bytes at the beginning of the input + if (lastNonPad !== -1) { + /// Add the previous block to the result memMap + memMap.set(firstNonPad, bytes.subarray(firstNonPad, lastNonPad + 1)); + } + + skippingBytes = true; + } + } else { + if (skippingBytes) { + skippingBytes = false; + firstNonPad = addr; + } + lastNonPad = addr; + consecutivePads = 0; + } + } + + // At EOF, add the last block if not skipping bytes already (and input not empty) + if (!skippingBytes && lastNonPad !== -1) { + memMap.set(firstNonPad, bytes.subarray(firstNonPad, l)); + } + + return memMap; + } + }]); + + return MemoryMap; +}(); + +return MemoryMap; + +}))); +//# sourceMappingURL=intel-hex.browser.js.map diff --git a/tests.html b/tests.html index 1b83dbc2..d9ed66d7 100644 --- a/tests.html +++ b/tests.html @@ -16,6 +16,7 @@ +