From c43e20b51b50c7f3209f0c52357918d7399f24fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Uhl=C3=AD=C5=99?= Date: Tue, 22 Oct 2019 12:22:59 +0200 Subject: [PATCH 1/7] feat: migration 8 --- migrations/index.js | 1 + migrations/migration-8/blocks-to-multihash.js | 79 +++++++++++ migrations/migration-8/index.js | 38 +++++ migrations/migration-8/keys-encoding.js | 62 +++++++++ package.json | 13 +- test/browser.js | 4 + test/migrations/migration-8-test.js | 131 ++++++++++++++++++ test/node.js | 4 + 8 files changed, 327 insertions(+), 5 deletions(-) create mode 100644 migrations/migration-8/blocks-to-multihash.js create mode 100644 migrations/migration-8/index.js create mode 100644 migrations/migration-8/keys-encoding.js create mode 100644 test/migrations/migration-8-test.js diff --git a/migrations/index.js b/migrations/index.js index e4e7d8f..75cce24 100644 --- a/migrations/index.js +++ b/migrations/index.js @@ -12,4 +12,5 @@ const emptyMigration = { module.exports = [ Object.assign({version: 7}, emptyMigration), + require('./migration-8') ] diff --git a/migrations/migration-8/blocks-to-multihash.js b/migrations/migration-8/blocks-to-multihash.js new file mode 100644 index 0000000..d6897cd --- /dev/null +++ b/migrations/migration-8/blocks-to-multihash.js @@ -0,0 +1,79 @@ +const path = require('path') +const CID = require('cids') +const Key = require('interface-datastore').Key +const core = require('datastore-core') +const ShardingStore = core.ShardingDatastore +const base32 = require('base32.js') +const utils = require('../../src/utils') +const log = require('debug')('ipfs-repo-migrations:migration-8') + +// This function in js-ipfs-repo defaults to not using sharding +// but the default value of the options.sharding is True hence this +// function defaults to use sharding. +async function maybeWithSharding (filestore, options) { + if (options.sharding === false) { + return filestore + } + + const shard = new core.shard.NextToLast(2) + return await ShardingStore.createOrOpen(filestore, shard) +} + +function keyToMultihash(key){ + // Key to CID + const decoder = new base32.Decoder() + const buff = decoder.finalize(key.toString().slice(1)) + const cid = new CID(Buffer.from(buff)) + + // CID to multihash + const enc = new base32.Encoder() + return new Key('/' + enc.finalize(cid.multihash), false) +} + +function keyToCid(key){ + // Key to CID + const decoder = new base32.Decoder() + const buff = decoder.write(key.toString().slice(1)).finalize() + const cid = new CID(1, 'raw', Buffer.from(buff)) + + // CID to Key + const enc = new base32.Encoder() + return new Key('/' + enc.finalize(cid.buffer), false) +} + +async function process(repoPath, options, keyFunction){ + const { StorageBackend, storageOptions } = utils.getDatastoreAndOptions(options, 'blocks') + + const baseStore = new StorageBackend(path.join(repoPath, 'blocks'), storageOptions) + const store = await maybeWithSharding(baseStore, storageOptions) + + try { + const batch = store.batch() + let counter = 0 + for await (const block of store.query({})) { + const newKey = keyFunction(block.key) + + // If the Key is CIDv0 then it is raw multihash and nothing is changing + if(newKey.toString() !== block.key.toString()){ + counter += 1 + + log(`Migrating Block from ${block.key.toString()} to ${newKey.toString()}`) + batch.delete(block.key) + batch.put(newKey, block.value) + } + } + + log(`Changing ${ counter } blocks`) + await batch.commit() + } finally { + await store.close() + } +} + +exports.migrate = function blocksMigrate (repoPath, options) { + return process(repoPath, options, keyToMultihash) +} + +exports.revert = function blocksRevert (repoPath, options) { + return process(repoPath, options, keyToCid) +} diff --git a/migrations/migration-8/index.js b/migrations/migration-8/index.js new file mode 100644 index 0000000..949f7d5 --- /dev/null +++ b/migrations/migration-8/index.js @@ -0,0 +1,38 @@ +'use strict' + +const keysEncoding = require('./keys-encoding') +const blocksToMultihash = require('./blocks-to-multihash') +const log = require('debug')('ipfs-repo-migrations:migration-8') + +async function migrate (repoPath, options) { + await keysEncoding.migrate(repoPath, options) + + try{ + await blocksToMultihash.migrate(repoPath, options) + }catch (e) { + log('During migration of Blockstore to multihash exception was raised! Reverting keys part of migration!') + await keysEncoding.revert(repoPath, options) + + throw e + } +} + +async function revert (repoPath, options) { + await keysEncoding.revert(repoPath, options) + + try{ + await blocksToMultihash.revert(repoPath, options) + }catch (e) { + log('During reversion of Blockstore to CID exception was raised! Migrating keys part of migration!') + await keysEncoding.migrate(repoPath, options) + + throw e + } +} + +module.exports = { + version: 8, + description: 'Transforms key\'s names into base32 encoding and converts Block store to use multihashes', + migrate, + revert +} diff --git a/migrations/migration-8/keys-encoding.js b/migrations/migration-8/keys-encoding.js new file mode 100644 index 0000000..029e17d --- /dev/null +++ b/migrations/migration-8/keys-encoding.js @@ -0,0 +1,62 @@ +const utils = require('../../src/utils') +const path = require('path') +const base32 = require('base32.js') +const Key = require('interface-datastore').Key +const log = require('debug')('ipfs-repo-migrations:migration-8') + +const KEY_PREFIX = 'key_' + +function encode (name) { + name = Buffer.from(name) + const encoder = new base32.Encoder({ type: 'rfc4648' }) + return (KEY_PREFIX + encoder.finalize(name)).toLowerCase() +} + +function decode (name) { + if (!name.startsWith(KEY_PREFIX)) { + throw Error('Unknown format of key\'s name!') + } + + const decoder = new base32.Decoder({ type: 'rfc4648' }) + const decodedNameBuff = decoder.finalize(name.replace(KEY_PREFIX, '').toUpperCase()) + return Buffer.from(decodedNameBuff).toString() +} + +async function processFolder (store, prefix, fileNameProcessor) { + const query = { + prefix: `/${ prefix }` + } + + const files = store.query(query) + for await (let file of files) { + const name = String(file.key._buf).replace(`/${ prefix }/`, '') + const encodedFileName = fileNameProcessor(name) + const newKey = new Key(`${ prefix }/${ encodedFileName }`) + + await store.delete(file.key) + log(`Translating key's name '${ file.key }' into '${ newKey }'`) + await store.put(newKey, file.value) + } +} + +async function process (repoPath, options, processor) { + const { StorageBackend, storageOptions } = utils.getDatastoreAndOptions(options, 'keys') + + const store = new StorageBackend(path.join(repoPath, 'keys'), storageOptions) + try { + const info = processFolder(store, 'info', processor) + const data = processFolder(store, 'pkcs8', processor) + + return await Promise.all([info, data]) + } finally { + await store.close() + } +} + +exports.migrate = async function keyEncode (repoPath, options) { + return process(repoPath, options, encode) +} + +exports.revert = async function keyDecode (repoPath, options) { + return process(repoPath, options, decode) +} diff --git a/package.json b/package.json index a80f413..5b26148 100644 --- a/package.json +++ b/package.json @@ -44,13 +44,16 @@ "docs": "aegir docs" }, "dependencies": { + "base32.js": "~0.1.0", "chalk": "^2.4.2", - "datastore-fs": "~0.9.1", - "datastore-level": "~0.12.1", + "cids": "~0.7.0", + "datastore-core": "~0.7.0", + "datastore-fs": "~0.9.0", + "datastore-level": "~0.12.0", "debug": "^4.1.0", - "interface-datastore": "~0.8.0", - "proper-lockfile": "^4.1.1", - "yargs": "^14.2.0", + "interface-datastore": "~0.7.0", + "proper-lockfile": "^3.2.0", + "yargs": "^12.0.5", "yargs-promise": "^1.1.0" }, "devDependencies": { diff --git a/test/browser.js b/test/browser.js index b341bf9..f0646cf 100644 --- a/test/browser.js +++ b/test/browser.js @@ -43,6 +43,10 @@ describe('Browser specific tests', () => { require('./version-test')(createRepo, repoCleanup) }) + describe('migrations tests', () => { + require('./migrations/migration-8-test')(createRepo, repoCleanup) + }) + describe('init tests', () => { require('./init-test')(createRepo, repoCleanup) }) diff --git a/test/migrations/migration-8-test.js b/test/migrations/migration-8-test.js new file mode 100644 index 0000000..ee44ee3 --- /dev/null +++ b/test/migrations/migration-8-test.js @@ -0,0 +1,131 @@ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +chai.use(require('dirty-chai')) +const chaiAsPromised = require('chai-as-promised') +chai.use(chaiAsPromised) +const expect = chai.expect + +const path = require('path') +const keysMigration = require('../../migrations/migration-8/keys-encoding') +const blocksMigration = require('../../migrations/migration-8/blocks-to-multihash') +const Key = require('interface-datastore').Key +const Datastore = require('datastore-fs') +const core = require('datastore-core') +const ShardingStore = core.ShardingDatastore + +const keysFixtures = [ + ['aAa', 'key_mfawc'], + ['bbb', 'key_mjrge'], + ['self', 'key_onswyzq'] +] + +const blocksFixtures = [ + ['AFKREIBFG77IKIKDMBDUFDCSPK7H5TE5LNPMCSXYLPML27WSTT5YA5IUNU', 'CIQCKN76QUQUGYCHIKGFE6V6P3GJ2W26YFFPQW6YXV7NFHH3QB2RI3I'] +] + +async function bootstrapKeys (dir, encoded) { + const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: true }) + await store.open() + + let name + for (const keyNames of keysFixtures) { + name = encoded ? keyNames[1] : keyNames[0] + await store.put(new Key(`/pkcs8/${name}`), '') + await store.put(new Key(`/info/${name}`), '') + } + + await store.close() +} + +async function validateKeys (dir, shouldBeEncoded) { + const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: false }) + await store.open() + + let name + for (const keyNames of keysFixtures) { + name = shouldBeEncoded ? keyNames[1] : keyNames[0] + expect(await store.has(new Key(`/pkcs8/${name}`))).to.be.true(name) + expect(await store.has(new Key(`/info/${name}`))).to.be.true(name) + } + + await store.close() +} + +async function bootstrapBlocks (dir, encoded) { + const baseStore = new Datastore(path.join(dir, 'blocks'), { extension: '.data', createIfMissing: true }) + const shard = new core.shard.NextToLast(2) + const store = await ShardingStore.createOrOpen(baseStore, shard) + + let name + for (const blocksNames of blocksFixtures) { + name = encoded ? blocksNames[1] : blocksNames[0] + await store.put(new Key(name), '') + } + + await store.close() +} + +async function validateBlocks (dir, shouldBeEncoded) { + const baseStore = new Datastore(path.join(dir, 'blocks'), { extension: '.data', createIfMissing: false }) + const shard = new core.shard.NextToLast(2) + const store = await ShardingStore.createOrOpen(baseStore, shard) + + let newName, oldName + for (const blockNames of blocksFixtures) { + newName = shouldBeEncoded ? blockNames[1] : blockNames[0] + oldName = shouldBeEncoded ? blockNames[0] : blockNames[1] + expect(await store.has(new Key(oldName))).to.be.false(oldName) + expect(await store.has(new Key(newName))).to.be.true(newName) + } + + await store.close() +} + +module.exports = (setup, cleanup) => { + describe('migration 8', () => { + let dir + + beforeEach(async () => { + dir = await setup() + }) + afterEach(() => cleanup(dir)) + + it('should migrate keys forward', async () => { + await bootstrapKeys(dir, false) + await keysMigration.migrate(dir) + await validateKeys(dir, true) + }) + + it('should migrate keys backward', async () => { + await bootstrapKeys(dir, true) + await keysMigration.revert(dir) + await validateKeys(dir, false) + }) + + it('should fail to migrate keys backward with invalid key name', async () => { + const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: true }) + await store.open() + + await store.put(new Key('/pkcs8/mfawc'), '') + await store.put(new Key('/info/mfawc'), '') + + await store.close() + + expect(keysMigration.revert(dir)).to.eventually.rejectedWith('Unknown format of key\'s name!') + }) + + it('should migrate blocks forward', async () => { + await bootstrapBlocks(dir, false) + await blocksMigration.migrate(dir) + await validateBlocks(dir, true) + }) + // + // it('should migrate blocks backward', async () => { + // await bootstrapKeys(dir, true) + // await blocksMigration.revert(dir) + // await validateKeys(dir, false) + // }) + }) +} diff --git a/test/node.js b/test/node.js index 032889c..584275d 100644 --- a/test/node.js +++ b/test/node.js @@ -43,6 +43,10 @@ describe('Node specific tests', () => { require('./version-test')(createRepo, repoCleanup) }) + describe('migrations tests', () => { + require('./migrations/migration-8-test')(createRepo, repoCleanup) + }) + describe('init tests', () => { require('./init-test')(createRepo, repoCleanup) }) From fd79824561cbc950be72d9a18a1bf0db5c17ef8b Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 3 Jun 2020 15:34:07 +0100 Subject: [PATCH 2/7] chore: update deps --- migrations/migration-8/keys-encoding.js | 18 ++++++++++++------ package.json | 12 ++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/migrations/migration-8/keys-encoding.js b/migrations/migration-8/keys-encoding.js index 029e17d..2056761 100644 --- a/migrations/migration-8/keys-encoding.js +++ b/migrations/migration-8/keys-encoding.js @@ -19,6 +19,7 @@ function decode (name) { const decoder = new base32.Decoder({ type: 'rfc4648' }) const decodedNameBuff = decoder.finalize(name.replace(KEY_PREFIX, '').toUpperCase()) + return Buffer.from(decodedNameBuff).toString() } @@ -27,14 +28,19 @@ async function processFolder (store, prefix, fileNameProcessor) { prefix: `/${ prefix }` } - const files = store.query(query) - for await (let file of files) { + for await (let file of store.query(query)) { const name = String(file.key._buf).replace(`/${ prefix }/`, '') const encodedFileName = fileNameProcessor(name) const newKey = new Key(`${ prefix }/${ encodedFileName }`) + if (await store.has(newKey)) { + continue + } + await store.delete(file.key) + log(`Translating key's name '${ file.key }' into '${ newKey }'`) + await store.put(newKey, file.value) } } @@ -43,11 +49,11 @@ async function process (repoPath, options, processor) { const { StorageBackend, storageOptions } = utils.getDatastoreAndOptions(options, 'keys') const store = new StorageBackend(path.join(repoPath, 'keys'), storageOptions) - try { - const info = processFolder(store, 'info', processor) - const data = processFolder(store, 'pkcs8', processor) + await store.open() - return await Promise.all([info, data]) + try { + await processFolder(store, 'info', processor) + await processFolder(store, 'pkcs8', processor) } finally { await store.close() } diff --git a/package.json b/package.json index b7d491a..2a64cd4 100644 --- a/package.json +++ b/package.json @@ -47,25 +47,25 @@ "base32.js": "~0.1.0", "buffer": "^5.6.0", "chalk": "^4.0.0", - "cids": "^0.7.0", - "datastore-core": "~0.7.0", + "cids": "^0.8.1", + "datastore-core": "^1.1.0", "datastore-fs": "^1.0.0", "datastore-idb": "^1.0.2", "debug": "^4.1.0", - "interface-datastore": "~0.8.3", + "interface-datastore": "^1.0.2", "proper-lockfile": "^4.1.1", - "yargs": "^14.2.0", + "yargs": "^15.3.1", "yargs-promise": "^1.1.0" }, "devDependencies": { - "aegir": "^21.9.0", + "aegir": "^22.0.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "dirty-chai": "^2.0.1", "just-safe-set": "^2.1.0", "ncp": "^2.0.0", "rimraf": "^3.0.0", - "sinon": "^7.5.0" + "sinon": "^9.0.2" }, "engines": { "node": ">=10.0.0", From a8c8c6765f39e37e48b8cfafdbc5c9a6caae5f2d Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 9 Jun 2020 19:13:38 +0100 Subject: [PATCH 3/7] chore: fix up migration tool to work with newest datastores --- migrations/migration-8/blocks-to-multihash.js | 86 +++++++++++++------ migrations/migration-8/index.js | 8 +- migrations/migration-8/keys-encoding.js | 33 ++++--- package.json | 4 +- test/migrations/migration-8-test.js | 47 +++++----- 5 files changed, 113 insertions(+), 65 deletions(-) diff --git a/migrations/migration-8/blocks-to-multihash.js b/migrations/migration-8/blocks-to-multihash.js index d6897cd..677600a 100644 --- a/migrations/migration-8/blocks-to-multihash.js +++ b/migrations/migration-8/blocks-to-multihash.js @@ -6,9 +6,29 @@ const ShardingStore = core.ShardingDatastore const base32 = require('base32.js') const utils = require('../../src/utils') const log = require('debug')('ipfs-repo-migrations:migration-8') +const errCode = require('err-code') +const multihashes = require('multihashes') + +function isValidMultihash (buf) { + try { + multihashes.validate(buf) + return true + } catch (err) { + return false + } +} + +function isValidCid (buf) { + try { + CID.validateCID(new CID(buf)) + return true + } catch (err) { + return false + } +} // This function in js-ipfs-repo defaults to not using sharding -// but the default value of the options.sharding is True hence this +// but the default value of the options.sharding is true hence this // function defaults to use sharding. async function maybeWithSharding (filestore, options) { if (options.sharding === false) { @@ -16,55 +36,73 @@ async function maybeWithSharding (filestore, options) { } const shard = new core.shard.NextToLast(2) - return await ShardingStore.createOrOpen(filestore, shard) + + return ShardingStore.createOrOpen(filestore, shard) } -function keyToMultihash(key){ - // Key to CID +function keyToMultihash (key) { const decoder = new base32.Decoder() - const buff = decoder.finalize(key.toString().slice(1)) - const cid = new CID(Buffer.from(buff)) + const buf = Buffer.from(decoder.finalize(key.toString().slice(1))) - // CID to multihash + if (isValidMultihash(buf)) { + throw errCode(new Error('Key is already a multihash'), 'ERR_ALREADY_MIGRATED') + } + + // Extract multihash from CID const enc = new base32.Encoder() - return new Key('/' + enc.finalize(cid.multihash), false) + const multihash = enc.finalize(new CID(buf).multihash) + + return new Key(`/${multihash}`, false) } -function keyToCid(key){ - // Key to CID +function keyToCid (key) { const decoder = new base32.Decoder() - const buff = decoder.write(key.toString().slice(1)).finalize() - const cid = new CID(1, 'raw', Buffer.from(buff)) + const buf = Buffer.from(decoder.finalize(key.toString().slice(1))) + + if (isValidCid(buf) && new CID(buf).version === 1) { + throw errCode(new Error('Key is already a CID'), 'ERR_ALREADY_MIGRATED') + } + + if (!isValidMultihash(buf)) { + throw errCode(new Error('Key is already a CID'), 'ERR_ALREADY_MIGRATED') + } // CID to Key const enc = new base32.Encoder() - return new Key('/' + enc.finalize(cid.buffer), false) + return new Key('/' + enc.finalize(new CID(1, 'raw', buf).buffer), false) } -async function process(repoPath, options, keyFunction){ +async function process (repoPath, options, keyFunction){ const { StorageBackend, storageOptions } = utils.getDatastoreAndOptions(options, 'blocks') const baseStore = new StorageBackend(path.join(repoPath, 'blocks'), storageOptions) + await baseStore.open() const store = await maybeWithSharding(baseStore, storageOptions) + await store.open() try { - const batch = store.batch() let counter = 0 + for await (const block of store.query({})) { - const newKey = keyFunction(block.key) + try { + const newKey = keyFunction(block.key) - // If the Key is CIDv0 then it is raw multihash and nothing is changing - if(newKey.toString() !== block.key.toString()){ - counter += 1 + // If the Key is CIDv0 then it is raw multihash and nothing is changing + if(newKey.toString() !== block.key.toString()){ + counter += 1 - log(`Migrating Block from ${block.key.toString()} to ${newKey.toString()}`) - batch.delete(block.key) - batch.put(newKey, block.value) + log(`Migrating Block from ${block.key.toString()} to ${newKey.toString()}`) + await store.delete(block.key) + await store.put(newKey, block.value) + } + } catch (err) { + if (err.code !== 'ERR_ALREADY_MIGRATED') { + throw err + } } } - log(`Changing ${ counter } blocks`) - await batch.commit() + log(`Changed ${ counter } blocks`) } finally { await store.close() } diff --git a/migrations/migration-8/index.js b/migrations/migration-8/index.js index 949f7d5..390d984 100644 --- a/migrations/migration-8/index.js +++ b/migrations/migration-8/index.js @@ -7,9 +7,9 @@ const log = require('debug')('ipfs-repo-migrations:migration-8') async function migrate (repoPath, options) { await keysEncoding.migrate(repoPath, options) - try{ + try { await blocksToMultihash.migrate(repoPath, options) - }catch (e) { + } catch (e) { log('During migration of Blockstore to multihash exception was raised! Reverting keys part of migration!') await keysEncoding.revert(repoPath, options) @@ -20,9 +20,9 @@ async function migrate (repoPath, options) { async function revert (repoPath, options) { await keysEncoding.revert(repoPath, options) - try{ + try { await blocksToMultihash.revert(repoPath, options) - }catch (e) { + } catch (e) { log('During reversion of Blockstore to CID exception was raised! Migrating keys part of migration!') await keysEncoding.migrate(repoPath, options) diff --git a/migrations/migration-8/keys-encoding.js b/migrations/migration-8/keys-encoding.js index 2056761..e9df9b7 100644 --- a/migrations/migration-8/keys-encoding.js +++ b/migrations/migration-8/keys-encoding.js @@ -3,10 +3,15 @@ const path = require('path') const base32 = require('base32.js') const Key = require('interface-datastore').Key const log = require('debug')('ipfs-repo-migrations:migration-8') +const errCode = require('err-code') const KEY_PREFIX = 'key_' function encode (name) { + if (name.startsWith(KEY_PREFIX)) { + throw errCode(new Error('Key has already been encoded'), 'ERR_ALREADY_MIGRATED') + } + name = Buffer.from(name) const encoder = new base32.Encoder({ type: 'rfc4648' }) return (KEY_PREFIX + encoder.finalize(name)).toLowerCase() @@ -14,7 +19,7 @@ function encode (name) { function decode (name) { if (!name.startsWith(KEY_PREFIX)) { - throw Error('Unknown format of key\'s name!') + throw errCode(new Error('Key has already been decoded'), 'ERR_ALREADY_MIGRATED') } const decoder = new base32.Decoder({ type: 'rfc4648' }) @@ -28,20 +33,26 @@ async function processFolder (store, prefix, fileNameProcessor) { prefix: `/${ prefix }` } - for await (let file of store.query(query)) { - const name = String(file.key._buf).replace(`/${ prefix }/`, '') - const encodedFileName = fileNameProcessor(name) - const newKey = new Key(`${ prefix }/${ encodedFileName }`) + for await (let { key, value } of store.query(query)) { + try { + const name = String(key._buf).replace(`/${ prefix }/`, '') + const encodedFileName = fileNameProcessor(name) + const newKey = new Key(`${ prefix }/${ encodedFileName }`) - if (await store.has(newKey)) { - continue - } + if (await store.has(newKey)) { + continue + } - await store.delete(file.key) + await store.delete(key) - log(`Translating key's name '${ file.key }' into '${ newKey }'`) + log(`Translating key's name '${ key }' into '${ newKey }'`) - await store.put(newKey, file.value) + await store.put(newKey, value) + } catch (err) { + if (err.code !== 'ERR_ALREADY_MIGRATED') { + throw err + } + } } } diff --git a/package.json b/package.json index 2a64cd4..4814f37 100644 --- a/package.json +++ b/package.json @@ -50,9 +50,11 @@ "cids": "^0.8.1", "datastore-core": "^1.1.0", "datastore-fs": "^1.0.0", - "datastore-idb": "^1.0.2", + "datastore-idb": "ipfs/js-datastore-idb#fix/wrap-operations-in-transaction", "debug": "^4.1.0", + "err-code": "^2.0.0", "interface-datastore": "^1.0.2", + "multihashes": "^0.4.19", "proper-lockfile": "^4.1.1", "yargs": "^15.3.1", "yargs-promise": "^1.1.0" diff --git a/test/migrations/migration-8-test.js b/test/migrations/migration-8-test.js index ee44ee3..4a185fa 100644 --- a/test/migrations/migration-8-test.js +++ b/test/migrations/migration-8-test.js @@ -22,7 +22,8 @@ const keysFixtures = [ ] const blocksFixtures = [ - ['AFKREIBFG77IKIKDMBDUFDCSPK7H5TE5LNPMCSXYLPML27WSTT5YA5IUNU', 'CIQCKN76QUQUGYCHIKGFE6V6P3GJ2W26YFFPQW6YXV7NFHH3QB2RI3I'] + ['AFKREIBFG77IKIKDMBDUFDCSPK7H5TE5LNPMCSXYLPML27WSTT5YA5IUNU', + 'CIQCKN76QUQUGYCHIKGFE6V6P3GJ2W26YFFPQW6YXV7NFHH3QB2RI3I'] ] async function bootstrapKeys (dir, encoded) { @@ -43,11 +44,15 @@ async function validateKeys (dir, shouldBeEncoded) { const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: false }) await store.open() - let name for (const keyNames of keysFixtures) { - name = shouldBeEncoded ? keyNames[1] : keyNames[0] - expect(await store.has(new Key(`/pkcs8/${name}`))).to.be.true(name) - expect(await store.has(new Key(`/info/${name}`))).to.be.true(name) + const newName = shouldBeEncoded ? keyNames[1] : keyNames[0] + const oldName = shouldBeEncoded ? keyNames[0] : keyNames[1] + + expect(await store.has(new Key(`/pkcs8/${newName}`))).to.be.true(`/pkcs8/${oldName} was not migrated to /pkcs8/${newName}`) + expect(await store.has(new Key(`/info/${newName}`))).to.be.true(`/info/${oldName} was not migrated to /info/${newName}`) + + expect(await store.has(new Key(`/pkcs8/${oldName}`))).to.be.false(`/pkcs8/${oldName} was not removed`) + expect(await store.has(new Key(`/info/${oldName}`))).to.be.false(`/info/${oldName} was not removed`) } await store.close() @@ -56,6 +61,8 @@ async function validateKeys (dir, shouldBeEncoded) { async function bootstrapBlocks (dir, encoded) { const baseStore = new Datastore(path.join(dir, 'blocks'), { extension: '.data', createIfMissing: true }) const shard = new core.shard.NextToLast(2) + + await baseStore.open() const store = await ShardingStore.createOrOpen(baseStore, shard) let name @@ -70,14 +77,16 @@ async function bootstrapBlocks (dir, encoded) { async function validateBlocks (dir, shouldBeEncoded) { const baseStore = new Datastore(path.join(dir, 'blocks'), { extension: '.data', createIfMissing: false }) const shard = new core.shard.NextToLast(2) + + await baseStore.open() const store = await ShardingStore.createOrOpen(baseStore, shard) let newName, oldName for (const blockNames of blocksFixtures) { newName = shouldBeEncoded ? blockNames[1] : blockNames[0] oldName = shouldBeEncoded ? blockNames[0] : blockNames[1] - expect(await store.has(new Key(oldName))).to.be.false(oldName) - expect(await store.has(new Key(newName))).to.be.true(newName) + expect(await store.has(new Key(`/${oldName}`))).to.be.false(`${oldName} was not migrated to ${newName}`) + expect(await store.has(new Key(`/${newName}`))).to.be.true(`${newName} was not removed`) } await store.close() @@ -104,28 +113,16 @@ module.exports = (setup, cleanup) => { await validateKeys(dir, false) }) - it('should fail to migrate keys backward with invalid key name', async () => { - const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: true }) - await store.open() - - await store.put(new Key('/pkcs8/mfawc'), '') - await store.put(new Key('/info/mfawc'), '') - - await store.close() - - expect(keysMigration.revert(dir)).to.eventually.rejectedWith('Unknown format of key\'s name!') - }) - it('should migrate blocks forward', async () => { await bootstrapBlocks(dir, false) await blocksMigration.migrate(dir) await validateBlocks(dir, true) }) - // - // it('should migrate blocks backward', async () => { - // await bootstrapKeys(dir, true) - // await blocksMigration.revert(dir) - // await validateKeys(dir, false) - // }) + + it('should migrate blocks backward', async () => { + await bootstrapBlocks(dir, true) + await blocksMigration.revert(dir) + await validateBlocks(dir, false) + }) }) } From 274e6de5b9f7d9ab95d3a8c29d41758217cf4904 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 15 Jun 2020 14:03:19 +0100 Subject: [PATCH 4/7] fix: use datastore-level in the browser --- package.json | 4 ++-- test/browser.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 4814f37..9631de7 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "main": "src/index.js", "browser": { "./src/repo/lock.js": "./src/repo/lock-memory.js", - "datastore-fs": "datastore-idb" + "datastore-fs": "datastore-level" }, "bin": { "jsipfs-migrations": "./src/cli.js" @@ -50,7 +50,7 @@ "cids": "^0.8.1", "datastore-core": "^1.1.0", "datastore-fs": "^1.0.0", - "datastore-idb": "ipfs/js-datastore-idb#fix/wrap-operations-in-transaction", + "datastore-level": "^1.1.0", "debug": "^4.1.0", "err-code": "^2.0.0", "interface-datastore": "^1.0.2", diff --git a/test/browser.js b/test/browser.js index 71c974c..a8f55dd 100644 --- a/test/browser.js +++ b/test/browser.js @@ -3,7 +3,7 @@ const { Buffer } = require('buffer') const loadFixture = require('aegir/fixtures') -const Datastore = require('datastore-idb') +const Datastore = require('datastore-level') const Key = require('interface-datastore').Key const CONFIG_KEY = new Key('config') From 3630f7081e8d04927b59fc89d7eabd7507a9c350 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 23 Jun 2020 14:14:42 +0100 Subject: [PATCH 5/7] chore: formatting --- migrations/migration-8/blocks-to-multihash.js | 4 ++-- migrations/migration-8/index.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/migrations/migration-8/blocks-to-multihash.js b/migrations/migration-8/blocks-to-multihash.js index 677600a..2f37590 100644 --- a/migrations/migration-8/blocks-to-multihash.js +++ b/migrations/migration-8/blocks-to-multihash.js @@ -87,8 +87,8 @@ async function process (repoPath, options, keyFunction){ try { const newKey = keyFunction(block.key) - // If the Key is CIDv0 then it is raw multihash and nothing is changing - if(newKey.toString() !== block.key.toString()){ + // If the Key is base32 CIDv0 then there's nothing to do + if(newKey.toString() !== block.key.toString()) { counter += 1 log(`Migrating Block from ${block.key.toString()} to ${newKey.toString()}`) diff --git a/migrations/migration-8/index.js b/migrations/migration-8/index.js index 390d984..59edd44 100644 --- a/migrations/migration-8/index.js +++ b/migrations/migration-8/index.js @@ -32,7 +32,7 @@ async function revert (repoPath, options) { module.exports = { version: 8, - description: 'Transforms key\'s names into base32 encoding and converts Block store to use multihashes', + description: 'Transforms key names into base32 encoding and converts Block store to use bare multihashes encoded as base32', migrate, revert } From 04f677467c385f360b3154e62d0a568b904dbebf Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 24 Jun 2020 18:43:04 +0100 Subject: [PATCH 6/7] fix: failing test --- migrations/migration-8/blocks-to-multihash.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/migrations/migration-8/blocks-to-multihash.js b/migrations/migration-8/blocks-to-multihash.js index 91053dd..82d3080 100644 --- a/migrations/migration-8/blocks-to-multihash.js +++ b/migrations/migration-8/blocks-to-multihash.js @@ -60,7 +60,7 @@ function keyToMultihash (key) { } function keyToCid (key) { - const buf = mb.decode(`b${key.toString().substring(1)}`) + const buf = mb.decode(`b${key.toString().slice(1)}`) if (isValidCid(buf) && new CID(buf).version === 1) { throw errCode(new Error('Key is already a CID'), 'ERR_ALREADY_MIGRATED') @@ -71,8 +71,9 @@ function keyToCid (key) { } // CID to Key - const multihash = mb.encode('base32', new CID(1, 'raw', buf).buffer) - return new Key(`/${multihash.slice(1)}`, false) + const multihash = mb.encode('base32', new CID(1, 'raw', buf).buffer).slice(1) + + return new Key(`/${multihash}`.toUpperCase(), false) } async function process (repoPath, options, keyFunction){ From f69ff38229f67fd26b865ee54c5f2466836dc9a0 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 25 Jun 2020 11:10:18 +0100 Subject: [PATCH 7/7] chore: descope key encoding, only migrate block keys --- migrations/migration-8/blocks-to-multihash.js | 121 ------------------ migrations/migration-8/index.js | 90 ++++++++++--- migrations/migration-8/keys-encoding.js | 77 ----------- package.json | 2 - test/migrations/migration-8-test.js | 57 +-------- 5 files changed, 73 insertions(+), 274 deletions(-) delete mode 100644 migrations/migration-8/blocks-to-multihash.js delete mode 100644 migrations/migration-8/keys-encoding.js diff --git a/migrations/migration-8/blocks-to-multihash.js b/migrations/migration-8/blocks-to-multihash.js deleted file mode 100644 index 82d3080..0000000 --- a/migrations/migration-8/blocks-to-multihash.js +++ /dev/null @@ -1,121 +0,0 @@ -const path = require('path') -const CID = require('cids') -const Key = require('interface-datastore').Key -const core = require('datastore-core') -const ShardingStore = core.ShardingDatastore -const mb = require('multibase') -const utils = require('../../src/utils') -const log = require('debug')('ipfs-repo-migrations:migration-8') -const errCode = require('err-code') -const multihashes = require('multihashes') - -function isValidMultihash (buf) { - try { - multihashes.validate(buf) - return true - } catch (err) { - return false - } -} - -function isValidCid (buf) { - try { - CID.validateCID(new CID(buf)) - return true - } catch (err) { - return false - } -} - -// This function in js-ipfs-repo defaults to not using sharding -// but the default value of the options.sharding is true hence this -// function defaults to use sharding. -async function maybeWithSharding (filestore, options) { - if (options.sharding === false) { - return filestore - } - - const shard = new core.shard.NextToLast(2) - - return ShardingStore.createOrOpen(filestore, shard) -} - -function keyToMultihash (key) { - const buf = mb.decode(`b${key.toString().slice(1)}`) - - if (isValidMultihash(buf)) { - throw errCode(new Error('Key is already a multihash'), 'ERR_ALREADY_MIGRATED') - } - - // Extract multihash from CID - let multihash = new CID(buf).multihash - - // Encode and slice off multibase codec - multihash = mb.encode('base32', multihash).slice(1) - - // Should be uppercase for interop with go - multihash = multihash.toString().toUpperCase() - - return new Key(`/${multihash}`, false) -} - -function keyToCid (key) { - const buf = mb.decode(`b${key.toString().slice(1)}`) - - if (isValidCid(buf) && new CID(buf).version === 1) { - throw errCode(new Error('Key is already a CID'), 'ERR_ALREADY_MIGRATED') - } - - if (!isValidMultihash(buf)) { - throw errCode(new Error('Key is already a CID'), 'ERR_ALREADY_MIGRATED') - } - - // CID to Key - const multihash = mb.encode('base32', new CID(1, 'raw', buf).buffer).slice(1) - - return new Key(`/${multihash}`.toUpperCase(), false) -} - -async function process (repoPath, options, keyFunction){ - const { StorageBackend, storageOptions } = utils.getDatastoreAndOptions(options, 'blocks') - - const baseStore = new StorageBackend(path.join(repoPath, 'blocks'), storageOptions) - await baseStore.open() - const store = await maybeWithSharding(baseStore, storageOptions) - await store.open() - - try { - let counter = 0 - - for await (const block of store.query({})) { - try { - const newKey = keyFunction(block.key) - - // If the Key is base32 CIDv0 then there's nothing to do - if(newKey.toString() !== block.key.toString()) { - counter += 1 - - log(`Migrating Block from ${block.key.toString()} to ${newKey.toString()}`) - await store.delete(block.key) - await store.put(newKey, block.value) - } - } catch (err) { - if (err.code !== 'ERR_ALREADY_MIGRATED') { - throw err - } - } - } - - log(`Changed ${ counter } blocks`) - } finally { - await store.close() - } -} - -exports.migrate = function blocksMigrate (repoPath, options) { - return process(repoPath, options, keyToMultihash) -} - -exports.revert = function blocksRevert (repoPath, options) { - return process(repoPath, options, keyToCid) -} diff --git a/migrations/migration-8/index.js b/migrations/migration-8/index.js index 59edd44..69febe9 100644 --- a/migrations/migration-8/index.js +++ b/migrations/migration-8/index.js @@ -1,38 +1,88 @@ 'use strict' -const keysEncoding = require('./keys-encoding') -const blocksToMultihash = require('./blocks-to-multihash') +const path = require('path') +const CID = require('cids') +const Key = require('interface-datastore').Key +const core = require('datastore-core') +const ShardingStore = core.ShardingDatastore +const mb = require('multibase') +const utils = require('../../src/utils') const log = require('debug')('ipfs-repo-migrations:migration-8') -async function migrate (repoPath, options) { - await keysEncoding.migrate(repoPath, options) +// This function in js-ipfs-repo defaults to not using sharding +// but the default value of the options.sharding is true hence this +// function defaults to use sharding. +async function maybeWithSharding (filestore, options) { + if (options.sharding === false) { + return filestore + } - try { - await blocksToMultihash.migrate(repoPath, options) - } catch (e) { - log('During migration of Blockstore to multihash exception was raised! Reverting keys part of migration!') - await keysEncoding.revert(repoPath, options) + const shard = new core.shard.NextToLast(2) - throw e - } + return ShardingStore.createOrOpen(filestore, shard) +} + +function keyToMultihash (key) { + const buf = mb.decode(`b${key.toString().slice(1)}`) + + // Extract multihash from CID + let multihash = new CID(buf).multihash + + // Encode and slice off multibase codec + multihash = mb.encode('base32', multihash).slice(1) + + // Should be uppercase for interop with go + multihash = multihash.toString().toUpperCase() + + return new Key(`/${multihash}`, false) } -async function revert (repoPath, options) { - await keysEncoding.revert(repoPath, options) +function keyToCid (key) { + const buf = mb.decode(`b${key.toString().slice(1)}`) + + // CID to Key + const multihash = mb.encode('base32', new CID(1, 'raw', buf).buffer).slice(1) + + return new Key(`/${multihash}`.toUpperCase(), false) +} + +async function process (repoPath, options, keyFunction){ + const { StorageBackend, storageOptions } = utils.getDatastoreAndOptions(options, 'blocks') + + const baseStore = new StorageBackend(path.join(repoPath, 'blocks'), storageOptions) + await baseStore.open() + const store = await maybeWithSharding(baseStore, storageOptions) + await store.open() try { - await blocksToMultihash.revert(repoPath, options) - } catch (e) { - log('During reversion of Blockstore to CID exception was raised! Migrating keys part of migration!') - await keysEncoding.migrate(repoPath, options) + let counter = 0 - throw e + for await (const block of store.query({})) { + const newKey = keyFunction(block.key) + + // If the Key is base32 CIDv0 then there's nothing to do + if(newKey.toString() !== block.key.toString()) { + counter += 1 + + log(`Migrating Block from ${block.key.toString()} to ${newKey.toString()}`) + await store.delete(block.key) + await store.put(newKey, block.value) + } + } + + log(`Changed ${ counter } blocks`) + } finally { + await store.close() } } module.exports = { version: 8, description: 'Transforms key names into base32 encoding and converts Block store to use bare multihashes encoded as base32', - migrate, - revert + migrate: (repoPath, options = {}) => { + return process(repoPath, options, keyToMultihash) + }, + revert: (repoPath, options = {}) => { + return process(repoPath, options, keyToCid) + } } diff --git a/migrations/migration-8/keys-encoding.js b/migrations/migration-8/keys-encoding.js deleted file mode 100644 index 5f44e9b..0000000 --- a/migrations/migration-8/keys-encoding.js +++ /dev/null @@ -1,77 +0,0 @@ -const utils = require('../../src/utils') -const path = require('path') -const mb = require('multibase') -const Key = require('interface-datastore').Key -const log = require('debug')('ipfs-repo-migrations:migration-8') -const errCode = require('err-code') - -const KEY_PREFIX = 'key_' - -function encode (name) { - if (name.startsWith(KEY_PREFIX)) { - throw errCode(new Error('Key has already been encoded'), 'ERR_ALREADY_MIGRATED') - } - - name = Buffer.from(name) - return `${KEY_PREFIX}${mb.encode('base32', name).slice(1)}`.toLowerCase() -} - -function decode (name) { - if (!name.startsWith(KEY_PREFIX)) { - throw errCode(new Error('Key has already been decoded'), 'ERR_ALREADY_MIGRATED') - } - - const decodedNameBuff = mb.decode(`b${name.replace(KEY_PREFIX, '').toUpperCase()}`) - - return Buffer.from(decodedNameBuff).toString() -} - -async function processFolder (store, prefix, fileNameProcessor) { - const query = { - prefix: `/${ prefix }` - } - - for await (let { key, value } of store.query(query)) { - try { - const name = String(key._buf).replace(`/${ prefix }/`, '') - const encodedFileName = fileNameProcessor(name) - const newKey = new Key(`${ prefix }/${ encodedFileName }`) - - if (await store.has(newKey)) { - continue - } - - await store.delete(key) - - log(`Translating key's name '${ key }' into '${ newKey }'`) - - await store.put(newKey, value) - } catch (err) { - if (err.code !== 'ERR_ALREADY_MIGRATED') { - throw err - } - } - } -} - -async function process (repoPath, options, processor) { - const { StorageBackend, storageOptions } = utils.getDatastoreAndOptions(options, 'keys') - - const store = new StorageBackend(path.join(repoPath, 'keys'), storageOptions) - await store.open() - - try { - await processFolder(store, 'info', processor) - await processFolder(store, 'pkcs8', processor) - } finally { - await store.close() - } -} - -exports.migrate = async function keyEncode (repoPath, options) { - return process(repoPath, options, encode) -} - -exports.revert = async function keyDecode (repoPath, options) { - return process(repoPath, options, decode) -} diff --git a/package.json b/package.json index 52879d9..d1decf4 100644 --- a/package.json +++ b/package.json @@ -51,10 +51,8 @@ "datastore-fs": "^1.0.0", "datastore-level": "^1.1.0", "debug": "^4.1.0", - "err-code": "^2.0.3", "interface-datastore": "^1.0.2", "multibase": "^1.0.1", - "multihashes": "^1.0.1", "proper-lockfile": "^4.1.1", "yargs": "^15.3.1", "yargs-promise": "^1.1.0" diff --git a/test/migrations/migration-8-test.js b/test/migrations/migration-8-test.js index 4a185fa..aee36b0 100644 --- a/test/migrations/migration-8-test.js +++ b/test/migrations/migration-8-test.js @@ -8,56 +8,17 @@ chai.use(chaiAsPromised) const expect = chai.expect const path = require('path') -const keysMigration = require('../../migrations/migration-8/keys-encoding') -const blocksMigration = require('../../migrations/migration-8/blocks-to-multihash') +const migration = require('../../migrations/migration-8') const Key = require('interface-datastore').Key const Datastore = require('datastore-fs') const core = require('datastore-core') const ShardingStore = core.ShardingDatastore -const keysFixtures = [ - ['aAa', 'key_mfawc'], - ['bbb', 'key_mjrge'], - ['self', 'key_onswyzq'] -] - const blocksFixtures = [ ['AFKREIBFG77IKIKDMBDUFDCSPK7H5TE5LNPMCSXYLPML27WSTT5YA5IUNU', 'CIQCKN76QUQUGYCHIKGFE6V6P3GJ2W26YFFPQW6YXV7NFHH3QB2RI3I'] ] -async function bootstrapKeys (dir, encoded) { - const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: true }) - await store.open() - - let name - for (const keyNames of keysFixtures) { - name = encoded ? keyNames[1] : keyNames[0] - await store.put(new Key(`/pkcs8/${name}`), '') - await store.put(new Key(`/info/${name}`), '') - } - - await store.close() -} - -async function validateKeys (dir, shouldBeEncoded) { - const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: false }) - await store.open() - - for (const keyNames of keysFixtures) { - const newName = shouldBeEncoded ? keyNames[1] : keyNames[0] - const oldName = shouldBeEncoded ? keyNames[0] : keyNames[1] - - expect(await store.has(new Key(`/pkcs8/${newName}`))).to.be.true(`/pkcs8/${oldName} was not migrated to /pkcs8/${newName}`) - expect(await store.has(new Key(`/info/${newName}`))).to.be.true(`/info/${oldName} was not migrated to /info/${newName}`) - - expect(await store.has(new Key(`/pkcs8/${oldName}`))).to.be.false(`/pkcs8/${oldName} was not removed`) - expect(await store.has(new Key(`/info/${oldName}`))).to.be.false(`/info/${oldName} was not removed`) - } - - await store.close() -} - async function bootstrapBlocks (dir, encoded) { const baseStore = new Datastore(path.join(dir, 'blocks'), { extension: '.data', createIfMissing: true }) const shard = new core.shard.NextToLast(2) @@ -101,27 +62,15 @@ module.exports = (setup, cleanup) => { }) afterEach(() => cleanup(dir)) - it('should migrate keys forward', async () => { - await bootstrapKeys(dir, false) - await keysMigration.migrate(dir) - await validateKeys(dir, true) - }) - - it('should migrate keys backward', async () => { - await bootstrapKeys(dir, true) - await keysMigration.revert(dir) - await validateKeys(dir, false) - }) - it('should migrate blocks forward', async () => { await bootstrapBlocks(dir, false) - await blocksMigration.migrate(dir) + await migration.migrate(dir) await validateBlocks(dir, true) }) it('should migrate blocks backward', async () => { await bootstrapBlocks(dir, true) - await blocksMigration.revert(dir) + await migration.revert(dir) await validateBlocks(dir, false) }) })