diff --git a/package.json b/package.json index 778a7b3263..76a600a12f 100644 --- a/package.json +++ b/package.json @@ -99,14 +99,14 @@ "hapi": "^16.5.2", "hapi-set-header": "^1.0.2", "hoek": "^4.2.0", - "ipfs-api": "^14.3.3", + "ipfs-api": "^14.3.5", "ipfs-bitswap": "~0.17.2", "ipfs-block": "~0.6.0", "ipfs-block-service": "~0.12.0", "ipfs-multipart": "~0.1.0", "ipfs-repo": "~0.17.0", "ipfs-unixfs": "~0.1.13", - "ipfs-unixfs-engine": "~0.22.3", + "ipfs-unixfs-engine": "~0.22.5", "ipld-resolver": "~0.13.2", "is-ipfs": "^0.3.0", "is-stream": "^1.1.0", diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index 82fe7ecb18..61b849ff06 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -8,6 +8,7 @@ const pull = require('pull-stream') const paramap = require('pull-paramap') const zip = require('pull-zip') const toPull = require('stream-to-pull-stream') +const mh = require('multihashes') const utils = require('../../utils') const print = require('../../utils').print @@ -151,6 +152,11 @@ module.exports = { type: 'boolean', default: false, describe: 'Write no output' + }, + hash: { + type: 'string', + choices: [undefined].concat(Object.keys(mh.names)), + describe: 'Hash function to use. Will set Cid version to 1 if used. (experimental)' } }, @@ -160,8 +166,9 @@ module.exports = { const options = { strategy: argv.trickle ? 'trickle' : 'balanced', shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity, - 'cid-version': argv['cid-version'], - 'raw-leaves': argv['raw-leaves'] + cidVersion: argv.cidVersion, + rawLeaves: argv.rawLeaves, + hashAlg: argv.hash } // Temporary restriction on raw-leaves: @@ -172,11 +179,23 @@ module.exports = { // cid-version > 0 unless explicitly set to false. // // This retains feature parity without having to implement raw-leaves. - if (argv['cid-version'] > 0 && argv['raw-leaves'] !== false) { + if (argv.cidVersion > 0 && argv.rawLeaves !== false) { throw new Error('Implied argument raw-leaves must be passed and set to false when cid-version is > 0') } - if (argv['raw-leaves']) { + // Temporary restriction on raw-leaves: + // When hash != undefined then raw-leaves MUST be present and false. + // + // This is because raw-leaves is not yet implemented in js-ipfs, + // and go-ipfs changes the value of raw-leaves to true when + // hash != undefined unless explicitly set to false. + // + // This retains feature parity without having to implement raw-leaves. + if (argv.hash && argv.rawLeaves !== false) { + throw new Error('Implied argument raw-leaves must be passed and set to false when hash argument is specified') + } + + if (argv.rawLeaves) { throw new Error('Not implemented: raw-leaves') } diff --git a/src/core/components/files.js b/src/core/components/files.js index 6a9f5fbced..754bce6d76 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -20,6 +20,10 @@ module.exports = function files (self) { shardSplitThreshold: self._options.EXPERIMENTAL.sharding ? 1000 : Infinity }, options) + if (opts.hashAlg && opts.cidVersion !== 1) { + opts.cidVersion = 1 + } + return pull( pull.map(normalizeContent), pull.flatten(), @@ -65,6 +69,13 @@ module.exports = function files (self) { return callback(new Error('Invalid arguments, data must be an object, Buffer or readable stream')) } + options = options || {} + + // CID v0 is for multihashes encoded with sha2-256 + if (options.hashAlg && options.cidVersion !== 1) { + options.cidVersion = 1 + } + pull( pull.values(normalizeContent(data)), importer(self._ipldResolver, options), @@ -117,15 +128,15 @@ module.exports = function files (self) { function prepareFile (self, opts, file, callback) { opts = opts || {} - waterfall([ - (cb) => self.object.get(file.multihash, cb), - (node, cb) => { - let cid = new CID(node.multihash) + let cid = new CID(file.multihash) - if (opts['cid-version'] === 1) { - cid = cid.toV1() - } + if (opts.cidVersion === 1) { + cid = cid.toV1() + } + waterfall([ + (cb) => self.object.get(cid, cb), + (node, cb) => { const b58Hash = cid.toBaseEncodedString() cb(null, { diff --git a/src/core/components/object.js b/src/core/components/object.js index 4c716c4504..8402e9befe 100644 --- a/src/core/components/object.js +++ b/src/core/components/object.js @@ -10,20 +10,6 @@ const mh = require('multihashes') const Unixfs = require('ipfs-unixfs') const assert = require('assert') -function normalizeMultihash (multihash, enc) { - if (typeof multihash === 'string') { - if (enc === 'base58' || !enc) { - return multihash - } - - return new Buffer(multihash, enc) - } else if (Buffer.isBuffer(multihash)) { - return multihash - } else { - throw new Error('unsupported multihash') - } -} - function parseBuffer (buf, encoding, callback) { switch (encoding) { case 'json': @@ -178,20 +164,17 @@ module.exports = function object (self) { } }), - get: promisify((multihash, options, callback) => { + get: promisify((cid, options, callback) => { if (typeof options === 'function') { callback = options options = {} } - let mh - try { - mh = normalizeMultihash(multihash, options.enc) + cid = new CID(cid) } catch (err) { return callback(err) } - const cid = new CID(mh) self._ipldResolver.get(cid, (err, result) => { if (err) { @@ -204,13 +187,19 @@ module.exports = function object (self) { }) }), - data: promisify((multihash, options, callback) => { + data: promisify((cid, options, callback) => { if (typeof options === 'function') { callback = options options = {} } - self.object.get(multihash, options, (err, node) => { + try { + cid = new CID(cid) + } catch (err) { + return callback(err) + } + + self.object.get(cid, options, (err, node) => { if (err) { return callback(err) } diff --git a/src/http-api/resources/files.js b/src/http-api/resources/files.js index 35c53a32e6..5b4b7fd7c4 100644 --- a/src/http-api/resources/files.js +++ b/src/http-api/resources/files.js @@ -145,19 +145,19 @@ exports.add = { query: Joi.object() .keys({ 'cid-version': Joi.number().integer().min(0).max(1), + hash: Joi.string().valid(Object.keys(mh.names)), // Temporary restriction on raw-leaves: - // When cid-version=1 then raw-leaves MUST be present and false. + // When cid-version > 0 or hash != undefined then raw-leaves MUST be + // present and false. // // This is because raw-leaves is not yet implemented in js-ipfs, // and go-ipfs changes the value of raw-leaves to true when - // cid-version > 0 unless explicitly set to false. + // cid-version > 0 or hash != undefined unless explicitly set to false. // // This retains feature parity without having to implement raw-leaves. - 'raw-leaves': Joi.any().when('cid-version', { - is: 1, - then: Joi.boolean().valid(false).required(), - otherwise: Joi.boolean().valid(false) - }) + 'raw-leaves': Joi.boolean().valid(false) + .when('cid-version', { is: 1, then: Joi.required() }) + .when('hash', { is: Joi.string(), then: Joi.required() }) }) // TODO: Necessary until validate "recursive", "stream-channels" etc. .options({ allowUnknown: true }) @@ -208,8 +208,9 @@ exports.add = { }) const options = { - 'cid-version': request.query['cid-version'], - 'raw-leaves': request.query['raw-leaves'] + cidVersion: request.query['cid-version'], + rawLeaves: request.query['raw-leaves'], + hashAlg: request.query['hash'] } pull( diff --git a/src/http-api/resources/object.js b/src/http-api/resources/object.js index 4a366f4e60..091312b682 100644 --- a/src/http-api/resources/object.js +++ b/src/http-api/resources/object.js @@ -7,6 +7,7 @@ const DAGLink = dagPB.DAGLink const DAGNode = dagPB.DAGNode const waterfall = require('async/waterfall') const series = require('async/series') +const CID = require('cids') const debug = require('debug') const log = debug('jsipfs:http-api:object') log.error = debug('jsipfs:http-api:object:error') @@ -20,9 +21,7 @@ exports.parseKey = (request, reply) => { } try { - return reply({ - key: mh.fromB58String(request.query.arg) - }) + return reply({ key: new CID(request.query.arg) }) } catch (err) { log.error(err) return reply({ diff --git a/test/cli/files.js b/test/cli/files.js index 85f7fbd9b4..599868dc0c 100644 --- a/test/cli/files.js +++ b/test/cli/files.js @@ -6,8 +6,22 @@ const fs = require('fs') const path = require('path') const compareDir = require('dir-compare').compareSync const rimraf = require('rimraf').sync +const CID = require('cids') +const mh = require('multihashes') const runOnAndOff = require('../utils/on-and-off') +// TODO: Test against all algorithms Object.keys(mh.names) +// This subset is known to work with both go-ipfs and js-ipfs as of 2017-09-05 +const HASH_ALGS = [ + 'sha1', + 'sha2-256', + 'sha2-512', + 'keccak-224', + 'keccak-256', + 'keccak-384', + 'keccak-512' +] + describe('files', () => runOnAndOff((thing) => { let ipfs const readme = fs.readFileSync(path.join(process.cwd(), '/src/init-files/init-docs/readme')) @@ -222,6 +236,17 @@ describe('files', () => runOnAndOff((thing) => { }) }) + HASH_ALGS.forEach((name) => { + it(`add with hash=${name} and raw-leaves=false`, () => { + return ipfs(`add src/init-files/init-docs/readme --hash=${name} --raw-leaves=false`) + .then((out) => { + const hash = out.split(' ')[1] + const cid = new CID(hash) + expect(mh.decode(cid.multihash).name).to.equal(name) + }) + }) + }) + it('cat', () => { return ipfs('files cat QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') .then((out) => {