From d10904e3c32843df1f4ab4751fec370f3e606c5c Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Fri, 8 Sep 2017 10:58:50 +0100 Subject: [PATCH 1/9] Starts tests for only-hash --- test/files.spec.js | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/files.spec.js b/test/files.spec.js index 8ea2f9f58..5e87b8ffb 100644 --- a/test/files.spec.js +++ b/test/files.spec.js @@ -8,6 +8,8 @@ const expect = chai.expect chai.use(dirtyChai) const isNode = require('detect-node') const loadFixture = require('aegir/fixtures') +const concat = require('concat-stream') +const through = require('through2') const FactoryClient = require('./ipfs-factory/client') @@ -73,6 +75,32 @@ describe('.files (the MFS API part)', function () { }) }) + it.only('files.add with onlyHash', (done) => { + const content = String(Math.random() + Date.now()) + const inputFile = { + path: content + '.txt', + content: Buffer.from(content) + } + + ipfs.files.add([inputFile], { onlyHash: true }, (err, res) => { + expect(err).to.not.exist() + + const hash = res[0].hash + + ipfs.get(hash, (err, res) => { + expect(err).to.not.exist() + + res.pipe(through.obj((file, enc, next) => { + file.content.pipe(concat((ret) => { + // console.log(content.toString, content) + expect(content).to.equal() + next() + })) + }, done)) + }) + }) + }) + it('files.mkdir', (done) => { ipfs.files.mkdir('/test-folder', done) }) From c490b65b7c546dd484ac8e540403afefaa0b437a Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Fri, 8 Sep 2017 16:20:44 +0100 Subject: [PATCH 2/9] Adds support for only-hash flags This PR adds the code to skip the step where files are written to disk if you pass in the --only-hash flag over an http request --- src/files/add.js | 29 +++++++++++++++++++++- src/files/get.js | 1 - src/utils/get-dagnode.js | 1 - src/utils/request-api.js | 1 + test/files.spec.js | 52 +++++++++++++++++++++++++++++++--------- 5 files changed, 70 insertions(+), 14 deletions(-) diff --git a/src/files/add.js b/src/files/add.js index d5aac24c5..d9cde39b9 100644 --- a/src/files/add.js +++ b/src/files/add.js @@ -21,8 +21,35 @@ module.exports = (send) => { return callback(new Error('"files" must be a buffer, readable stream, or array of objects')) } - const request = { path: 'add', files: files, qs: opts } + const qs = {} + if (opts['cid-version'] != null) { + qs['cid-version'] = opts['cid-version'] + } else if (opts.cidVersion != null) { + qs['cid-version'] = opts.cidVersion + } + + if (opts['raw-leaves'] != null) { + qs['raw-leaves'] = opts['raw-leaves'] + } else if (opts.rawLeaves != null) { + qs['raw-leaves'] = opts.rawLeaves + } + + if (opts.hash != null) { + qs.hash = opts.hash + } else if (opts.hashAlg != null) { + qs.hash = opts.hashAlg + } + + if (opts['only-hash'] != null) { + qs['only-hash'] = opts['only-hash'] + } else if (opts.onlyHash != null) { + qs['only-hash'] = opts.onlyHash + } + + const request = { path: 'add', files: files, qs: qs } + + if (qs['only-hash']) return send(request, callback) // Transform the response stream to DAGNode values const transform = (res, callback) => DAGNodeStream.streamToValue(send, res, callback) send.andTransform(request, transform, callback) diff --git a/src/files/get.js b/src/files/get.js index 056c46d01..c1b997255 100644 --- a/src/files/get.js +++ b/src/files/get.js @@ -33,7 +33,6 @@ module.exports = (send) => { args: path, qs: opts } - // Convert the response stream to TarStream objects send.andTransform(request, TarStreamToObjects, callback) }) diff --git a/src/utils/get-dagnode.js b/src/utils/get-dagnode.js index edb89c4d9..3b8525332 100644 --- a/src/utils/get-dagnode.js +++ b/src/utils/get-dagnode.js @@ -10,7 +10,6 @@ module.exports = function (send, hash, callback) { // Until js-ipfs supports object/get and object/data by CID // we need to convert our CID or multihash hash into a multihash const multihash = mh.toB58String(new CID(hash).multihash) - // Retrieve the object and its data in parallel, then produce a DAGNode // instance using this information. parallel([ diff --git a/src/utils/request-api.js b/src/utils/request-api.js index 5f78b1e07..32a852870 100644 --- a/src/utils/request-api.js +++ b/src/utils/request-api.js @@ -147,6 +147,7 @@ function requestAPI (config, options, callback) { return qsDefaultEncoder(data) } }) + const req = request(config.protocol)({ hostname: config.host, path: `${config['api-path']}${options.path}?${qs}`, diff --git a/test/files.spec.js b/test/files.spec.js index 5e87b8ffb..f81bd8662 100644 --- a/test/files.spec.js +++ b/test/files.spec.js @@ -10,6 +10,7 @@ const isNode = require('detect-node') const loadFixture = require('aegir/fixtures') const concat = require('concat-stream') const through = require('through2') +const streamToValue = require('../src/utils/stream-to-value') const FactoryClient = require('./ipfs-factory/client') @@ -17,6 +18,14 @@ const testfile = isNode ? loadFixture(__dirname, '/fixtures/testfile.txt') : loadFixture(__dirname, 'fixtures/testfile.txt') +function createTestFile (content) { + content = content || String(Math.random() + Date.now()) + return { + path: content + '.txt', + content: Buffer.from(content) + } +} + describe('.files (the MFS API part)', function () { this.timeout(120 * 1000) @@ -75,28 +84,49 @@ describe('.files (the MFS API part)', function () { }) }) - it.only('files.add with onlyHash', (done) => { + it('files.add without only-hash', (done) => { const content = String(Math.random() + Date.now()) - const inputFile = { - path: content + '.txt', - content: Buffer.from(content) - } + const inputFile = createTestFile(content) - ipfs.files.add([inputFile], { onlyHash: true }, (err, res) => { + ipfs.files.add([inputFile], { onlyHash: false }, (err, res) => { expect(err).to.not.exist() const hash = res[0].hash + let retrievedContent = '' ipfs.get(hash, (err, res) => { expect(err).to.not.exist() - res.pipe(through.obj((file, enc, next) => { - file.content.pipe(concat((ret) => { - // console.log(content.toString, content) - expect(content).to.equal() + res.pipe(through.obj((file, encoding, next) => { + file.content.pipe(concat(retrieved => { + retrievedContent += retrieved.toString() next() })) - }, done)) + }, () => { + expect(content).to.equal(retrievedContent) + done() + })) + }) + }) + }) + + it.only('files.add with only-hash', (done) => { + const inputFile = createTestFile() + + ipfs.files.add([inputFile], {'only-hash': true}, (err, res) => { + expect(err).to.not.exist() + + streamToValue(res, (err, collected) => { + expect(err).to.not.exist() + + const hash = collected[0].Hash + + ipfs.files.get(hash, (err, res) => { + expect(err).to.exist() + const message = `Failed to get block for ${hash}: context canceled` + expect(err.message.indexOf(message)).to.be.above(-1) + }) + done() }) }) }) From 64dab7376a24c03ab99f90fc06cb6d5c08c04042 Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Fri, 8 Sep 2017 16:43:50 +0100 Subject: [PATCH 3/9] Update request-api.js --- src/utils/request-api.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/utils/request-api.js b/src/utils/request-api.js index 32a852870..5f78b1e07 100644 --- a/src/utils/request-api.js +++ b/src/utils/request-api.js @@ -147,7 +147,6 @@ function requestAPI (config, options, callback) { return qsDefaultEncoder(data) } }) - const req = request(config.protocol)({ hostname: config.host, path: `${config['api-path']}${options.path}?${qs}`, From 25e922e9950f42a452c01af9994a7e9912d41eac Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Fri, 8 Sep 2017 16:44:11 +0100 Subject: [PATCH 4/9] Update get.js --- src/files/get.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/files/get.js b/src/files/get.js index c1b997255..e9d560ae3 100644 --- a/src/files/get.js +++ b/src/files/get.js @@ -33,6 +33,7 @@ module.exports = (send) => { args: path, qs: opts } + // Convert the response stream to TarStream objects send.andTransform(request, TarStreamToObjects, callback) }) From a59cb6f196eb1bf25f69289562997efe751c047f Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Fri, 8 Sep 2017 16:44:38 +0100 Subject: [PATCH 5/9] Update get.js --- src/files/get.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/files/get.js b/src/files/get.js index e9d560ae3..056c46d01 100644 --- a/src/files/get.js +++ b/src/files/get.js @@ -33,7 +33,7 @@ module.exports = (send) => { args: path, qs: opts } - + // Convert the response stream to TarStream objects send.andTransform(request, TarStreamToObjects, callback) }) From 5cb8ca12c9190f594ed47db836512b4d0cd4303e Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Fri, 8 Sep 2017 17:12:45 +0100 Subject: [PATCH 6/9] Queries ipfs.refs.local for hash Rather than use ipfs.get to search for a hash (which should not be there anyway) just use ipfs.refs.local to verify that the hash has not been added to ipfs --- test/files.spec.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/files.spec.js b/test/files.spec.js index 6d9bc34fa..9d2cefb37 100644 --- a/test/files.spec.js +++ b/test/files.spec.js @@ -27,6 +27,8 @@ function createTestFile (content) { } } +// TODO: Test against all algorithms Object.keys(mh.names) +function createTestFile (content) { +// This subset is known to work with both go-ipfs and js-ipfs as of 2017-09-05 const HASH_ALGS = [ 'sha1', 'sha2-256', @@ -121,7 +123,7 @@ describe('.files (the MFS API part)', function () { }) }) - it.only('files.add with only-hash', (done) => { + it('files.add with only-hash', (done) => { const inputFile = createTestFile() ipfs.files.add([inputFile], {'only-hash': true}, (err, res) => { @@ -132,12 +134,11 @@ describe('.files (the MFS API part)', function () { const hash = collected[0].Hash - ipfs.files.get(hash, (err, res) => { - expect(err).to.exist() - const message = `Failed to get block for ${hash}: context canceled` - expect(err.message.indexOf(message)).to.be.above(-1) + ipfs.refs.local().then((refs) => { + const hashes = refs.map(r => r.Ref) + expect(hashes.indexOf(hash)).to.equal(-1) + done() }) - done() }) }) }) From ab1534310b1d22aa1a7c548aa1969d4a00498e56 Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Fri, 8 Sep 2017 17:16:29 +0100 Subject: [PATCH 7/9] Clean typo in comment after merge conflict --- test/files.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/files.spec.js b/test/files.spec.js index 9d2cefb37..13af62f87 100644 --- a/test/files.spec.js +++ b/test/files.spec.js @@ -27,7 +27,7 @@ function createTestFile (content) { } } -// TODO: Test against all algorithms Object.keys(mh.names) +function createTestFile (content) { +// TODO: Test against all algorithms Object.keys(mh.names) // This subset is known to work with both go-ipfs and js-ipfs as of 2017-09-05 const HASH_ALGS = [ 'sha1', From 65648bfdaadd82ec4971e0480a1a9fb472344ad7 Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Fri, 8 Sep 2017 17:24:28 +0100 Subject: [PATCH 8/9] Promised based tests Adds promise based tests and uses same method to verify test behaviour --- test/files.spec.js | 53 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/test/files.spec.js b/test/files.spec.js index 13af62f87..5d88a13c5 100644 --- a/test/files.spec.js +++ b/test/files.spec.js @@ -39,7 +39,7 @@ const HASH_ALGS = [ 'keccak-512' ] -describe('.files (the MFS API part)', function () { +describe.only('.files (the MFS API part)', function () { this.timeout(120 * 1000) let ipfs @@ -105,20 +105,11 @@ describe('.files (the MFS API part)', function () { expect(err).to.not.exist() const hash = res[0].hash - let retrievedContent = '' - ipfs.get(hash, (err, res) => { - expect(err).to.not.exist() - - res.pipe(through.obj((file, encoding, next) => { - file.content.pipe(concat(retrieved => { - retrievedContent += retrieved.toString() - next() - })) - }, () => { - expect(content).to.equal(retrievedContent) - done() - })) + ipfs.refs.local().then((refs) => { + const hashes = refs.map(r => r.Ref) + expect(hashes.indexOf(hash)).to.be.above(-1) + done() }) }) }) @@ -319,6 +310,40 @@ describe('.files (the MFS API part)', function () { }) }) + it('files.add without only-hash', (done) => { + const inputFile = createTestFile() + + ipfs.files.add([inputFile], { onlyHash: false }) + .then((res) => { + const hash = res[0].hash + + ipfs.refs.local().then((refs) => { + const hashes = refs.map(r => r.Ref) + expect(hashes.indexOf(hash)).to.be.above(-1) + done() + }) + }) + }) + + it('files.add with only-hash', (done) => { + const inputFile = createTestFile() + + ipfs.files.add([inputFile], {'only-hash': true}) + .then((res) => { + streamToValue(res, (err, collected) => { + expect(err).to.not.exist() + + const hash = collected[0].Hash + + ipfs.refs.local().then((refs) => { + const hashes = refs.map(r => r.Ref) + expect(hashes.indexOf(hash)).to.equal(-1) + done() + }) + }) + }) + }) + HASH_ALGS.forEach((name) => { it(`files.add with hash=${name} and raw-leaves=false`, () => { const content = String(Math.random() + Date.now()) From ac2b268c2b4428d1cfa8363b44749754a5558bea Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Fri, 8 Sep 2017 17:26:31 +0100 Subject: [PATCH 9/9] removes .only --- test/files.spec.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/files.spec.js b/test/files.spec.js index 5d88a13c5..ae72e4f3f 100644 --- a/test/files.spec.js +++ b/test/files.spec.js @@ -8,8 +8,6 @@ const expect = chai.expect chai.use(dirtyChai) const isNode = require('detect-node') const loadFixture = require('aegir/fixtures') -const concat = require('concat-stream') -const through = require('through2') const streamToValue = require('../src/utils/stream-to-value') const mh = require('multihashes') const CID = require('cids') @@ -39,7 +37,7 @@ const HASH_ALGS = [ 'keccak-512' ] -describe.only('.files (the MFS API part)', function () { +describe('.files (the MFS API part)', function () { this.timeout(120 * 1000) let ipfs