From 0f86d13d1c91d10e3d30c75ba3cbb499e3bdd897 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 14 Aug 2018 22:20:32 +0100 Subject: [PATCH 1/4] fix: remove external URLs from addFromURL tests Relying on external URLs makes our tests dependent on the availability and response time of these services as well as not being able to run without an internet connection. This PR removes those requests to external URLs and instead spins up test HTTP(s) servers as they are needed to serve the content that `ipfs.util.addFromURL` consumes. resolves #803 License: MIT Signed-off-by: Alan Shaw --- test/fixtures/ssl/cert.pem | 20 +++ test/fixtures/ssl/privkey.pem | 27 ++++ test/util.spec.js | 264 +++++++++++++++++++++++++++------- 3 files changed, 263 insertions(+), 48 deletions(-) create mode 100644 test/fixtures/ssl/cert.pem create mode 100644 test/fixtures/ssl/privkey.pem diff --git a/test/fixtures/ssl/cert.pem b/test/fixtures/ssl/cert.pem new file mode 100644 index 000000000..654550a16 --- /dev/null +++ b/test/fixtures/ssl/cert.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDOzCCAiMCCQCVqVeRIp9pFDANBgkqhkiG9w0BAQUFADBnMQswCQYDVQQGEwJV +UzENMAsGA1UECAwEVXRhaDEOMAwGA1UEBwwFUHJvdm8xIzAhBgNVBAoMGkFDTUUg +U2lnbmluZyBBdXRob3JpdHkgSW5jMRQwEgYDVQQDDAtleGFtcGxlLmNvbTAeFw0x +ODA4MTQyMDEzNTdaFw0xOTEyMjcyMDEzNTdaMFgxCzAJBgNVBAYTAlVTMQ0wCwYD +VQQIDARVdGFoMQ4wDAYDVQQHDAVQcm92bzEWMBQGA1UECgwNQUNNRSBUZWNoIElu +YzESMBAGA1UEAwwJMTI3LjAuMC4xMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEA6x6mTXV+rC35QW/sPutT1O1cugtnw+UsJx7EGgzyjh7EoXE3gb7sO96P +tOI5zknb0vecckbiVkesmLnAs2iNa1u9EiRr6WHdc+1MfUCxyHRfP731vRZyo0kx +bSXerE0qZ2N3M1XyndZF7VMthKDKIg0ZR0TvdjwLqyLYEHAnRBhJLRS0Oy0fC6Of +VWCO3gIuk1HkTXH+/ZMA/obqrtlisxY85mMdlRz+1PNdZBMf+NxmrXN59uq+JqUu +8/v1oQ8jH2iU9IWeqyawHDEvPW3aDorfaWGyats5Xd3cT2Ph4xF9tBLT+3PDGU8c +oBmTHWDenYn+TCkCseayo1JCO5igJQIDAQABMA0GCSqGSIb3DQEBBQUAA4IBAQCr +R7eZxicHjJoRcbsPBDQpzx9uSux3uvpN93pyJrXXHiil/5SE7CYeDqv5+nV2p6HA +6KONUAmpId0iHAEi9u+0/LgPWyYQMzT3sfBhkO8RRaMYI87VuKbk5PFmlZbD843+ +Qmg3Se2F7BDnTf88xA6QWR4DCejy+ZHfDRFrh3xfFl4tX1UNgqiTGfjPCzblhWx9 +ygzlT+flN2j3NkAlhUEV89pnH4EQWILePMTT4wh2XOQj1VFJ+2ATojHFVUTtNWAJ +xrY/Q9cMYsZ++I8i9bHMZoyc1bSUd5CNFpQdfjVzlgMPT9Jj/fzWIQz+wq0KeRLI +dLWsa2MZr0GZnTU39YwH +-----END CERTIFICATE----- diff --git a/test/fixtures/ssl/privkey.pem b/test/fixtures/ssl/privkey.pem new file mode 100644 index 000000000..1e78e3d1a --- /dev/null +++ b/test/fixtures/ssl/privkey.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA6x6mTXV+rC35QW/sPutT1O1cugtnw+UsJx7EGgzyjh7EoXE3 +gb7sO96PtOI5zknb0vecckbiVkesmLnAs2iNa1u9EiRr6WHdc+1MfUCxyHRfP731 +vRZyo0kxbSXerE0qZ2N3M1XyndZF7VMthKDKIg0ZR0TvdjwLqyLYEHAnRBhJLRS0 +Oy0fC6OfVWCO3gIuk1HkTXH+/ZMA/obqrtlisxY85mMdlRz+1PNdZBMf+NxmrXN5 +9uq+JqUu8/v1oQ8jH2iU9IWeqyawHDEvPW3aDorfaWGyats5Xd3cT2Ph4xF9tBLT ++3PDGU8coBmTHWDenYn+TCkCseayo1JCO5igJQIDAQABAoIBAH5fbfFqOpie6T8T +wj4bTGbA4bsZkD9JeU7ZiXubA/ABd5xyduwky2JugH0vrvRC3IVrE0qU8OiBA7Le +/EUx5/kRSPFsZBf/wwChRiB4WlYsvllLZ76nRxyepZNN7H5dx3Hkk1gjVREi71jd +ATUtGxfsRG77DV5WbcshIlLLhT9iaohsalmClAFBmwhqnRMvOXHiQyRbvB0fOX08 +uVlObOqo9jLB8N5C/ux+wFEP4wi/AxVqs9ih7Ss7T7+pmOCVWhOnbYcoY2jdaJ11 +iLK4F3rv/jQ82OwUpzrWsPedmZUzlOO8xdV3b8hOcPHs/BKvYed7aHSn6b5eVKKT +zT8vQoECgYEA+K9pvw9K/7+F810MHG+nZ0gtVWmXJp49jB7zQ6QMIex2sUajY2y9 +bEJX8T6rdu3qd+HYU4zl3qt+MUnsVQEBNkLPAn3od0qIWXxu1SL2GF8SDV1xJWK1 +Fp0YDe9blaz1JsmSgieNcSoSwqE2V97Wfd/m+EUfyhQt9HX55H5UgAUCgYEA8gkW +0xZKuHhBMYpcES2P5H5q6HN2fcEQycMuS3agAOhrFPYUT1DVNhbfNVmbOvL2NRWI +hXixo5SkuEuq2fjmEoeLPTmeKO5LM4IVqovWCYomSftKDpzw4HRn2jvKzi2+mg8J +qktIMqRqHu/O1NUIsszCIu4c5DzUdhr4N7GXOaECgYAEd1oF1Wd6al0kfsJN7G9s +Om6d/xR43BSs5I1n5JVXMqD7FBKxIW3ReOuNaJu5uhIg7wxsi7ZBJoFQr0wwRqFX +8SE4oTxAkDUcrlBrQYJ785Embkwu6LPp4Q5iia7yZDXO6YXZEo7GvoOxvSV1tInT +nubOBKfKgExG/KttQBuSZQKBgAzYOqPdLP35M8yDQTuQJXDE3LuVVRZ7Zn6uowhS +NU+XBgfIv28uJQKH2DSmmrxYJITQrbwXmaXKv6sgKOMEeIFHPDZ1llUpwEftgWTZ +ovRCpqGKenWoEoh25QQJ5Eto1hKq9aJZ+GznmNIne9yDqcCDaVIdPN9H8yaJa97Y +x+PBAoGAOiK6xAbPyJSKDSTGZzdv8+yeOdNeJjRHxKJs+4YsDchmdumrqow83DBP +7ulIJD9pcmsWj+8fntMcsTX5mvzJd5LsKc7Maa5/LtitsLsynu78QFg4Njj8sAKn +3991i8J98DZ9zqmkxJJhGwstCHG+c+Q7lA6kZ1UdbWJwYwIHjAs= +-----END RSA PRIVATE KEY----- diff --git a/test/util.spec.js b/test/util.spec.js index f905911cc..213e19ba1 100644 --- a/test/util.spec.js +++ b/test/util.spec.js @@ -10,6 +10,11 @@ const isNode = require('detect-node') const path = require('path') const fs = require('fs') const os = require('os') +const http = require('http') +const https = require('https') +const each = require('async/each') +const waterfall = require('async/waterfall') +const parallel = require('async/parallel') const IPFSApi = require('../src') const f = require('./utils/factory') @@ -32,7 +37,8 @@ describe('.util', () => { }) }) - after((done) => { + after(function (done) { + this.timeout(10 * 1000) if (!ipfsd) return done() ipfsd.stop(done) }) @@ -113,81 +119,243 @@ describe('.util', () => { }) describe('.urlAdd', () => { - it('http', function (done) { - this.timeout(40 * 1000) + let testServers = [] + + const sslOpts = { + key: fs.readFileSync(path.join(__dirname, 'fixtures', 'ssl', 'privkey.pem')), + cert: fs.readFileSync(path.join(__dirname, 'fixtures', 'ssl', 'cert.pem')) + } + + const startTestServer = (handler, opts, cb) => { + if (typeof opts === 'function') { + cb = opts + opts = {} + } + + const agent = opts.secure ? https : http + const serverOpts = opts.secure ? sslOpts : {} + const server = agent.createServer(serverOpts, handler) + + server.listen((err) => { + if (err) return cb(err) + testServers.push(server) + cb(null, server) + }) + } - ipfs.util.addFromURL('http://example.com/', (err, result) => { - expect(err).to.not.exist() - expect(result.length).to.equal(1) - done() + beforeEach(() => { + // Instructs node to not reject our snake oil SSL certificate when it + // can't verify the cerificate authority + process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0 + }) + + afterEach((done) => { + // Reinstate unauthorised SSL cert rejection + process.env.NODE_TLS_REJECT_UNAUTHORIZED = 1 + + each(testServers, (server, cb) => server.close(cb), (err) => { + testServers = [] + done(err) }) }) - it('https', function (done) { - this.timeout(40 * 1000) + it('http', (done) => { + const handler = (req, res) => { + res.write(`TEST${Date.now()}`) + res.end() + } - ipfs.util.addFromURL('https://example.com/', (err, result) => { + startTestServer(handler, (err, server) => { expect(err).to.not.exist() - expect(result.length).to.equal(1) - done() + + const url = `http://127.0.0.1:${server.address().port}/` + ipfs.util.addFromURL(url, (err, result) => { + expect(err).to.not.exist() + expect(result.length).to.equal(1) + done() + }) }) }) - it('http with redirection', function (done) { - this.timeout(40 * 1000) + it('https', (done) => { + const handler = (req, res) => { + res.write(`TEST${Date.now()}`) + res.end() + } - ipfs.util.addFromURL('http://covers.openlibrary.org/book/id/969165.jpg', (err, result) => { + startTestServer(handler, { secure: true }, (err, server) => { expect(err).to.not.exist() - expect(result[0].hash).to.equal('QmaL9zy7YUfvWmtD5ZXp42buP7P4xmZJWFkm78p8FJqgjg') - done() + + const url = `https://127.0.0.1:${server.address().port}/` + ipfs.util.addFromURL(url, (err, result) => { + expect(err).to.not.exist() + expect(result.length).to.equal(1) + done() + }) }) }) - it('https with redirection', function (done) { - this.timeout(40 * 1000) - - ipfs.util.addFromURL('https://coverartarchive.org/release/6e2a1694-d8b9-466a-aa33-b1077b2333c1', (err, result) => { + it('http with redirection', (done) => { + const data = Buffer.from(`TEST${Date.now()}`) + + waterfall([ + (cb) => { + const handler = (req, res) => { + res.write(data) + res.end() + } + startTestServer(handler, cb) + }, + (serverA, cb) => { + const url = `http://127.0.0.1:${serverA.address().port}` + const handler = (req, res) => { + res.statusCode = 302 + res.setHeader('Location', url) + res.end() + } + startTestServer(handler, (err, serverB) => { + if (err) return cb(err) + cb(null, { a: serverA, b: serverB }) + }) + } + ], (err, servers) => { expect(err).to.not.exist() - expect(result[0].hash).to.equal('QmSUdDvmXuq5YGrL4M3SEz7UZh5eT9WMuAsd9K34sambSj') - done() + + ipfs.add(data, (err, res) => { + expect(err).to.not.exist() + + const expectedHash = res[0].hash + const url = `http://127.0.0.1:${servers.b.address().port}` + + ipfs.util.addFromURL(url, (err, result) => { + expect(err).to.not.exist() + expect(result[0].hash).to.equal(expectedHash) + done() + }) + }) }) }) - it('with only-hash=true', function () { - this.timeout(40 * 1000) + it('https with redirection', (done) => { + const data = Buffer.from(`TEST${Date.now()}`) + + waterfall([ + (cb) => { + const handler = (req, res) => { + res.write(data) + res.end() + } + startTestServer(handler, { secure: true }, cb) + }, + (serverA, cb) => { + const url = `https://127.0.0.1:${serverA.address().port}` + const handler = (req, res) => { + res.statusCode = 302 + res.setHeader('Location', url) + res.end() + } + startTestServer(handler, { secure: true }, (err, serverB) => { + if (err) return cb(err) + cb(null, { a: serverA, b: serverB }) + }) + } + ], (err, servers) => { + expect(err).to.not.exist() - return ipfs.util.addFromURL('http://www.randomtext.me/#/gibberish', { onlyHash: true }) - .then(out => expectTimeout(ipfs.object.get(out[0].hash), 4000)) + ipfs.add(data, (err, res) => { + expect(err).to.not.exist() + + const expectedHash = res[0].hash + const url = `https://127.0.0.1:${servers.b.address().port}` + + ipfs.util.addFromURL(url, (err, result) => { + expect(err).to.not.exist() + expect(result[0].hash).to.equal(expectedHash) + done() + }) + }) + }) }) - it('with wrap-with-directory=true', function (done) { - this.timeout(40 * 1000) + it('with only-hash=true', (done) => { + const handler = (req, res) => { + res.write(`TEST${Date.now()}`) + res.end() + } - ipfs.util.addFromURL('http://ipfs.io/ipfs/QmWjppACLcFLQ2qL38unKQvJBhXH3RUtcGLPk7zmrTwV61/969165.jpg?foo=bar#buzz', { - wrapWithDirectory: true - }, (err, result) => { + startTestServer(handler, (err, server) => { expect(err).to.not.exist() - expect(result[0].hash).to.equal('QmaL9zy7YUfvWmtD5ZXp42buP7P4xmZJWFkm78p8FJqgjg') - expect(result[0].path).to.equal('969165.jpg') - expect(result[1].hash).to.equal('QmWjppACLcFLQ2qL38unKQvJBhXH3RUtcGLPk7zmrTwV61') - expect(result.length).to.equal(2) - done() + + const url = `http://127.0.0.1:${server.address().port}/` + + ipfs.util.addFromURL(url, { onlyHash: true }, (err, res) => { + expect(err).to.not.exist() + + // A successful object.get for this size data took my laptop ~14ms + let didTimeout = false + const timeoutId = setTimeout(() => { + didTimeout = true + done() + }, 500) + + ipfs.object.get(res[0].hash, () => { + clearTimeout(timeoutId) + if (didTimeout) return + expect(new Error('did not timeout')).to.not.exist() + }) + }) }) }) - it('with wrap-with-directory=true and URL-escaped file name', function (done) { - this.timeout(40 * 1000) + it('with wrap-with-directory=true', (done) => { + const filename = `TEST${Date.now()}.txt` + const data = Buffer.from(`TEST${Date.now()}`) + + parallel({ + server: (cb) => startTestServer((req, res) => { + res.write(data) + res.end() + }, cb), + expectedResult: (cb) => { + ipfs.add([{ path: filename, content: data }], { wrapWithDirectory: true }, cb) + } + }, (err, taskResult) => { + expect(err).to.not.exist() + + const { server, expectedResult } = taskResult + const url = `http://127.0.0.1:${server.address().port}/${filename}?foo=bar#buzz` + + ipfs.util.addFromURL(url, { wrapWithDirectory: true }, (err, result) => { + expect(err).to.not.exist() + expect(result).to.deep.equal(expectedResult) + done() + }) + }) + }) - // Sample URL contains URL-escaped ( ) and local diacritics - ipfs.util.addFromURL('https://upload.wikimedia.org/wikipedia/commons/thumb/c/cf/Doma%C5%BElice%2C_Jir%C3%A1skova_43_%289102%29.jpg/320px-Doma%C5%BElice%2C_Jir%C3%A1skova_43_%289102%29.jpg?foo=bar#buzz', { - wrapWithDirectory: true - }, (err, result) => { + it('with wrap-with-directory=true and URL-escaped file name', (done) => { + const filename = '320px-Domažlice,_Jiráskova_43_(9102).jpg' + const data = Buffer.from(`TEST${Date.now()}`) + + parallel({ + server: (cb) => startTestServer((req, res) => { + res.write(data) + res.end() + }, cb), + expectedResult: (cb) => { + ipfs.add([{ path: filename, content: data }], { wrapWithDirectory: true }, cb) + } + }, (err, taskResult) => { expect(err).to.not.exist() - expect(result[0].hash).to.equal('QmRJ9ExxSMV4BLF9ZJUb2mLngupm6BXZEek755VHGTJo2Y') - expect(result[0].path).to.equal('320px-Domažlice,_Jiráskova_43_(9102).jpg') - expect(result[1].hash).to.equal('QmbxsHFU3sCfr8wszDHuDLA76C2xCv9HT8L3aC1pBwgaHk') - expect(result.length).to.equal(2) - done() + + const { server, expectedResult } = taskResult + const url = `http://127.0.0.1:${server.address().port}/${encodeURIComponent(filename)}?foo=bar#buzz` + + ipfs.util.addFromURL(url, { wrapWithDirectory: true }, (err, result) => { + expect(err).to.not.exist() + expect(result).to.deep.equal(expectedResult) + done() + }) }) }) From 3f19bceea3a4b6b2487c61e08991dd1c26a1867b Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 14 Aug 2018 22:31:02 +0100 Subject: [PATCH 2/4] fix: assert the response received is the same as direct add License: MIT Signed-off-by: Alan Shaw --- test/util.spec.js | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/test/util.spec.js b/test/util.spec.js index 213e19ba1..6e8b134ab 100644 --- a/test/util.spec.js +++ b/test/util.spec.js @@ -20,7 +20,7 @@ const IPFSApi = require('../src') const f = require('./utils/factory') const expectTimeout = require('./utils/expect-timeout') -describe('.util', () => { +describe.only('.util', () => { if (!isNode) { return } let ipfsd @@ -145,7 +145,7 @@ describe('.util', () => { beforeEach(() => { // Instructs node to not reject our snake oil SSL certificate when it - // can't verify the cerificate authority + // can't verify the certificate authority process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0 }) @@ -160,36 +160,50 @@ describe('.util', () => { }) it('http', (done) => { - const handler = (req, res) => { - res.write(`TEST${Date.now()}`) - res.end() - } + const data = Buffer.from(`TEST${Date.now()}`) - startTestServer(handler, (err, server) => { + parallel({ + server: (cb) => { + const handler = (req, res) => { + res.write(data) + res.end() + } + startTestServer(handler, cb) + }, + expectedResult: (cb) => ipfs.add(data, cb) + }, (err, taskResult) => { expect(err).to.not.exist() + const { server, expectedResult } = taskResult const url = `http://127.0.0.1:${server.address().port}/` ipfs.util.addFromURL(url, (err, result) => { expect(err).to.not.exist() - expect(result.length).to.equal(1) + expect(result).to.deep.equal(expectedResult) done() }) }) }) it('https', (done) => { - const handler = (req, res) => { - res.write(`TEST${Date.now()}`) - res.end() - } + const data = Buffer.from(`TEST${Date.now()}`) - startTestServer(handler, { secure: true }, (err, server) => { + parallel({ + server: (cb) => { + const handler = (req, res) => { + res.write(data) + res.end() + } + startTestServer(handler, { secure: true }, cb) + }, + expectedResult: (cb) => ipfs.add(data, cb) + }, (err, taskResult) => { expect(err).to.not.exist() + const { server, expectedResult } = taskResult const url = `https://127.0.0.1:${server.address().port}/` ipfs.util.addFromURL(url, (err, result) => { expect(err).to.not.exist() - expect(result.length).to.equal(1) + expect(result).to.deep.equal(expectedResult) done() }) }) From 0301d02bfe3ebb71a3381838c7a6f9551088ee7d Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 14 Aug 2018 22:39:00 +0100 Subject: [PATCH 3/4] fix: no options for http createServer in Node.js 8 License: MIT Signed-off-by: Alan Shaw --- test/util.spec.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/util.spec.js b/test/util.spec.js index 6e8b134ab..f2e2a6c8f 100644 --- a/test/util.spec.js +++ b/test/util.spec.js @@ -20,7 +20,7 @@ const IPFSApi = require('../src') const f = require('./utils/factory') const expectTimeout = require('./utils/expect-timeout') -describe.only('.util', () => { +describe('.util', () => { if (!isNode) { return } let ipfsd @@ -118,7 +118,7 @@ describe.only('.util', () => { }) }) - describe('.urlAdd', () => { + describe.only('.urlAdd', () => { let testServers = [] const sslOpts = { @@ -132,9 +132,9 @@ describe.only('.util', () => { opts = {} } - const agent = opts.secure ? https : http - const serverOpts = opts.secure ? sslOpts : {} - const server = agent.createServer(serverOpts, handler) + const server = opts.secure + ? https.createServer(sslOpts, handler) + : http.createServer(handler) server.listen((err) => { if (err) return cb(err) From eed55e347e3cf418d62c80a670723bb00b59ac4c Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 14 Aug 2018 22:42:33 +0100 Subject: [PATCH 4/4] fix: remove .only License: MIT Signed-off-by: Alan Shaw --- test/util.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/util.spec.js b/test/util.spec.js index f2e2a6c8f..6109e5cc9 100644 --- a/test/util.spec.js +++ b/test/util.spec.js @@ -118,7 +118,7 @@ describe('.util', () => { }) }) - describe.only('.urlAdd', () => { + describe('.urlAdd', () => { let testServers = [] const sslOpts = {