diff --git a/lib/api/apiUtils/integrity/validateChecksums.js b/lib/api/apiUtils/integrity/validateChecksums.js index 2e0aec3d84..64f945b2ae 100644 --- a/lib/api/apiUtils/integrity/validateChecksums.js +++ b/lib/api/apiUtils/integrity/validateChecksums.js @@ -5,6 +5,9 @@ const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt'); const { errors: ArsenalErrors, errorInstances } = require('arsenal'); const { config } = require('../../../Config'); +const defaultChecksumData = Object.freeze( + { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }); + const errAlgoNotSupported = errorInstances.InvalidRequest.customizeDescription( 'The algorithm type you specified in x-amz-checksum- header is invalid.'); const errAlgoNotSupportedSDK = errorInstances.InvalidRequest.customizeDescription( @@ -260,9 +263,8 @@ function getChecksumDataFromHeaders(headers) { } if (!checksumHeader) { - // There was no x-amz-checksum- or x-amz-trailer return crc64nvme. - // The calculated crc64nvme will be stored in the object metadata. - return { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }; + // No x-amz-checksum- or x-amz-trailer header. + return null; } // No x-amz-sdk-checksum-algorithm we expect one x-amz-checksum-[crc64nvme, crc32, crc32C, sha1, sha256]. @@ -476,6 +478,7 @@ function getChecksumDataFromMPUHeaders(headers) { module.exports = { ChecksumError, + defaultChecksumData, validateChecksumsNoChunking, validateMethodChecksumNoChunking, getChecksumDataFromHeaders, diff --git a/lib/api/apiUtils/object/createAndStoreObject.js b/lib/api/apiUtils/object/createAndStoreObject.js index 5523e4290c..48d1ec5f6f 100644 --- a/lib/api/apiUtils/object/createAndStoreObject.js +++ b/lib/api/apiUtils/object/createAndStoreObject.js @@ -16,6 +16,7 @@ const validateWebsiteHeader = require('./websiteServing') const applyZenkoUserMD = require('./applyZenkoUserMD'); const { algorithms, + defaultChecksumData, getChecksumDataFromHeaders, arsenalErrorFromChecksumError, } = require('../integrity/validateChecksums'); @@ -37,7 +38,7 @@ const externalVersioningErrorMessage = 'We do not currently support putting ' + * @return {undefined} */ function zeroSizeBodyChecksumCheck(headers, metadataStoreParams, callback) { - const checksumData = getChecksumDataFromHeaders(headers); + const checksumData = getChecksumDataFromHeaders(headers) || defaultChecksumData; if (checksumData.error) { return callback(arsenalErrorFromChecksumError(checksumData)); } @@ -291,8 +292,16 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, } } + const headerChecksum = getChecksumDataFromHeaders(request.headers); + if (headerChecksum && headerChecksum.error) { + return next(arsenalErrorFromChecksumError(headerChecksum)); + } + const checksums = { + primary: headerChecksum || defaultChecksumData, + secondary: null, + }; return dataStore(objectKeyContext, cipherBundle, request, size, - streamingV4Params, backendInfo, log, next); + streamingV4Params, backendInfo, checksums, log, next); }, function processDataResult(dataGetInfo, calculatedHash, checksum, next) { if (dataGetInfo === null || dataGetInfo === undefined) { @@ -320,7 +329,11 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, dataGetInfoArr[0].size = mdOnlySize; } metadataStoreParams.contentMD5 = calculatedHash; - metadataStoreParams.checksum = checksum; + if (checksum) { + // eslint-disable-next-line no-param-reassign + checksum.type = 'FULL_OBJECT'; + metadataStoreParams.checksum = checksum; + } return next(null, dataGetInfoArr); }, function getVersioningInfo(infoArr, next) { diff --git a/lib/api/apiUtils/object/prepareStream.js b/lib/api/apiUtils/object/prepareStream.js index 855b7a8ecd..c267936b83 100644 --- a/lib/api/apiUtils/object/prepareStream.js +++ b/lib/api/apiUtils/object/prepareStream.js @@ -1,37 +1,33 @@ const V4Transform = require('../../../auth/streamingV4/V4Transform'); const TrailingChecksumTransform = require('../../../auth/streamingV4/trailingChecksumTransform'); const ChecksumTransform = require('../../../auth/streamingV4/ChecksumTransform'); -const { - getChecksumDataFromHeaders, - arsenalErrorFromChecksumError, -} = require('../../apiUtils/integrity/validateChecksums'); const { errors, errorInstances, jsutil } = require('arsenal'); const { unsupportedSignatureChecksums } = require('../../../../constants'); /** * Prepares the request stream for data storage by wrapping it in the * appropriate transform pipeline based on the x-amz-content-sha256 header. - * Always returns a ChecksumTransform as the final stream. - * If no checksum was sent by the client a CRC64NVME ChecksumTransform is returned. + * The returned stream is always the primary ChecksumTransform (the stored + * checksum). When a secondary checksum is requested it is inserted upstream + * of the primary and exposed via secondaryChecksumStream. * * @param {object} request - incoming HTTP request with headers and body stream * @param {object|null} streamingV4Params - v4 streaming auth params (accessKey, * signatureFromRequest, region, scopeDate, timestamp, credentialScope), or * null/undefined for non-v4 requests + * @param {object} checksums - checksum configuration + * @param {object} checksums.primary - primary checksum + * ({ algorithm, isTrailer, expected }) — validated and its digest returned + * @param {object|null} checksums.secondary - optional secondary checksum + * ({ algorithm, isTrailer, expected }) — only validated; used for MPU parts * @param {RequestLogger} log - request logger * @param {function} errCb - error callback invoked if a stream error occurs - * @return {{ error: Arsenal.Error|null, stream: ChecksumTransform|null }} - * error is set and stream is null if the request headers are invalid; - * otherwise error is null and stream is the ChecksumTransform to read from + * @return {{ error: Arsenal.Error|null, stream: ChecksumTransform|null, + * secondaryChecksumStream: ChecksumTransform|null }} */ -function prepareStream(request, streamingV4Params, log, errCb) { +function prepareStream(request, streamingV4Params, checksums, log, errCb) { const xAmzContentSHA256 = request.headers['x-amz-content-sha256']; - - const checksumAlgo = getChecksumDataFromHeaders(request.headers); - if (checksumAlgo.error) { - log.debug('prepareStream invalid checksum headers', checksumAlgo); - return { error: arsenalErrorFromChecksumError(checksumAlgo), stream: null }; - } + const { primary, secondary } = checksums; switch (xAmzContentSHA256) { case 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD': { @@ -54,37 +50,49 @@ function prepareStream(request, streamingV4Params, log, errCb) { request.pipe(v4Transform); v4Transform.headers = request.headers; - const checksumedStream = new ChecksumTransform( - checksumAlgo.algorithm, - checksumAlgo.expected, - checksumAlgo.isTrailer, - log, - ); - checksumedStream.on('error', onStreamError); - v4Transform.pipe(checksumedStream); - return { error: null, stream: checksumedStream }; + let secondaryChecksumStream = null; + let stream = v4Transform; + if (secondary) { + secondaryChecksumStream = new ChecksumTransform( + secondary.algorithm, secondary.expected, + secondary.isTrailer, log); + secondaryChecksumStream.on('error', onStreamError); + stream = v4Transform.pipe(secondaryChecksumStream); + } + + const primaryStream = new ChecksumTransform( + primary.algorithm, primary.expected, primary.isTrailer, log); + primaryStream.on('error', onStreamError); + return { error: null, stream: stream.pipe(primaryStream), secondaryChecksumStream }; } case 'STREAMING-UNSIGNED-PAYLOAD-TRAILER': { - // Use a once-guard so that auto-destroying both piped streams - // when one errors does not result in errCb being called twice. const onStreamError = jsutil.once(errCb); const trailingChecksumTransform = new TrailingChecksumTransform(log); trailingChecksumTransform.on('error', onStreamError); request.pipe(trailingChecksumTransform); trailingChecksumTransform.headers = request.headers; - const checksumedStream = new ChecksumTransform( - checksumAlgo.algorithm, - checksumAlgo.expected, - checksumAlgo.isTrailer, - log, - ); - checksumedStream.on('error', onStreamError); - trailingChecksumTransform.on('trailer', (name, value) => { - checksumedStream.setExpectedChecksum(name, value); - }); - trailingChecksumTransform.pipe(checksumedStream); - return { error: null, stream: checksumedStream }; + let secondaryChecksumStream = null; + let stream = trailingChecksumTransform; + if (secondary) { + secondaryChecksumStream = new ChecksumTransform( + secondary.algorithm, secondary.expected, + secondary.isTrailer, log); + secondaryChecksumStream.on('error', onStreamError); + stream = trailingChecksumTransform.pipe(secondaryChecksumStream); + trailingChecksumTransform.on('trailer', (name, value) => { + secondaryChecksumStream.setExpectedChecksum(name, value); + }); + } + + const primaryStream = new ChecksumTransform(primary.algorithm, primary.expected, primary.isTrailer, log); + primaryStream.on('error', onStreamError); + if (!secondary) { + trailingChecksumTransform.on('trailer', (name, value) => { + primaryStream.setExpectedChecksum(name, value); + }); + } + return { error: null, stream: stream.pipe(primaryStream), secondaryChecksumStream }; } case 'UNSIGNED-PAYLOAD': // Fallthrough default: { @@ -95,15 +103,20 @@ function prepareStream(request, streamingV4Params, log, errCb) { }; } - const checksumedStream = new ChecksumTransform( - checksumAlgo.algorithm, - checksumAlgo.expected, - checksumAlgo.isTrailer, - log, - ); - checksumedStream.on('error', errCb); - request.pipe(checksumedStream); - return { error: null, stream: checksumedStream }; + const onStreamError = secondary ? jsutil.once(errCb) : errCb; + let secondaryChecksumStream = null; + let stream = request; + if (secondary) { + secondaryChecksumStream = new ChecksumTransform( + secondary.algorithm, secondary.expected, + secondary.isTrailer, log); + secondaryChecksumStream.on('error', onStreamError); + stream = request.pipe(secondaryChecksumStream); + } + + const primaryStream = new ChecksumTransform(primary.algorithm, primary.expected, primary.isTrailer, log); + primaryStream.on('error', onStreamError); + return { error: null, stream: stream.pipe(primaryStream), secondaryChecksumStream }; } } } diff --git a/lib/api/apiUtils/object/storeObject.js b/lib/api/apiUtils/object/storeObject.js index 3ea53e3891..fe4baf7300 100644 --- a/lib/api/apiUtils/object/storeObject.js +++ b/lib/api/apiUtils/object/storeObject.js @@ -38,9 +38,7 @@ function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, checksumStre return cb(errors.BadDigest); }); } - const checksum = checksumStream.digest - ? { algorithm: checksumStream.algoName, value: checksumStream.digest, type: 'FULL_OBJECT' } - : null; + const checksum = { algorithm: checksumStream.algoName, value: checksumStream.digest }; return cb(null, dataRetrievalInfo, completedHash, checksum); } @@ -56,12 +54,14 @@ function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, checksumStre * credentialScope (to be used for streaming v4 auth if applicable) * @param {BackendInfo} backendInfo - info to determine which data * backend to use + * @param {object} checksums - checksum configuration + * @param {object} checksums.primary - primary checksum data + * @param {object|null} checksums.secondary - secondary checksum data * @param {RequestLogger} log - the current stream logger * @param {function} cb - callback containing result for the next task * @return {undefined} */ -function dataStore(objectContext, cipherBundle, stream, size, - streamingV4Params, backendInfo, log, cb) { +function dataStore(objectContext, cipherBundle, stream, size, streamingV4Params, backendInfo, checksums, log, cb) { const cbOnce = jsutil.once(cb); // errCb is delegated through a mutable reference so it can be upgraded to @@ -69,7 +69,7 @@ function dataStore(objectContext, cipherBundle, stream, size, let onStreamError = cbOnce; const errCb = err => onStreamError(err); - const checksumedStream = prepareStream(stream, streamingV4Params, log, errCb); + const checksumedStream = prepareStream(stream, streamingV4Params, checksums, log, errCb); if (checksumedStream.error) { log.debug('dataStore failed to prepare stream', checksumedStream); return process.nextTick(() => cbOnce(checksumedStream.error)); @@ -99,27 +99,60 @@ function dataStore(objectContext, cipherBundle, stream, size, }); }; + // stream is always the primary (end of pipe, stored checksum). + // secondaryChecksumStream is upstream and only validated. + const { secondaryChecksumStream } = checksumedStream; + const doValidate = () => { - const checksumErr = checksumedStream.stream.validateChecksum(); - if (checksumErr) { - log.debug('failed checksum validation stream', { error: checksumErr }); + // Validate the secondary (checked-only) checksum first. + if (secondaryChecksumStream) { + const secondaryErr = secondaryChecksumStream.validateChecksum(); + if (secondaryErr) { + log.debug('failed secondary checksum validation', { error: secondaryErr }); + return data.batchDelete([dataRetrievalInfo], null, null, log, deleteErr => { + if (deleteErr) { + log.error('dataStore failed to delete old data', { error: deleteErr }); + } + return cbOnce(arsenalErrorFromChecksumError(secondaryErr)); + }); + } + } + // Validate the primary (stored) checksum. + const primaryErr = checksumedStream.stream.validateChecksum(); + if (primaryErr) { + log.debug('failed primary checksum validation', { error: primaryErr }); return data.batchDelete([dataRetrievalInfo], null, null, log, deleteErr => { if (deleteErr) { - // Failure of batch delete is only logged. log.error('dataStore failed to delete old data', { error: deleteErr }); } - return cbOnce(arsenalErrorFromChecksumError(checksumErr)); + return cbOnce(arsenalErrorFromChecksumError(primaryErr)); }); } + if (!secondaryChecksumStream) { + return checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, + checksumedStream.stream, log, cbOnce); + } + // Dual-checksum: checkHashMatchMD5 returns the primary + // (storage) checksum. Swap it to the client-facing one + // from the secondary stream and attach the primary as + // storageChecksum. return checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, - checksumedStream.stream, log, cbOnce); + checksumedStream.stream, log, (err, dataInfo, hash, primaryChecksum) => { + if (err) {return cbOnce(err);} + const checksum = { + algorithm: secondaryChecksumStream.algoName, // Used for the response headers. + value: secondaryChecksumStream.digest, + storageChecksum: primaryChecksum, + }; + return cbOnce(null, dataInfo, hash, checksum); + }); }; // ChecksumTransform._flush computes the digest asynchronously for // some algorithms (e.g. crc64nvme). writableFinished is true once // _flush has called its callback, guaranteeing this.digest is set. - // Stream piping ordering means this is virtually always true here, - // but we wait for 'finish' explicitly to be safe. + // stream is the primary (end of pipe) — when it finishes all + // upstream transforms (including the secondary) have flushed. if (checksumedStream.stream.writableFinished) { return doValidate(); } diff --git a/lib/api/objectPutPart.js b/lib/api/objectPutPart.js index 863221b09f..f115b8b727 100644 --- a/lib/api/objectPutPart.js +++ b/lib/api/objectPutPart.js @@ -21,6 +21,10 @@ const { BackendInfo } = models; const writeContinue = require('../utilities/writeContinue'); const { parseObjectEncryptionHeaders } = require('./apiUtils/bucket/bucketEncryption'); const validateChecksumHeaders = require('./apiUtils/object/validateChecksumHeaders'); +const { + getChecksumDataFromHeaders, + arsenalErrorFromChecksumError, +} = require('./apiUtils/integrity/validateChecksums'); const { validateQuotas } = require('./apiUtils/quotas/quotaUtils'); const { setSSEHeaders } = require('./apiUtils/object/sseHeaders'); const { storeServerAccessLogInfo } = require('../metadata/metadataUtils'); @@ -113,6 +117,8 @@ function objectPutPart(authInfo, request, streamingV4Params, log, // `requestType` is the general 'objectPut'. const requestType = request.apiMethods || 'objectPutPart'; let partChecksum; + let mpuChecksumAlgo; + let mpuChecksumIsDefault; return async.waterfall([ // Get the destination bucket. @@ -196,6 +202,9 @@ function objectPutPart(authInfo, request, streamingV4Params, log, return next(errors.AccessDenied, destinationBucket); } + mpuChecksumAlgo = res.checksumAlgorithm; + mpuChecksumIsDefault = res.checksumIsDefault; + const objectLocationConstraint = res.controllingLocationConstraint; const sseAlgo = res['x-amz-server-side-encryption']; @@ -316,8 +325,43 @@ function objectPutPart(authInfo, request, streamingV4Params, log, }; const backendInfo = new BackendInfo(config, objectLocationConstraint); + + const headerChecksum = getChecksumDataFromHeaders(request.headers); + if (headerChecksum && headerChecksum.error) { + return next(arsenalErrorFromChecksumError(headerChecksum), destinationBucket); + } + + // If the MPU specifies a non-default checksum algo and the + // client sends a different algo, reject the request. + if (headerChecksum && mpuChecksumAlgo && !mpuChecksumIsDefault + && headerChecksum.algorithm !== mpuChecksumAlgo) { + return next(errors.InvalidRequest.customizeDescription( + `Checksum algorithm '${headerChecksum.algorithm}' is not the same ` + + `as the checksum algorithm '${mpuChecksumAlgo}' specified during ` + + 'CreateMultipartUpload.' + ), destinationBucket); + } + + const primaryAlgo = mpuChecksumAlgo || 'crc64nvme'; + let checksums; + if (headerChecksum && headerChecksum.algorithm === mpuChecksumAlgo) { + checksums = { + primary: headerChecksum, // MPU and Header match only need to calculate one. + secondary: null, + }; + } else if (headerChecksum) { + checksums = { + primary: { algorithm: primaryAlgo, isTrailer: false, expected: undefined }, + secondary: headerChecksum, // MPU and Header mismatch, need to verify the header checksum. + }; + } else { + checksums = { + primary: { algorithm: primaryAlgo, isTrailer: false, expected: undefined }, + secondary: null, // No Header checksum, we only calculate the MPU one. + }; + } return dataStore(objectContext, cipherBundle, request, - size, streamingV4Params, backendInfo, log, + size, streamingV4Params, backendInfo, checksums, log, (err, dataGetInfo, hexDigest, checksum) => { if (err) { return next(err, destinationBucket); @@ -356,6 +400,15 @@ function objectPutPart(authInfo, request, streamingV4Params, log, 'content-length': size, 'owner-id': destinationBucket.getOwner(), }; + if (partChecksum) { + if (partChecksum.storageChecksum) { + omVal.checksumValue = partChecksum.storageChecksum.value; + omVal.checksumAlgorithm = partChecksum.storageChecksum.algorithm; + } else { + omVal.checksumValue = partChecksum.value; + omVal.checksumAlgorithm = partChecksum.algorithm; + } + } const mdParams = { overheadField: constants.overheadField }; return metadata.putObjectMD(mpuBucketName, partKey, omVal, mdParams, log, err => { diff --git a/lib/routes/routeBackbeat.js b/lib/routes/routeBackbeat.js index 718d19d9eb..72f319930c 100644 --- a/lib/routes/routeBackbeat.js +++ b/lib/routes/routeBackbeat.js @@ -31,6 +31,11 @@ const { standardMetadataValidateBucketAndObj, metadataGetObject } = require('../metadata/metadataUtils'); const { config } = require('../Config'); const constants = require('../../constants'); +const { + defaultChecksumData, + getChecksumDataFromHeaders, + arsenalErrorFromChecksumError, +} = require('../api/apiUtils/integrity/validateChecksums'); const { BackendInfo } = models; const { pushReplicationMetric } = require('./utilities/pushReplicationMetric'); const kms = require('../kms/wrapper'); @@ -445,14 +450,22 @@ function putData(request, response, bucketInfo, objMd, log, callback) { }); return callback(errors.InternalError); } + const headerChecksum = getChecksumDataFromHeaders(request.headers); + if (headerChecksum && headerChecksum.error) { + return callback(arsenalErrorFromChecksumError(headerChecksum)); + } + const checksums = { + primary: headerChecksum || defaultChecksumData, + secondary: null, + }; return dataStore( context, cipherBundle, request, payloadLen, {}, - backendInfo, log, + backendInfo, checksums, log, // The callback's 4th arg (checksum) is intentionally ignored: any - // x-amz-checksum-* header sent by Backbeat is already validated - // inside dataStore by ChecksumTransform. The computed value is not - // stored here because this is a data-only write — metadata is - // written separately by Backbeat, which should propagate the source + // x-amz-checksum-* header sent by Backbeat is validated inside + // dataStore by ChecksumTransform. The computed value is not stored + // here because this is a data-only write — metadata is written + // separately by Backbeat, which should propagate the source // object's checksum. (err, retrievalInfo, md5) => { if (err) { @@ -859,13 +872,22 @@ function putObject(request, response, log, callback) { } const payloadLen = parseInt(request.headers['content-length'], 10); const backendInfo = new BackendInfo(config, storageLocation); - return dataStore(context, CIPHER, request, payloadLen, {}, backendInfo, log, + const headerChecksum = getChecksumDataFromHeaders(request.headers); + if (headerChecksum && headerChecksum.error) { + return callback(arsenalErrorFromChecksumError(headerChecksum)); + } + const checksums = { + primary: headerChecksum || defaultChecksumData, + secondary: null, + }; + return dataStore(context, CIPHER, request, payloadLen, {}, backendInfo, + checksums, log, // The callback's 4th arg (checksum) is intentionally ignored: any - // x-amz-checksum-* header sent by Backbeat is already validated inside - // dataStore by ChecksumTransform. The computed value is not stored here - // because this is a data-only write to an external backend — metadata - // is managed separately by Backbeat, which should propagate the source - // object's checksum. + // x-amz-checksum-* header sent by Backbeat is validated inside + // dataStore by ChecksumTransform. The computed value is not stored + // here because this is a data-only write to an external backend — + // metadata is managed separately by Backbeat, which should propagate + // the source object's checksum. (err, retrievalInfo, md5) => { if (err) { log.error('error putting data', { diff --git a/tests/functional/aws-node-sdk/test/object/mpuUploadPartChecksum.js b/tests/functional/aws-node-sdk/test/object/mpuUploadPartChecksum.js new file mode 100644 index 0000000000..afd31aaa9e --- /dev/null +++ b/tests/functional/aws-node-sdk/test/object/mpuUploadPartChecksum.js @@ -0,0 +1,186 @@ +const assert = require('assert'); +const { + CreateBucketCommand, + CreateMultipartUploadCommand, + AbortMultipartUploadCommand, + UploadPartCommand, + DeleteBucketCommand, +} = require('@aws-sdk/client-s3'); + +const withV4 = require('../support/withV4'); +const BucketUtility = require('../../lib/utility/bucket-util'); +const { algorithms } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); + +const bucket = `mpu-part-checksum-test-${Date.now()}`; +const key = 'test-part-checksum-key'; +const partBody = Buffer.from('I am a part body for checksum testing', 'utf8'); + +const allAlgos = ['CRC32', 'CRC32C', 'SHA1', 'SHA256']; + +// Maps algo name to the UploadPartCommand checksum field name +const checksumField = { + CRC32: 'ChecksumCRC32', + CRC32C: 'ChecksumCRC32C', + SHA1: 'ChecksumSHA1', + SHA256: 'ChecksumSHA256', +}; + +// Pre-compute correct digests for partBody +const correctDigest = {}; +// A valid-length but incorrect digest for each algo +const wrongDigest = {}; + +before(async () => { + for (const algo of allAlgos) { + + correctDigest[algo] = await algorithms[algo.toLowerCase()].digest(partBody); + } + // Generate wrong digests: flip the first character + for (const algo of allAlgos) { + const correct = correctDigest[algo]; + const flipped = correct[0] === 'A' ? `B${correct.slice(1)}` : `A${correct.slice(1)}`; + wrongDigest[algo] = flipped; + } +}); + +describe('UploadPart checksum validation', () => + withV4(sigCfg => { + let bucketUtil; + let s3; + + before(async () => { + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + await s3.send(new CreateBucketCommand({ Bucket: bucket })); + }); + + after(async () => { + await bucketUtil.empty(bucket); + await s3.send(new DeleteBucketCommand({ Bucket: bucket })); + }); + + // For each non-default MPU algo, test that: + // - matching algo with correct digest succeeds + // - matching algo with wrong digest fails with BadDigest + // - every other algo is rejected with InvalidRequest + // - no checksum header is accepted + allAlgos.forEach(mpuAlgo => { + describe(`MPU created with ${mpuAlgo}`, () => { + let uploadId; + let partNum = 0; + + before(async () => { + const res = await s3.send(new CreateMultipartUploadCommand({ + Bucket: bucket, Key: key, + ChecksumAlgorithm: mpuAlgo, + })); + uploadId = res.UploadId; + }); + + after(async () => { + await s3.send(new AbortMultipartUploadCommand({ + Bucket: bucket, Key: key, UploadId: uploadId, + })); + }); + + it(`should accept ${mpuAlgo} with correct digest`, async () => { + partNum++; + const res = await s3.send(new UploadPartCommand({ + Bucket: bucket, Key: key, UploadId: uploadId, + PartNumber: partNum, Body: partBody, + [checksumField[mpuAlgo]]: correctDigest[mpuAlgo], + })); + assert.strictEqual(res[checksumField[mpuAlgo]], correctDigest[mpuAlgo]); + }); + + it(`should reject ${mpuAlgo} with wrong digest (BadDigest)`, async () => { + partNum++; + try { + await s3.send(new UploadPartCommand({ + Bucket: bucket, Key: key, UploadId: uploadId, + PartNumber: partNum, Body: partBody, + [checksumField[mpuAlgo]]: wrongDigest[mpuAlgo], + })); + assert.fail('Expected BadDigest error'); + } catch (err) { + assert.strictEqual(err.name, 'BadDigest'); + } + }); + + // Note: AWS SDK v3 always sends a default crc32 checksum, + // so "no checksum header" cannot be tested via the SDK for + // non-default MPUs (it would be rejected as a mismatch). + + allAlgos.filter(a => a !== mpuAlgo).forEach(otherAlgo => { + it(`should reject ${otherAlgo} when MPU is ${mpuAlgo} (InvalidRequest)`, async () => { + partNum++; + try { + await s3.send(new UploadPartCommand({ + Bucket: bucket, Key: key, UploadId: uploadId, + PartNumber: partNum, Body: partBody, + [checksumField[otherAlgo]]: correctDigest[otherAlgo], + })); + assert.fail('Expected InvalidRequest error'); + } catch (err) { + assert.strictEqual(err.name, 'InvalidRequest'); + } + }); + }); + }); + }); + + // Default MPU (no ChecksumAlgorithm) should accept any algo + describe('MPU created with no checksum (default)', () => { + let uploadId; + let partNum = 0; + + before(async () => { + const res = await s3.send(new CreateMultipartUploadCommand({ + Bucket: bucket, Key: key, + })); + uploadId = res.UploadId; + }); + + after(async () => { + await s3.send(new AbortMultipartUploadCommand({ + Bucket: bucket, Key: key, UploadId: uploadId, + })); + }); + + allAlgos.forEach(algo => { + it(`should accept ${algo} with correct digest`, async () => { + partNum++; + const res = await s3.send(new UploadPartCommand({ + Bucket: bucket, Key: key, UploadId: uploadId, + PartNumber: partNum, Body: partBody, + [checksumField[algo]]: correctDigest[algo], + })); + assert.strictEqual(res[checksumField[algo]], correctDigest[algo]); + }); + + it(`should reject ${algo} with wrong digest (BadDigest)`, async () => { + partNum++; + try { + await s3.send(new UploadPartCommand({ + Bucket: bucket, Key: key, UploadId: uploadId, + PartNumber: partNum, Body: partBody, + [checksumField[algo]]: wrongDigest[algo], + })); + assert.fail('Expected BadDigest error'); + } catch (err) { + assert.strictEqual(err.name, 'BadDigest'); + } + }); + }); + + it('should accept part with no checksum header', async () => { + partNum++; + const res = await s3.send(new UploadPartCommand({ + Bucket: bucket, Key: key, UploadId: uploadId, + PartNumber: partNum, Body: partBody, + })); + assert(res.ETag); + }); + }); + }) +); diff --git a/tests/unit/api/apiUtils/integrity/validateChecksums.js b/tests/unit/api/apiUtils/integrity/validateChecksums.js index fe2952e30f..cfd0253714 100644 --- a/tests/unit/api/apiUtils/integrity/validateChecksums.js +++ b/tests/unit/api/apiUtils/integrity/validateChecksums.js @@ -471,14 +471,14 @@ describe('getChecksumDataFromHeaders', () => { crc64nvme: 'AAAAAAAAAAA=', // 12 chars }; - it('should return crc64nvme with isTrailer=false and expected=undefined when no headers', () => { + it('should return null when no headers', () => { const result = getChecksumDataFromHeaders({}); - assert.deepStrictEqual(result, { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }); + assert.strictEqual(result, null); }); - it('should return crc64nvme default when no checksum headers, no trailer, no sdk algo', () => { + it('should return null when no checksum headers, no trailer, no sdk algo', () => { const result = getChecksumDataFromHeaders({ 'content-type': 'application/octet-stream' }); - assert.deepStrictEqual(result, { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }); + assert.strictEqual(result, null); }); for (const [algo, digest] of Object.entries(validDigests)) { diff --git a/tests/unit/api/apiUtils/object/prepareStream.js b/tests/unit/api/apiUtils/object/prepareStream.js index ff54da33b9..d0492615e5 100644 --- a/tests/unit/api/apiUtils/object/prepareStream.js +++ b/tests/unit/api/apiUtils/object/prepareStream.js @@ -5,13 +5,19 @@ const { prepareStream } = require('../../../../../lib/api/apiUtils/object/prepar const ChecksumTransform = require('../../../../../lib/auth/streamingV4/ChecksumTransform'); const { DummyRequestLogger } = require('../../../helpers'); const DummyRequest = require('../../../DummyRequest'); +const { defaultChecksumData } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); const log = new DummyRequestLogger(); +const defaultChecksums = { primary: defaultChecksumData, secondary: null }; function makeRequest(headers, body) { return new DummyRequest({ headers }, body != null ? Buffer.from(body) : undefined); } +function makeChecksums(algo, expected, isTrailer) { + return { primary: { algorithm: algo, expected, isTrailer: !!isTrailer }, secondary: null }; +} + const mockV4Params = { accessKey: 'AKIAIOSFODNN7EXAMPLE', signatureFromRequest: 'abc123', @@ -25,26 +31,16 @@ describe('prepareStream', () => { describe('return value shape', () => { it('should return { error: null, stream: ChecksumTransform } for UNSIGNED-PAYLOAD', () => { const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); - const result = prepareStream(request, null, log, () => {}); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); assert.strictEqual(result.error, null); assert(result.stream instanceof ChecksumTransform); }); - it('should return { error: InvalidRequest, stream: null } on invalid checksum headers', () => { - const request = makeRequest({ - 'x-amz-checksum-crc32': 'AAAAAA==', - 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', - }); - const result = prepareStream(request, null, log, () => {}); - assert.strictEqual(result.error.message, 'InvalidRequest'); - assert.strictEqual(result.stream, null); - }); - it('should return { error: BadRequest, stream: null } for unsupported x-amz-content-sha256', () => { const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER', }); - const result = prepareStream(request, null, log, () => {}); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); assert.strictEqual(result.error.message, 'BadRequest'); assert.strictEqual(result.stream, null); }); @@ -53,23 +49,23 @@ describe('prepareStream', () => { describe('STREAMING-AWS4-HMAC-SHA256-PAYLOAD', () => { it('should return ChecksumTransform as final stream with valid streamingV4Params', () => { const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); - const result = prepareStream(request, mockV4Params, log, () => {}); + const result = prepareStream(request, mockV4Params, defaultChecksums, log, () => {}); assert.strictEqual(result.error, null); assert(result.stream instanceof ChecksumTransform); }); - it('should use crc64nvme by default with valid streamingV4Params', () => { + it('should use crc64nvme when default checksums are passed', () => { const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); - const result = prepareStream(request, mockV4Params, log, () => {}); + const result = prepareStream(request, mockV4Params, defaultChecksums, log, () => {}); assert.strictEqual(result.stream.algoName, 'crc64nvme'); }); - it('should use crc32c algorithm when x-amz-checksum-crc32c header is present', () => { + it('should use crc32c algorithm when crc32c checksums are passed', () => { const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD', - 'x-amz-checksum-crc32c': 'AAAAAA==', }); - const result = prepareStream(request, mockV4Params, log, () => {}); + const checksums = makeChecksums('crc32c', 'AAAAAA=='); + const result = prepareStream(request, mockV4Params, checksums, log, () => {}); assert.strictEqual(result.error, null); assert.strictEqual(result.stream.algoName, 'crc32c'); assert.strictEqual(result.stream.expectedDigest, 'AAAAAA=='); @@ -77,14 +73,14 @@ describe('prepareStream', () => { it('should return InvalidArgument error with null streamingV4Params', () => { const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); - const result = prepareStream(request, null, log, () => {}); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); assert.deepStrictEqual(result.error, errors.InvalidArgument); assert.strictEqual(result.stream, null); }); it('should return InvalidArgument error with non-object streamingV4Params', () => { const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); - const result = prepareStream(request, 'not-an-object', log, () => {}); + const result = prepareStream(request, 'not-an-object', defaultChecksums, log, () => {}); assert.deepStrictEqual(result.error, errors.InvalidArgument); assert.strictEqual(result.stream, null); }); @@ -96,7 +92,8 @@ describe('prepareStream', () => { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', 'x-amz-trailer': 'x-amz-checksum-crc32', }); - const result = prepareStream(request, null, log, () => {}); + const checksums = makeChecksums('crc32', undefined, true); + const result = prepareStream(request, null, checksums, log, () => {}); assert.strictEqual(result.error, null); assert(result.stream instanceof ChecksumTransform); assert.strictEqual(result.stream.isTrailer, true); @@ -109,7 +106,8 @@ describe('prepareStream', () => { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', 'x-amz-trailer': 'x-amz-checksum-crc32', }, body); - const result = prepareStream(request, null, log, done); + const checksums = makeChecksums('crc32', undefined, true); + const result = prepareStream(request, null, checksums, log, done); result.stream.resume(); result.stream.on('finish', () => { assert.strictEqual(result.stream.trailerChecksumName, 'x-amz-checksum-crc32'); @@ -125,7 +123,8 @@ describe('prepareStream', () => { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', 'x-amz-trailer': 'x-amz-checksum-crc32', }, 'zz\r\n'); // invalid hex chunk size - prepareStream(request, null, log, err => { + const checksums = makeChecksums('crc32', undefined, true); + prepareStream(request, null, checksums, log, err => { assert.strictEqual(err.message, 'InvalidArgument'); done(); }); @@ -136,7 +135,8 @@ describe('prepareStream', () => { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', 'x-amz-trailer': 'x-amz-checksum-crc32', }); - const result = prepareStream(request, null, log, err => { + const checksums = makeChecksums('crc32', undefined, true); + const result = prepareStream(request, null, checksums, log, err => { assert.deepStrictEqual(err, errors.InternalError); done(); }); @@ -147,24 +147,22 @@ describe('prepareStream', () => { describe('UNSIGNED-PAYLOAD', () => { it('should return ChecksumTransform as final stream', () => { const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); - const result = prepareStream(request, null, log, () => {}); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); assert.strictEqual(result.error, null); assert(result.stream instanceof ChecksumTransform); }); - it('should set algorithm and expected digest from headers on ChecksumTransform', () => { - const request = makeRequest({ - 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - 'x-amz-checksum-crc32': 'AAAAAA==', - }); - const result = prepareStream(request, null, log, () => {}); + it('should set algorithm and expected digest from checksums on ChecksumTransform', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const checksums = makeChecksums('crc32', 'AAAAAA=='); + const result = prepareStream(request, null, checksums, log, () => {}); assert.strictEqual(result.stream.algoName, 'crc32'); assert.strictEqual(result.stream.expectedDigest, 'AAAAAA=='); }); it('should call errCb when ChecksumTransform emits an error', done => { const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); - const result = prepareStream(request, null, log, err => { + const result = prepareStream(request, null, defaultChecksums, log, err => { assert.deepStrictEqual(err, errors.InternalError); done(); }); @@ -172,10 +170,69 @@ describe('prepareStream', () => { }); }); + describe('secondary checksum (dual-checksum)', () => { + it('should return secondaryChecksumStream when secondary is provided for UNSIGNED-PAYLOAD', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const checksums = { + primary: { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }, + secondary: { algorithm: 'crc32', isTrailer: false, expected: 'DUoRhQ==' }, + }; + const result = prepareStream(request, null, checksums, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof ChecksumTransform); + assert.strictEqual(result.stream.algoName, 'crc64nvme'); + assert(result.secondaryChecksumStream instanceof ChecksumTransform); + assert.strictEqual(result.secondaryChecksumStream.algoName, 'crc32'); + }); + + it('should return null secondaryChecksumStream when secondary is null', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); + assert.strictEqual(result.secondaryChecksumStream, null); + }); + + it('should return secondaryChecksumStream for STREAMING-AWS4-HMAC-SHA256-PAYLOAD', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); + const checksums = { + primary: { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }, + secondary: { algorithm: 'sha256', isTrailer: false, expected: undefined }, + }; + const result = prepareStream(request, mockV4Params, checksums, log, () => {}); + assert.strictEqual(result.error, null); + assert.strictEqual(result.stream.algoName, 'crc64nvme'); + assert(result.secondaryChecksumStream instanceof ChecksumTransform); + assert.strictEqual(result.secondaryChecksumStream.algoName, 'sha256'); + }); + + it('should wire trailer to secondaryChecksumStream for STREAMING-UNSIGNED-PAYLOAD-TRAILER', done => { + const body = '0\r\nx-amz-checksum-sha256:test-value\r\n'; + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + }, body); + const checksums = { + primary: { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }, + secondary: { algorithm: 'sha256', isTrailer: true, expected: undefined }, + }; + const result = prepareStream(request, null, checksums, log, () => {}); + assert.strictEqual(result.stream.algoName, 'crc64nvme'); + assert.strictEqual(result.secondaryChecksumStream.algoName, 'sha256'); + result.stream.resume(); + result.stream.on('finish', () => { + assert.strictEqual(result.secondaryChecksumStream.trailerChecksumName, 'x-amz-checksum-sha256'); + assert.strictEqual(result.secondaryChecksumStream.trailerChecksumValue, 'test-value'); + // Primary should NOT have trailer set + assert.strictEqual(result.stream.trailerChecksumValue, undefined); + done(); + }); + result.stream.on('error', done); + }); + }); + describe('default (no x-amz-content-sha256)', () => { - it('should return ChecksumTransform with crc64nvme algorithm when no x-amz-content-sha256 header', () => { + it('should return ChecksumTransform with crc64nvme algorithm when default checksums passed', () => { const request = makeRequest({}); - const result = prepareStream(request, null, log, () => {}); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); assert.strictEqual(result.error, null); assert(result.stream instanceof ChecksumTransform); assert.strictEqual(result.stream.algoName, 'crc64nvme'); @@ -185,14 +242,14 @@ describe('prepareStream', () => { const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER', }); - const result = prepareStream(request, null, log, () => {}); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); assert.strictEqual(result.error.message, 'BadRequest'); assert.strictEqual(result.stream, null); }); it('should call errCb when ChecksumTransform emits an error', done => { const request = makeRequest({}); - const result = prepareStream(request, null, log, err => { + const result = prepareStream(request, null, defaultChecksums, log, err => { assert.deepStrictEqual(err, errors.InternalError); done(); }); diff --git a/tests/unit/api/apiUtils/object/storeObject.js b/tests/unit/api/apiUtils/object/storeObject.js index 69861d3bb8..0ddd158a3c 100644 --- a/tests/unit/api/apiUtils/object/storeObject.js +++ b/tests/unit/api/apiUtils/object/storeObject.js @@ -6,8 +6,10 @@ const { dataStore } = require('../../../../../lib/api/apiUtils/object/storeObjec const dataWrapper = require('../../../../../lib/data/wrapper'); const { DummyRequestLogger } = require('../../../helpers'); const DummyRequest = require('../../../DummyRequest'); +const { defaultChecksumData } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); const log = new DummyRequestLogger(); +const defaultChecksums = { primary: defaultChecksumData, secondary: null }; const fakeDataRetrievalInfo = { key: 'test-key', dataStoreName: 'mem' }; @@ -44,7 +46,7 @@ describe('dataStore', () => { it('should call data.put with the stream returned by prepareStream', done => { putSucceeds(); const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); - dataStore({}, null, request, 0, null, {}, log, err => { + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { assert.strictEqual(err, null); assert(putStub.calledOnce); done(); @@ -54,7 +56,7 @@ describe('dataStore', () => { it('should call cb with (null, dataRetrievalInfo, completedHash) on success', done => { putSucceeds('abc123'); const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); - dataStore({}, null, request, 0, null, {}, log, (err, dataInfo, completedHash) => { + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, (err, dataInfo, completedHash) => { assert.strictEqual(err, null); assert.strictEqual(dataInfo, fakeDataRetrievalInfo); assert.strictEqual(completedHash, 'abc123'); @@ -68,7 +70,7 @@ describe('dataStore', () => { cb(errors.InternalError); }); const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); - dataStore({}, null, request, 0, null, {}, log, err => { + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { assert.deepStrictEqual(err, errors.InternalError); done(); }); @@ -80,7 +82,7 @@ describe('dataStore', () => { cb(errors.InternalError); }); const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); - dataStore({}, null, request, 0, null, {}, log, () => { + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, () => { assert(batchDeleteStub.notCalled); done(); }); @@ -92,7 +94,7 @@ describe('dataStore', () => { cb(null, null, null); }); const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); - dataStore({}, null, request, 0, null, {}, log, err => { + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { assert.deepStrictEqual(err, errors.InternalError); done(); }); @@ -103,7 +105,7 @@ describe('dataStore', () => { putSucceeds('correct-md5'); const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); request.contentMD5 = 'wrong-md5'; - dataStore({}, null, request, 0, null, {}, log, err => { + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { assert.deepStrictEqual(err, errors.BadDigest); assert(batchDeleteStub.calledOnce); done(); @@ -114,7 +116,7 @@ describe('dataStore', () => { putSucceeds('abc123'); const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); request.contentMD5 = 'abc123'; - dataStore({}, null, request, 0, null, {}, log, err => { + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { assert.strictEqual(err, null); assert(batchDeleteStub.notCalled); done(); @@ -125,7 +127,7 @@ describe('dataStore', () => { putSucceeds(); let cbCount = 0; const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); - dataStore({}, null, request, 0, null, {}, log, () => { + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, () => { cbCount++; setImmediate(() => { assert.strictEqual(cbCount, 1); @@ -136,23 +138,21 @@ describe('dataStore', () => { }); describe('checksum behaviour', () => { - it('should call cb with error from prepareStream when stream headers are invalid', done => { + it('should call cb with error from prepareStream when signature checksum is unsupported', done => { const request = makeStream({ - 'x-amz-checksum-crc32': 'AAAAAA==', - 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER', }); - dataStore({}, null, request, 0, null, {}, log, err => { - assert.strictEqual(err.message, 'InvalidRequest'); + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { + assert.strictEqual(err.message, 'BadRequest'); done(); }); }); it('should not call data.put when prepareStream returns an error', done => { const request = makeStream({ - 'x-amz-checksum-crc32': 'AAAAAA==', - 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER', }); - dataStore({}, null, request, 0, null, {}, log, () => { + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, () => { assert(putStub.notCalled); done(); }); @@ -164,9 +164,12 @@ describe('dataStore', () => { // CRC32 of 'hello world' is not 0x00000000 (AAAAAA==) const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - 'x-amz-checksum-crc32': 'AAAAAA==', }, 'hello world'); - dataStore({}, null, request, 0, null, {}, log, err => { + const badChecksums = { + primary: { algorithm: 'crc32', isTrailer: false, expected: 'AAAAAA==' }, + secondary: null, + }; + dataStore({}, null, request, 0, null, {}, badChecksums, log, err => { assert.strictEqual(err.message, 'BadDigest'); assert(batchDeleteStub.calledOnce); done(); @@ -177,9 +180,12 @@ describe('dataStore', () => { putSucceeds(); const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - 'x-amz-checksum-crc32': 'DUoRhQ==', }, 'hello world'); - dataStore({}, null, request, 0, null, {}, log, err => { + const goodChecksums = { + primary: { algorithm: 'crc32', isTrailer: false, expected: 'DUoRhQ==' }, + secondary: null, + }; + dataStore({}, null, request, 0, null, {}, goodChecksums, log, err => { assert.strictEqual(err, null); assert(batchDeleteStub.notCalled); done(); @@ -198,9 +204,12 @@ describe('dataStore', () => { }); const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - 'x-amz-checksum-crc32': 'DUoRhQ==', }, 'hello world'); - dataStore({}, null, request, 0, null, {}, log, err => { + const goodChecksums = { + primary: { algorithm: 'crc32', isTrailer: false, expected: 'DUoRhQ==' }, + secondary: null, + }; + dataStore({}, null, request, 0, null, {}, goodChecksums, log, err => { assert.strictEqual(err, null); assert(capturedStream.writableFinished); done(); @@ -217,7 +226,7 @@ describe('dataStore', () => { cb(null, fakeDataRetrievalInfo, { completedHash: null }); }); const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); - dataStore({}, null, request, 0, null, {}, log, err => { + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { assert.deepStrictEqual(err, errors.InternalError); assert(batchDeleteStub.calledOnce); done(); @@ -235,7 +244,7 @@ describe('dataStore', () => { cb(null, fakeDataRetrievalInfo, { completedHash: null }); }); const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); - dataStore({}, null, request, 0, null, {}, log, () => { + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, () => { cbCount++; setImmediate(() => { assert.strictEqual(cbCount, 1); @@ -253,7 +262,7 @@ describe('dataStore', () => { cb(null, fakeDataRetrievalInfo, { completedHash: null }); }); const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); - dataStore({}, null, request, 0, null, {}, log, () => { + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, () => { cbCount++; setImmediate(() => { assert.strictEqual(cbCount, 1); @@ -264,15 +273,76 @@ describe('dataStore', () => { }); }); + describe('dual-checksum behaviour', () => { + it('should return client-facing checksum from secondary and storageChecksum from primary', done => { + putSucceeds(); + const request = makeStream({ + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + }, 'hello world'); + // Primary: crc64nvme (storage), Secondary: crc32 (client-facing) + const dualChecksums = { + primary: { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }, + secondary: { algorithm: 'crc32', isTrailer: false, expected: 'DUoRhQ==' }, + }; + dataStore({}, null, request, 0, null, {}, dualChecksums, log, (err, dataInfo, hash, checksum) => { + assert.strictEqual(err, null); + assert.strictEqual(checksum.algorithm, 'crc32'); + assert.strictEqual(checksum.value, 'DUoRhQ=='); + assert(checksum.storageChecksum); + assert.strictEqual(checksum.storageChecksum.algorithm, 'crc64nvme'); + assert(checksum.storageChecksum.value); + done(); + }); + }); + + it('should fail with BadDigest when secondary checksum does not match', done => { + batchDeleteSucceeds(); + putSucceeds(); + const request = makeStream({ + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + }, 'hello world'); + const dualChecksums = { + primary: { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }, + secondary: { algorithm: 'crc32', isTrailer: false, expected: 'AAAAAA==' }, + }; + dataStore({}, null, request, 0, null, {}, dualChecksums, log, err => { + assert.strictEqual(err.message, 'BadDigest'); + assert(batchDeleteStub.calledOnce); + done(); + }); + }); + + it('should return no storageChecksum when secondary is null', done => { + putSucceeds(); + const request = makeStream({ + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + }, 'hello world'); + const singleChecksums = { + primary: { algorithm: 'crc32', isTrailer: false, expected: 'DUoRhQ==' }, + secondary: null, + }; + dataStore({}, null, request, 0, null, {}, singleChecksums, log, (err, dataInfo, hash, checksum) => { + assert.strictEqual(err, null); + assert.strictEqual(checksum.algorithm, 'crc32'); + assert.strictEqual(checksum.value, 'DUoRhQ=='); + assert.strictEqual(checksum.storageChecksum, undefined); + done(); + }); + }); + }); + describe('batchDelete failure paths', () => { it('should call cb with checksum error when validateChecksum fails and batchDelete also fails', done => { batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.InternalError)); putSucceeds(); const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - 'x-amz-checksum-crc32': 'AAAAAA==', }, 'hello world'); - dataStore({}, null, request, 0, null, {}, log, err => { + const badChecksums = { + primary: { algorithm: 'crc32', isTrailer: false, expected: 'AAAAAA==' }, + secondary: null, + }; + dataStore({}, null, request, 0, null, {}, badChecksums, log, err => { assert.strictEqual(err.message, 'BadDigest'); done(); }); @@ -283,7 +353,7 @@ describe('dataStore', () => { putSucceeds('correct-md5'); const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); request.contentMD5 = 'wrong-md5'; - dataStore({}, null, request, 0, null, {}, log, err => { + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { assert.deepStrictEqual(err, errors.BadDigest); done(); }); @@ -298,7 +368,7 @@ describe('dataStore', () => { cb(null, fakeDataRetrievalInfo, { completedHash: null }); }); const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); - dataStore({}, null, request, 0, null, {}, log, err => { + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { assert.deepStrictEqual(err, errors.InternalError); done(); }); diff --git a/tests/unit/api/objectPutPartChecksum.js b/tests/unit/api/objectPutPartChecksum.js new file mode 100644 index 0000000000..f518ca4885 --- /dev/null +++ b/tests/unit/api/objectPutPartChecksum.js @@ -0,0 +1,301 @@ +const assert = require('assert'); +const async = require('async'); +const crypto = require('crypto'); +const { storage } = require('arsenal'); +const { parseString } = require('xml2js'); + +const { bucketPut } = require('../../../lib/api/bucketPut'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); +const objectPutPart = require('../../../lib/api/objectPutPart'); +const constants = require('../../../constants'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); +const DummyRequest = require('../DummyRequest'); +const { algorithms } = require('../../../lib/api/apiUtils/integrity/validateChecksums'); + +const { metadata } = storage.metadata.inMemory.metadata; + +const log = new DummyRequestLogger(); +const canonicalID = 'accessKey1'; +const authInfo = makeAuthInfo(canonicalID); +const namespace = 'default'; +const bucketName = 'checksum-test-bucket'; +const objectKey = 'testObject'; +const mpuBucket = `${constants.mpuBucketPrefix}${bucketName}`; +const partBody = Buffer.from('I am a part body for checksum testing', 'utf8'); + +const bucketPutRequest = { + bucketName, + namespace, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + actionImplicitDenies: false, +}; + +function makeInitiateRequest(extraHeaders = {}) { + return { + socket: { remoteAddress: '1.1.1.1' }, + bucketName, + namespace, + objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + ...extraHeaders, + }, + url: `/${objectKey}?uploads`, + actionImplicitDenies: false, + }; +} + +function makePutPartRequest(uploadId, partNumber, body, extraHeaders = {}) { + const md5Hash = crypto.createHash('md5').update(body); + return new DummyRequest({ + bucketName, + namespace, + objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + ...extraHeaders, + }, + url: `/${objectKey}?partNumber=${partNumber}&uploadId=${uploadId}`, + query: { partNumber, uploadId }, + partHash: md5Hash.digest('hex'), + actionImplicitDenies: false, + }, body); +} + +function initiateMPU(initiateHeaders, cb) { + async.waterfall([ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => { + const req = makeInitiateRequest(initiateHeaders); + initiateMultipartUpload(authInfo, req, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + ], (err, json) => { + if (err) {return cb(err);} + return cb(null, json.InitiateMultipartUploadResult.UploadId[0]); + }); +} + +function getPartMetadata(uploadId) { + const mpuKeys = metadata.keyMaps.get(mpuBucket); + if (!mpuKeys) {return null;} + for (const [key, val] of mpuKeys) { + if (key.startsWith(uploadId) && !key.startsWith('overview')) { + return val; + } + } + return null; +} + +describe('objectPutPart checksum validation', () => { + beforeEach(() => cleanup()); + + describe('algo match validation', () => { + it('should accept part with matching checksum algo', done => { + initiateMPU({ 'x-amz-checksum-algorithm': 'crc32' }, (err, uploadId) => { + assert.ifError(err); + const request = makePutPartRequest(uploadId, 1, partBody, { + 'x-amz-checksum-crc32': 'AAAAAA==', + }); + objectPutPart(authInfo, request, undefined, log, err => { + // BadDigest is expected since the checksum value won't + // match the body, but NOT InvalidRequest — the algo is accepted. + if (err) { + assert.notStrictEqual(err.message, 'InvalidRequest'); + } + done(); + }); + }); + }); + + it('should reject part with mismatching checksum algo', done => { + initiateMPU({ 'x-amz-checksum-algorithm': 'sha256' }, (err, uploadId) => { + assert.ifError(err); + const request = makePutPartRequest(uploadId, 1, partBody, { + 'x-amz-checksum-crc32': 'AAAAAA==', + }); + objectPutPart(authInfo, request, undefined, log, err => { + assert(err, 'Expected an error'); + assert.strictEqual(err.message, 'InvalidRequest'); + done(); + }); + }); + }); + + it('should accept part with no checksum on non-default MPU', done => { + initiateMPU({ 'x-amz-checksum-algorithm': 'sha256' }, (err, uploadId) => { + assert.ifError(err); + // No checksum header sent + const request = makePutPartRequest(uploadId, 1, partBody); + objectPutPart(authInfo, request, undefined, log, err => { + assert.ifError(err); + done(); + }); + }); + }); + + it('should return BadDigest when matching algo but wrong digest', done => { + initiateMPU({ 'x-amz-checksum-algorithm': 'sha256' }, (err, uploadId) => { + assert.ifError(err); + // Algo matches MPU (sha256) but digest is wrong + const request = makePutPartRequest(uploadId, 1, partBody, { + 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + }); + objectPutPart(authInfo, request, undefined, log, err => { + assert(err, 'Expected an error'); + assert.strictEqual(err.message, 'BadDigest'); + done(); + }); + }); + }); + + it('should return InvalidRequest when MPU algo is sha256 and part sends crc32', done => { + initiateMPU({ 'x-amz-checksum-algorithm': 'sha256' }, (err, uploadId) => { + assert.ifError(err); + const request = makePutPartRequest(uploadId, 1, partBody, { + 'x-amz-checksum-crc32': 'DUoRhQ==', + }); + objectPutPart(authInfo, request, undefined, log, err => { + assert(err, 'Expected an error'); + assert.strictEqual(err.message, 'InvalidRequest'); + done(); + }); + }); + }); + + it('should accept any checksum algo on default (no algo specified) MPU', done => { + initiateMPU({}, (err, uploadId) => { + assert.ifError(err); + // Send sha256 checksum even though MPU is default crc64nvme + const request = makePutPartRequest(uploadId, 1, partBody, { + 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + }); + objectPutPart(authInfo, request, undefined, log, err => { + // BadDigest (wrong value) is fine; InvalidRequest (wrong algo) is not + if (err) { + assert.notStrictEqual(err.message, 'InvalidRequest'); + } + done(); + }); + }); + }); + }); + + describe('checksum stored in part metadata', () => { + it('should store checksumValue and checksumAlgorithm in part metadata', done => { + initiateMPU({}, (err, uploadId) => { + assert.ifError(err); + const request = makePutPartRequest(uploadId, 1, partBody); + objectPutPart(authInfo, request, undefined, log, err => { + assert.ifError(err); + const partMD = getPartMetadata(uploadId); + assert(partMD, 'Part metadata should exist'); + assert(partMD.checksumValue, 'checksumValue should be stored'); + assert.strictEqual(partMD.checksumAlgorithm, 'crc64nvme'); + done(); + }); + }); + }); + + it('should store the MPU algo checksum when client sends matching algo', done => { + initiateMPU({ 'x-amz-checksum-algorithm': 'crc64nvme' }, (err, uploadId) => { + assert.ifError(err); + const request = makePutPartRequest(uploadId, 1, partBody); + objectPutPart(authInfo, request, undefined, log, err => { + assert.ifError(err); + const partMD = getPartMetadata(uploadId); + assert(partMD); + assert.strictEqual(partMD.checksumAlgorithm, 'crc64nvme'); + assert(partMD.checksumValue); + done(); + }); + }); + }); + }); + + describe('dual-checksum', () => { + it('should store crc64nvme when default MPU and client sends different algo', done => { + initiateMPU({}, (err, uploadId) => { + assert.ifError(err); + const crc32Hash = algorithms.crc32.createHash(); + crc32Hash.update(partBody); + const crc64Hash = algorithms.crc64nvme.createHash(); + crc64Hash.update(partBody); + Promise.all([ + algorithms.crc32.digestFromHash(crc32Hash), + algorithms.crc64nvme.digestFromHash(crc64Hash), + ]).then(([crc32Digest, crc64Digest]) => { + const request = makePutPartRequest(uploadId, 1, partBody, { + 'x-amz-checksum-crc32': crc32Digest, + }); + objectPutPart(authInfo, request, undefined, log, (err, hexDigest, corsHeaders) => { + assert.ifError(err); + // Response header should be the client's algo (crc32) + assert.strictEqual(corsHeaders['x-amz-checksum-crc32'], crc32Digest); + // Stored metadata should be crc64nvme with correct value + const partMD = getPartMetadata(uploadId); + assert(partMD); + assert.strictEqual(partMD.checksumAlgorithm, 'crc64nvme'); + assert.strictEqual(partMD.checksumValue, crc64Digest); + done(); + }); + }).catch(done); + }); + }); + + it('should handle dual-checksum with trailer (STREAMING-UNSIGNED-PAYLOAD-TRAILER)', done => { + initiateMPU({}, (err, uploadId) => { + assert.ifError(err); + const hash = algorithms.sha256.createHash(); + hash.update(partBody); + const crc64Hash = algorithms.crc64nvme.createHash(); + crc64Hash.update(partBody); + Promise.all([ + algorithms.sha256.digestFromHash(hash), + algorithms.crc64nvme.digestFromHash(crc64Hash), + ]).then(([sha256Digest, crc64Digest]) => { + // Build chunked body with trailing checksum + const hexLen = partBody.length.toString(16); + const chunkedBody = `${hexLen}\r\n${partBody.toString()}\r\n` + + `0\r\nx-amz-checksum-sha256:${sha256Digest}\r\n`; + const request = makePutPartRequest(uploadId, 1, Buffer.from(chunkedBody), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + }); + request.parsedContentLength = partBody.length; + objectPutPart(authInfo, request, undefined, log, (err, hexDigest, corsHeaders) => { + assert.ifError(err); + // Response should echo the client's sha256 + assert.strictEqual(corsHeaders['x-amz-checksum-sha256'], sha256Digest); + // Stored metadata should be crc64nvme with correct value + const partMD = getPartMetadata(uploadId); + assert(partMD); + assert.strictEqual(partMD.checksumAlgorithm, 'crc64nvme'); + assert.strictEqual(partMD.checksumValue, crc64Digest); + done(); + }); + }).catch(done); + }); + }); + + it('should return client-facing checksum in response header for dual-checksum', done => { + initiateMPU({}, (err, uploadId) => { + assert.ifError(err); + const hash = algorithms.sha256.createHash(); + hash.update(partBody); + Promise.resolve(algorithms.sha256.digestFromHash(hash)).then(digest => { + const request = makePutPartRequest(uploadId, 1, partBody, { + 'x-amz-checksum-sha256': digest, + }); + objectPutPart(authInfo, request, undefined, log, (err, hexDigest, corsHeaders) => { + assert.ifError(err); + assert.strictEqual(corsHeaders['x-amz-checksum-sha256'], digest); + assert.strictEqual(corsHeaders['x-amz-checksum-crc64nvme'], undefined); + done(); + }); + }).catch(done); + }); + }); + }); +}); diff --git a/tests/unit/routes/routeBackbeat.js b/tests/unit/routes/routeBackbeat.js index 2f2621e9b2..5d21d61a70 100644 --- a/tests/unit/routes/routeBackbeat.js +++ b/tests/unit/routes/routeBackbeat.js @@ -162,7 +162,7 @@ describe('routeBackbeat', () => { callback(null, bucketInfo, objMd); }); storeObject.dataStore.callsFake((objectContext, cipherBundle, stream, size, - streamingV4Params, backendInfo, log, callback) => { + streamingV4Params, backendInfo, checksums, log, callback) => { callback(null, {}, md5); });