diff --git a/lib/api/apiUtils/integrity/validateChecksums.js b/lib/api/apiUtils/integrity/validateChecksums.js index ea42aac27b..22f5f0ca40 100644 --- a/lib/api/apiUtils/integrity/validateChecksums.js +++ b/lib/api/apiUtils/integrity/validateChecksums.js @@ -5,6 +5,7 @@ const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt'); const { errors: ArsenalErrors, errorInstances } = require('arsenal'); const { config } = require('../../../Config'); const { combinePartCrcs } = require('./crcCombine'); +const { supportedSignatureChecksums, unsupportedSignatureChecksums } = require('../../../../constants'); const defaultChecksumData = Object.freeze({ algorithm: 'crc64nvme', isTrailer: false, expected: undefined }); @@ -45,6 +46,21 @@ const errCopyChecksumAlgoNotSupported = errorInstances.InvalidRequest.customizeD '[CRC32, CRC32C, CRC64NVME, SHA1, SHA256]', ); +const errContentSHA256Mismatch = errorInstances.XAmzContentSHA256Mismatch; + +const errMissingContentSHA256 = errorInstances.InvalidRequest.customizeDescription( + 'Missing required header for this request: x-amz-content-sha256', +); + +const errInvalidContentSHA256 = errorInstances.InvalidArgument.customizeDescription( + 'x-amz-content-sha256 must be UNSIGNED-PAYLOAD, STREAMING-UNSIGNED-PAYLOAD-TRAILER, ' + + 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD, STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER, ' + + 'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD, STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER ' + + 'or a valid sha256 value.', +); + +const sha256HexRegex = /^[0-9a-f]{64}$/i; + // Methods that validate BOTH Content-MD5 and x-amz-checksum-* against the // buffered request body. For these, x-amz-checksum-* is the body digest. const checksumedMethods = Object.freeze({ @@ -98,6 +114,9 @@ const ChecksumError = Object.freeze({ MPUTypeWithoutAlgo: 'MPUTypeWithoutAlgo', MPUInvalidCombination: 'MPUInvalidCombination', CopyChecksumAlgoNotSupported: 'CopyChecksumAlgoNotSupported', + ContentSHA256Missing: 'ContentSHA256Missing', + ContentSHA256Invalid: 'ContentSHA256Invalid', + ContentSHA256Mismatch: 'ContentSHA256Mismatch', }); const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/; @@ -368,6 +387,90 @@ function validateContentMd5(headers, body) { return null; } +// Classification of an x-amz-content-sha256 header value, returned by +// parseContentSHA256. +const ContentSHA256Type = Object.freeze({ + Skip: 0, // not SigV4 header auth; do not enforce + Absent: 1, // SigV4 header auth but header missing + Unsigned: 2, // UNSIGNED-PAYLOAD + Streaming: 3, // a STREAMING-* token + HexSHA256: 4, // a hex sha256 of the payload + Invalid: 5, // anything else (malformed) +}); + +/** + * parseContentSHA256 - Classify x-amz-content-sha256 (the SigV4 payload hash) + * for a request. The header is only meaningful for SigV4 header-authenticated + * requests, so anything else (SigV2, presigned/query, anonymous) returns + * { type: ContentSHA256Type.Skip } and is not enforced. Otherwise the header + * value is classified. + * + * @param {object} headers - http request headers + * @return {object} { type, value } where type is a ContentSHA256Type and value + * is the raw header value (null when not sent). Streaming results also carry + * a { supported } boolean. + */ +function parseContentSHA256(headers) { + const value = headers['x-amz-content-sha256'] ?? null; + const authHeader = headers.authorization; + if (typeof authHeader !== 'string' || !authHeader.startsWith('AWS4')) { + return { type: ContentSHA256Type.Skip, value }; + } + if (value === null) { + return { type: ContentSHA256Type.Absent, value }; + } + if (value === 'UNSIGNED-PAYLOAD') { + return { type: ContentSHA256Type.Unsigned, value }; + } + if (supportedSignatureChecksums.has(value)) { + return { type: ContentSHA256Type.Streaming, value, supported: true }; + } + if (unsupportedSignatureChecksums.has(value)) { + return { type: ContentSHA256Type.Streaming, value, supported: false }; + } + if (sha256HexRegex.test(value)) { + return { type: ContentSHA256Type.HexSHA256, value }; + } + return { type: ContentSHA256Type.Invalid, value }; +} + +/** + * validateXAmzContentSHA256 - Validate the SHA256 of a SigV4 request. + * + * @param {object} headers - http request headers + * @param {Buffer} body - buffered http request body + * @return {object|null} - { error: ChecksumError } on missing/invalid/mismatch, + * else null + */ +function validateXAmzContentSHA256(headers, body) { + const parsed = parseContentSHA256(headers); + switch (parsed.type) { + case ContentSHA256Type.Absent: // required for SigV4 header auth + return { error: ChecksumError.ContentSHA256Missing }; + case ContentSHA256Type.Invalid: + return { error: ChecksumError.ContentSHA256Invalid, details: { value: parsed.value } }; + case ContentSHA256Type.HexSHA256: { + const computed = crypto.createHash('sha256').update(body).digest('hex'); + if (computed !== parsed.value.toLowerCase()) { + return { + error: ChecksumError.ContentSHA256Mismatch, + details: { calculated: computed, expected: parsed.value }, + }; + } + return null; + } + // Skip (non-SigV4-header auth), Unsigned and Streaming are not a literal + // payload hash, so there is nothing to validate here. + case ContentSHA256Type.Skip: + return null; + case ContentSHA256Type.Unsigned: + return null; + case ContentSHA256Type.Streaming: + return null; + } + return null; +} + /** * validateChecksumsNoChunking - Validate the checksums of a request. * @param {object} headers - http headers @@ -448,6 +551,12 @@ function arsenalErrorFromChecksumError(err) { ); case ChecksumError.CopyChecksumAlgoNotSupported: return errCopyChecksumAlgoNotSupported; + case ChecksumError.ContentSHA256Missing: + return errMissingContentSHA256; + case ChecksumError.ContentSHA256Invalid: + return errInvalidContentSHA256; + case ChecksumError.ContentSHA256Mismatch: + return errContentSHA256Mismatch; default: return ArsenalErrors.BadDigest; } @@ -547,6 +656,15 @@ function md5OnlyValidationFunc(request, body, log) { * @return {object} - error */ async function validateMethodChecksumNoChunking(request, body, log) { + const contentSHA256Err = validateXAmzContentSHA256(request.headers, body); + if (contentSHA256Err) { + log.debug('failed x-amz-content-sha256 validation', { + method: request.apiMethod, + ...contentSHA256Err.details, + }); + return arsenalErrorFromChecksumError(contentSHA256Err); + } + if (config.integrityChecks[request.apiMethod] === false) { return null; } @@ -738,6 +856,10 @@ module.exports = { ChecksumError, defaultChecksumData, validateChecksumsNoChunking, + ContentSHA256Type, + parseContentSHA256, + errInvalidContentSHA256, + validateXAmzContentSHA256, validateMethodChecksumNoChunking, getChecksumDataFromHeaders, arsenalErrorFromChecksumError, diff --git a/lib/api/apiUtils/object/prepareStream.js b/lib/api/apiUtils/object/prepareStream.js index c267936b83..66895fba6f 100644 --- a/lib/api/apiUtils/object/prepareStream.js +++ b/lib/api/apiUtils/object/prepareStream.js @@ -1,6 +1,8 @@ const V4Transform = require('../../../auth/streamingV4/V4Transform'); const TrailingChecksumTransform = require('../../../auth/streamingV4/trailingChecksumTransform'); const ChecksumTransform = require('../../../auth/streamingV4/ChecksumTransform'); +const ContentSHA256Transform = require('../../../auth/streamingV4/ContentSHA256Transform'); +const { parseContentSHA256, ContentSHA256Type } = require('../integrity/validateChecksums'); const { errors, errorInstances, jsutil } = require('arsenal'); const { unsupportedSignatureChecksums } = require('../../../../constants'); @@ -103,20 +105,33 @@ function prepareStream(request, streamingV4Params, checksums, log, errCb) { }; } - const onStreamError = secondary ? jsutil.once(errCb) : errCb; + const parsedContentSHA256 = parseContentSHA256(request.headers); + const shouldValidateContentSHA256 = parsedContentSHA256.type === ContentSHA256Type.HexSHA256; + const onStreamError = (secondary || shouldValidateContentSHA256) ? jsutil.once(errCb) : errCb; + let contentSHA256Stream = null; let secondaryChecksumStream = null; let stream = request; + if (shouldValidateContentSHA256) { + contentSHA256Stream = new ContentSHA256Transform(parsedContentSHA256.value, log); + contentSHA256Stream.on('error', onStreamError); + stream = stream.pipe(contentSHA256Stream); + } if (secondary) { secondaryChecksumStream = new ChecksumTransform( secondary.algorithm, secondary.expected, secondary.isTrailer, log); secondaryChecksumStream.on('error', onStreamError); - stream = request.pipe(secondaryChecksumStream); + stream = stream.pipe(secondaryChecksumStream); } const primaryStream = new ChecksumTransform(primary.algorithm, primary.expected, primary.isTrailer, log); primaryStream.on('error', onStreamError); - return { error: null, stream: stream.pipe(primaryStream), secondaryChecksumStream }; + return { + error: null, + stream: stream.pipe(primaryStream), + secondaryChecksumStream, + contentSHA256Stream, + }; } } } diff --git a/lib/api/apiUtils/object/storeObject.js b/lib/api/apiUtils/object/storeObject.js index ae4db49165..06dd669052 100644 --- a/lib/api/apiUtils/object/storeObject.js +++ b/lib/api/apiUtils/object/storeObject.js @@ -100,10 +100,24 @@ function dataStore(objectContext, cipherBundle, stream, size, streamingV4Params, }; // stream is always the primary (end of pipe, stored checksum). - // secondaryChecksumStream is upstream and only validated. - const { secondaryChecksumStream } = checksumedStream; + // secondaryChecksumStream and contentSHA256Stream are upstream and + // only validated. + const { secondaryChecksumStream, contentSHA256Stream } = checksumedStream; const doValidate = () => { + // Validate the SigV4 payload hash (x-amz-content-sha256) first. + if (contentSHA256Stream) { + const contentErr = contentSHA256Stream.validateChecksum(); + if (contentErr) { + log.debug('failed x-amz-content-sha256 validation', { error: contentErr }); + return data.batchDelete([dataRetrievalInfo], null, null, log, deleteErr => { + if (deleteErr) { + log.error('dataStore failed to delete old data', { error: deleteErr }); + } + return cbOnce(arsenalErrorFromChecksumError(contentErr)); + }); + } + } // Validate the secondary (checked-only) checksum first. if (secondaryChecksumStream) { const secondaryErr = secondaryChecksumStream.validateChecksum(); diff --git a/lib/api/apiUtils/object/validateChecksumHeaders.js b/lib/api/apiUtils/object/validateChecksumHeaders.js deleted file mode 100644 index d2a50395a3..0000000000 --- a/lib/api/apiUtils/object/validateChecksumHeaders.js +++ /dev/null @@ -1,33 +0,0 @@ -const { errorInstances } = require('arsenal'); - -const { unsupportedSignatureChecksums, supportedSignatureChecksums } = require('../../../../constants'); - -function validateChecksumHeaders(headers) { - // If the x-amz-trailer header is present the request is using one of the - // trailing checksum algorithms, which are not supported. - if (headers['x-amz-trailer'] !== undefined && - headers['x-amz-content-sha256'] !== 'STREAMING-UNSIGNED-PAYLOAD-TRAILER') { - return errorInstances.BadRequest.customizeDescription('signed trailing checksum is not supported'); - } - - const signatureChecksum = headers['x-amz-content-sha256']; - if (signatureChecksum === undefined) { - return null; - } - - if (supportedSignatureChecksums.has(signatureChecksum)) { - return null; - } - - // If the value is not one of the possible checksum algorithms - // the only other valid value is the actual sha256 checksum of the payload. - // Do a simple sanity check of the length to guard against future algos. - // If the value is an unknown algo, then it will fail checksum validation. - if (!unsupportedSignatureChecksums.has(signatureChecksum) && signatureChecksum.length === 64) { - return null; - } - - return errorInstances.BadRequest.customizeDescription('unsupported checksum algorithm'); -} - -module.exports = validateChecksumHeaders; diff --git a/lib/api/apiUtils/object/validatePayloadProtocol.js b/lib/api/apiUtils/object/validatePayloadProtocol.js new file mode 100644 index 0000000000..19134b6278 --- /dev/null +++ b/lib/api/apiUtils/object/validatePayloadProtocol.js @@ -0,0 +1,32 @@ +const { errorInstances } = require('arsenal'); + +const { parseContentSHA256, ContentSHA256Type, errInvalidContentSHA256 } = require('../integrity/validateChecksums'); + +/** + * Validate the SigV4 payload protocol selected by x-amz-content-sha256. + * + * @param {object} headers - http request headers + * @return {ArsenalError|null} - error if the protocol is unsupported/malformed, else null + */ +function validatePayloadProtocol(headers) { + const parsed = parseContentSHA256(headers); + switch (parsed.type) { + case ContentSHA256Type.Skip: // not SigV4 header auth; header not meaningful + return null; + case ContentSHA256Type.Absent: + return null; + case ContentSHA256Type.Unsigned: + return null; + case ContentSHA256Type.HexSHA256: + return null; + case ContentSHA256Type.Streaming: + return parsed.supported + ? null + : errorInstances.BadRequest.customizeDescription(`${parsed.value} is not supported`); + case ContentSHA256Type.Invalid: + return errInvalidContentSHA256; + } + return null; +} + +module.exports = validatePayloadProtocol; diff --git a/lib/api/objectPut.js b/lib/api/objectPut.js index 590fd7f1e5..8bdce061ed 100644 --- a/lib/api/objectPut.js +++ b/lib/api/objectPut.js @@ -18,7 +18,7 @@ const monitoring = require('../utilities/monitoringHandler'); const { validatePutVersionId } = require('./apiUtils/object/coldStorage'); const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders'); const { setSSEHeaders } = require('./apiUtils/object/sseHeaders'); -const validateChecksumHeaders = require('./apiUtils/object/validateChecksumHeaders'); +const validatePayloadProtocol = require('./apiUtils/object/validatePayloadProtocol'); const writeContinue = require('../utilities/writeContinue'); const { overheadField } = require('../../constants'); @@ -119,9 +119,9 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) { )); } - const checksumHeaderErr = validateChecksumHeaders(headers); - if (checksumHeaderErr) { - return callback(checksumHeaderErr); + const payloadProtocolErr = validatePayloadProtocol(headers); + if (payloadProtocolErr) { + return callback(payloadProtocolErr); } log.trace('owner canonicalID to send to data', { canonicalID }); diff --git a/lib/api/objectPutPart.js b/lib/api/objectPutPart.js index f115b8b727..73cda90a76 100644 --- a/lib/api/objectPutPart.js +++ b/lib/api/objectPutPart.js @@ -20,7 +20,7 @@ const { BackendInfo } = models; const writeContinue = require('../utilities/writeContinue'); const { parseObjectEncryptionHeaders } = require('./apiUtils/bucket/bucketEncryption'); -const validateChecksumHeaders = require('./apiUtils/object/validateChecksumHeaders'); +const validatePayloadProtocol = require('./apiUtils/object/validatePayloadProtocol'); const { getChecksumDataFromHeaders, arsenalErrorFromChecksumError, @@ -77,9 +77,9 @@ function objectPutPart(authInfo, request, streamingV4Params, log, return cb(errors.EntityTooLarge); } - const checksumHeaderErr = validateChecksumHeaders(request.headers); - if (checksumHeaderErr) { - return cb(checksumHeaderErr); + const payloadProtocolErr = validatePayloadProtocol(request.headers); + if (payloadProtocolErr) { + return cb(payloadProtocolErr); } // Note: Part sizes cannot be less than 5MB in size except for the last. diff --git a/lib/auth/streamingV4/ContentSHA256Transform.js b/lib/auth/streamingV4/ContentSHA256Transform.js new file mode 100644 index 0000000000..a7201db46f --- /dev/null +++ b/lib/auth/streamingV4/ContentSHA256Transform.js @@ -0,0 +1,40 @@ +const crypto = require('crypto'); +const { Transform } = require('stream'); +const { ChecksumError } = require('../../api/apiUtils/integrity/validateChecksums'); + +/** + * Computes the sha256 of the streamed body to verify it against a literal + * x-amz-content-sha256 header (the SigV4 payload hash) via validateChecksum(). + */ +class ContentSHA256Transform extends Transform { + constructor(expectedDigest, log) { + super({}); + this.log = log; + this.expectedDigest = (expectedDigest || '').toLowerCase(); + this.hash = crypto.createHash('sha256'); + this.digest = undefined; + } + + validateChecksum() { + if (this.digest !== this.expectedDigest) { + return { + error: ChecksumError.ContentSHA256Mismatch, + details: { calculated: this.digest, expected: this.expectedDigest }, + }; + } + return null; + } + + _transform(chunk, encoding, callback) { + const input = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + this.hash.update(input); + callback(null, input); + } + + _flush(callback) { + this.digest = this.hash.digest('hex'); + callback(); + } +} + +module.exports = ContentSHA256Transform; diff --git a/package.json b/package.json index aa7f2790f1..b44de4b576 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "@opentelemetry/instrumentation-ioredis": "~0.64.0", "@opentelemetry/instrumentation-mongodb": "~0.69.0", "@smithy/node-http-handler": "^3.0.0", - "arsenal": "git+https://github.com/scality/arsenal#8.5.0", + "arsenal": "git+https://github.com/scality/arsenal#improvement/ARSN-602-add-XAmzContentSHA256Mismatch-error", "async": "2.6.4", "bucketclient": "scality/bucketclient#8.2.7", "bufferutil": "^4.0.8", diff --git a/tests/functional/raw-node/test/xAmzChecksum.js b/tests/functional/raw-node/test/xAmzChecksum.js index 14df75dabc..57a0d59e2a 100644 --- a/tests/functional/raw-node/test/xAmzChecksum.js +++ b/tests/functional/raw-node/test/xAmzChecksum.js @@ -1,9 +1,12 @@ const assert = require('assert'); +const crypto = require('crypto'); const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4'); const bucket = 'xxx'; const objectKey = 'key'; const objData = Buffer.alloc(1, 'a'); +// SigV4 requires x-amz-content-sha256 to be the hex-encoded sha256 of the body. +const objDataSha256Hex = crypto.createHash('sha256').update(objData).digest('hex'); const authCredentials = { accessKey: 'accessKey1', @@ -147,7 +150,7 @@ describe('Test x-amz-checksums', () => { { method: method.HTTPMethod, headers: { - 'x-amz-content-sha256': 'ypeBEsobvcr6wjGzmiPcTaeG7/gUfE5yuYB3ha/uSLs=', + 'x-amz-content-sha256': objDataSha256Hex, 'content-length': objData.length, ...headers, }, @@ -283,7 +286,7 @@ describe('Test x-amz-checksums', () => { { method: method.HTTPMethod, headers: { - 'x-amz-content-sha256': 'ypeBEsobvcr6wjGzmiPcTaeG7/gUfE5yuYB3ha/uSLs=', + 'x-amz-content-sha256': objDataSha256Hex, 'content-length': objData.length, 'x-amz-sdk-checksum-algorithm': algo.name, [`x-amz-checksum-${algo.name.toLowerCase()}`]: algo.objDataDigest, diff --git a/tests/functional/raw-node/test/xAmzContentSha256Mismatch.js b/tests/functional/raw-node/test/xAmzContentSha256Mismatch.js new file mode 100644 index 0000000000..1c23df570d --- /dev/null +++ b/tests/functional/raw-node/test/xAmzContentSha256Mismatch.js @@ -0,0 +1,264 @@ +const assert = require('assert'); +const crypto = require('crypto'); +const async = require('async'); + +const { makeS3Request } = require('../utils/makeRequest'); +const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4'); + +const config = require('../../config.json'); +const { checksumedMethods } = require('../../../../lib/api/apiUtils/integrity/validateChecksums'); + +// Regression test for S3C-10916: "[SigV4] x-amz-content-sha256 value not checked". +// +// CloudServer used to trust x-amz-content-sha256 without recomputing the body's +// SHA256, accepting a request signed with a wrong-but-well-formed hash. The fix +// verifies the header against the body on the buffered and streaming paths. These +// tests assert the AWS-correct 400 XAmzContentSHA256Mismatch, passing against real +// AWS (AWS_ON_AIR=true) and CloudServer with the fix. + +const bucket = 'contentsha256mismatchbucket'; +const objectKey = 'key'; + +const authCredentials = { + accessKey: config.accessKey, + secretKey: config.secretKey, +}; + +const host = process.env.AWS_ON_AIR ? 's3.amazonaws.com' : '127.0.0.1'; +const port = process.env.AWS_ON_AIR ? 80 : 8000; +const itSkipIfAWS = process.env.AWS_ON_AIR ? it.skip : it; + +const objData = Buffer.from('the real request body content'); +const realSha256Hex = crypto.createHash('sha256').update(objData).digest('hex'); +const wrongSha256Hex = crypto.createHash('sha256') + .update('completely different content') + .digest('hex'); +const invalidSha256 = 'xxx'; + +// An arbitrary body that is never parsed: the x-amz-content-sha256 check rejects +// the request before the handler reads it, so its content is irrelevant. +const fakeBody = Buffer.from('not parsed before the content-sha256 check'); + +const bufferedEndpoints = { + multiObjectDelete: { method: 'POST', suffix: '?delete' }, + bucketPut: { method: 'PUT', suffix: '' }, // CreateBucket (no subresource) + bucketPutACL: { method: 'PUT', suffix: '?acl' }, + bucketPutCors: { method: 'PUT', suffix: '?cors' }, + bucketPutEncryption: { method: 'PUT', suffix: '?encryption' }, + bucketPutLifecycle: { method: 'PUT', suffix: '?lifecycle' }, + bucketPutLogging: { method: 'PUT', suffix: '?logging' }, + bucketPutNotification: { method: 'PUT', suffix: '?notification' }, + bucketPutPolicy: { method: 'PUT', suffix: '?policy' }, + bucketPutReplication: { method: 'PUT', suffix: '?replication' }, + bucketPutTagging: { method: 'PUT', suffix: '?tagging' }, + bucketPutVersioning: { method: 'PUT', suffix: '?versioning' }, + bucketPutWebsite: { method: 'PUT', suffix: '?website' }, + bucketPutObjectLock: { method: 'PUT', suffix: '?object-lock' }, + objectPutACL: { method: 'PUT', suffix: `/${objectKey}?acl` }, + objectPutLegalHold: { method: 'PUT', suffix: `/${objectKey}?legal-hold` }, + objectPutRetention: { method: 'PUT', suffix: `/${objectKey}?retention` }, + objectPutTagging: { method: 'PUT', suffix: `/${objectKey}?tagging` }, + objectRestore: { method: 'POST', suffix: `/${objectKey}?restore` }, + completeMultipartUpload: { method: 'POST', suffix: `/${objectKey}?uploadId=fakeUploadId` }, +}; + +const scalityExtensionEndpoints = { + bucketUpdateQuota: { method: 'PUT', suffix: '?quota' }, + bucketPutRateLimit: { method: 'PUT', suffix: '?rate-limit' }, +}; + +function doRequest(method, url, headers, body, callback) { + const req = new HttpRequestAuthV4( + url, + Object.assign({ method, headers }, authCredentials), + res => { + let data = ''; + res.on('data', chunk => { data += chunk; }); + res.on('end', () => callback(null, { + statusCode: res.statusCode, + body: data, + headers: res.headers, + })); + }, + ); + req.on('error', callback); + req.write(body); + req.end(); +} + +const doPutRequest = (url, headers, body, callback) => doRequest('PUT', url, headers, body, callback); + +function makeMismatchTests(urlFn, body = objData, correctHex = realSha256Hex) { + it('should reject a body whose x-amz-content-sha256 does not match with 400 XAmzContentSHA256Mismatch', + done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': wrongSha256Hex, + 'content-length': body.length, + }, body, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 400, + `expected 400, got ${res.statusCode}: ${res.body}`); + assert.match(res.body, /XAmzContentSHA256Mismatch/, + `expected XAmzContentSHA256Mismatch in "${res.body}"`); + done(); + }); + }); + + it('should accept a body whose x-amz-content-sha256 matches', done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': correctHex, + 'content-length': body.length, + }, body, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200, + `expected 200, got ${res.statusCode}: ${res.body}`); + done(); + }); + }); + + it('should reject an invalid x-amz-content-sha256 value with 400 InvalidArgument', done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': invalidSha256, + 'content-length': body.length, + }, body, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 400, + `expected 400, got ${res.statusCode}: ${res.body}`); + assert.match(res.body, /InvalidArgument/, + `expected InvalidArgument in "${res.body}"`); + done(); + }); + }); +} + +describe('SigV4 x-amz-content-sha256 body checksum validation (S3C-10916)', () => { + describe('PutObject', () => { + before(done => { + makeS3Request({ method: 'PUT', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + + after(done => { + makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => { + makeS3Request({ method: 'DELETE', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + }); + + makeMismatchTests(() => `http://${host}:${port}/${bucket}/${objectKey}`); + }); + + describe('UploadPart', () => { + let uploadId; + + before(done => { + async.series([ + next => makeS3Request({ method: 'PUT', authCredentials, bucket }, next), + next => makeS3Request({ + method: 'POST', + authCredentials, + bucket, + objectKey, + queryObj: { uploads: '' }, + }, (err, res) => { + if (err) { return next(err); } + const match = res.body.match(/([^<]+)<\/UploadId>/); + assert(match, `missing UploadId in response: ${res.body}`); + uploadId = match[1]; + return next(); + }), + ], err => { + assert.ifError(err); + done(); + }); + }); + + after(done => { + async.series([ + next => makeS3Request({ + method: 'DELETE', + authCredentials, + bucket, + objectKey, + queryObj: { uploadId }, + }, next), + // Delete the object key first (defensive: clears any state left by a previous run). + next => makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => next()), + next => makeS3Request({ method: 'DELETE', authCredentials, bucket }, next), + ], err => { + assert.ifError(err); + done(); + }); + }); + + makeMismatchTests(() => + `http://${host}:${port}/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId}`); + }); + + // Non-streaming (buffered) write path: validation happens in + // validateMethodChecksumNoChunking, before the handler reads the body. A + // mismatched hash must therefore be rejected on every concerned endpoint. + describe('buffered endpoints', () => { + before(done => { + makeS3Request({ method: 'PUT', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + + after(done => { + makeS3Request({ method: 'DELETE', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + + // Regression sweep: a wrong-but-well-formed hash is rejected everywhere. + Object.entries(bufferedEndpoints).forEach(([apiMethod, ep]) => { + it(`should return 400 XAmzContentSHA256Mismatch for ${apiMethod}`, done => { + doRequest(ep.method, `http://${host}:${port}/${bucket}${ep.suffix}`, { + 'x-amz-content-sha256': wrongSha256Hex, + 'content-length': fakeBody.length, + }, fakeBody, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 400, + `expected 400, got ${res.statusCode}: ${res.body}`); + assert.match(res.body, /XAmzContentSHA256Mismatch/, + `expected XAmzContentSHA256Mismatch in "${res.body}"`); + done(); + }); + }); + }); + + // Fails if a new checksumed/buffered method is added without test coverage. + it('should exercise every buffered checksumed method', () => { + const expected = new Set([...Object.keys(checksumedMethods), 'completeMultipartUpload']); + const covered = new Set(Object.keys(bufferedEndpoints)); + expected.forEach(method => + assert(covered.has(method), `missing buffered-endpoint coverage for ${method}`)); + }); + }); + + // Scality-only admin extensions: same choke point, but absent from AWS. + describe('buffered Scality extensions (skipped on AWS)', () => { + Object.entries(scalityExtensionEndpoints).forEach(([apiMethod, ep]) => { + itSkipIfAWS(`should return 400 XAmzContentSHA256Mismatch for ${apiMethod}`, done => { + doRequest(ep.method, `http://${host}:${port}/${bucket}${ep.suffix}`, { + 'x-amz-content-sha256': wrongSha256Hex, + 'content-length': fakeBody.length, + }, fakeBody, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 400, + `expected 400, got ${res.statusCode}: ${res.body}`); + assert.match(res.body, /XAmzContentSHA256Mismatch/, + `expected XAmzContentSHA256Mismatch in "${res.body}"`); + done(); + }); + }); + }); + }); +}); diff --git a/tests/functional/raw-node/utils/makeRequest.js b/tests/functional/raw-node/utils/makeRequest.js index 6759160c2e..bfad8e774b 100644 --- a/tests/functional/raw-node/utils/makeRequest.js +++ b/tests/functional/raw-node/utils/makeRequest.js @@ -139,8 +139,10 @@ function makeRequest(params, callback) { // decode path because signing code re-encodes it req.path = _decodeURI(encodedPath); if (authCredentials && !params.GCP) { + // Pass an explicit payload (never undefined) so generateV4Headers signs + // the real body for POST instead of falling back to the querystring. auth.client.generateV4Headers(req, queryObj || '', - authCredentials.accessKey, authCredentials.secretKey, 's3', undefined, undefined, requestBody); + authCredentials.accessKey, authCredentials.secretKey, 's3', undefined, undefined, requestBody || ''); } // restore original URL-encoded path req.path = savedPath; diff --git a/tests/unit/api/apiUtils/integrity/validateChecksums.js b/tests/unit/api/apiUtils/integrity/validateChecksums.js index 1e62ad0e41..0dd415f30f 100644 --- a/tests/unit/api/apiUtils/integrity/validateChecksums.js +++ b/tests/unit/api/apiUtils/integrity/validateChecksums.js @@ -5,6 +5,9 @@ const { DummyRequestLogger } = require('../../../helpers'); const { validateChecksumsNoChunking, ChecksumError, + ContentSHA256Type, + parseContentSHA256, + validateXAmzContentSHA256, validateMethodChecksumNoChunking, checksumedMethods, getChecksumDataFromHeaders, @@ -1298,3 +1301,162 @@ describe('getCopyObjectChecksumAlgorithm', () => { } }); }); + +const sigV4Auth = 'AWS4-HMAC-SHA256 Credential=AK/20260101/us-east-1/s3/aws4_request, ' + + 'SignedHeaders=host, Signature=abc'; + +describe('parseContentSHA256', () => { + // build SigV4 header-auth headers carrying the given x-amz-content-sha256 + const v4 = value => ({ authorization: sigV4Auth, 'x-amz-content-sha256': value }); + + it('should return Skip for non-SigV4 header auth, capturing the value', () => { + assert.deepStrictEqual( + parseContentSHA256({ authorization: 'AWS AKID:sig', 'x-amz-content-sha256': 'abc' }), + { type: ContentSHA256Type.Skip, value: 'abc' }); + }); + + it('should return Skip with null value when there is no auth header', () => { + assert.deepStrictEqual(parseContentSHA256({}), { type: ContentSHA256Type.Skip, value: null }); + }); + + it('should return Absent when SigV4 header auth but the header is missing', () => { + assert.deepStrictEqual(parseContentSHA256({ authorization: sigV4Auth }), + { type: ContentSHA256Type.Absent, value: null }); + }); + + it('should return Unsigned for UNSIGNED-PAYLOAD', () => { + assert.deepStrictEqual(parseContentSHA256(v4('UNSIGNED-PAYLOAD')), + { type: ContentSHA256Type.Unsigned, value: 'UNSIGNED-PAYLOAD' }); + }); + + it('should return Streaming (supported) for a supported streaming token', () => { + const tok = 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'; + assert.deepStrictEqual(parseContentSHA256(v4(tok)), + { type: ContentSHA256Type.Streaming, value: tok, supported: true }); + }); + + it('should return Streaming (not supported) for an unsupported streaming token', () => { + const tok = 'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD'; + assert.deepStrictEqual(parseContentSHA256(v4(tok)), + { type: ContentSHA256Type.Streaming, value: tok, supported: false }); + }); + + it('should return HexSHA256 for a hex sha256, preserving the raw value', () => { + const hex = 'a'.repeat(64); + assert.deepStrictEqual(parseContentSHA256(v4(hex)), + { type: ContentSHA256Type.HexSHA256, value: hex }); + }); + + it('should return HexSHA256 for an uppercase hex sha256', () => { + const hex = 'A'.repeat(64); + assert.deepStrictEqual(parseContentSHA256(v4(hex)), + { type: ContentSHA256Type.HexSHA256, value: hex }); + }); + + it('should return Invalid for a malformed value', () => { + assert.deepStrictEqual(parseContentSHA256(v4('xxx')), + { type: ContentSHA256Type.Invalid, value: 'xxx' }); + }); +}); + +describe('validateXAmzContentSHA256', () => { + const body = 'Hello, World!'; + const correctHex = crypto.createHash('sha256').update(body).digest('hex'); + const wrongHex = crypto.createHash('sha256').update('other').digest('hex'); + + it('should return null when the hash matches the body', () => { + assert.ifError(validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': correctHex }, body)); + }); + + it('should return null for an uppercase hash that matches (case-insensitive)', () => { + assert.ifError(validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': correctHex.toUpperCase() }, body)); + }); + + it('should return ContentSHA256Mismatch with calculated/expected details on mismatch', () => { + const result = validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, body); + assert.strictEqual(result.error, ChecksumError.ContentSHA256Mismatch); + assert.strictEqual(result.details.expected, wrongHex); + assert.strictEqual(result.details.calculated, correctHex); + }); + + it('should return ContentSHA256Missing when the header is absent', () => { + assert.deepStrictEqual(validateXAmzContentSHA256({ authorization: sigV4Auth }, body), + { error: ChecksumError.ContentSHA256Missing }); + }); + + it('should return ContentSHA256Invalid for a malformed value', () => { + const result = validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': 'xxx' }, body); + assert.strictEqual(result.error, ChecksumError.ContentSHA256Invalid); + assert.strictEqual(result.details.value, 'xxx'); + }); + + it('should return null for UNSIGNED-PAYLOAD', () => { + assert.ifError(validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }, body)); + }); + + it('should return null for a streaming token', () => { + assert.ifError(validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }, body)); + }); + + it('should return null (skip) for non-SigV4 auth even with a wrong hash', () => { + assert.ifError(validateXAmzContentSHA256( + { authorization: 'AWS AKID:sig', 'x-amz-content-sha256': wrongHex }, body)); + }); + + describe('mapped through arsenalErrorFromChecksumError', () => { + it('should map mismatch to XAmzContentSHA256Mismatch (400)', () => { + const result = validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, body); + const err = arsenalErrorFromChecksumError(result); + assert.strictEqual(err.message, 'XAmzContentSHA256Mismatch'); + assert.strictEqual(err.code, 400); + }); + + it('should map invalid to InvalidArgument (400)', () => { + const result = validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': 'xxx' }, body); + const err = arsenalErrorFromChecksumError(result); + assert.strictEqual(err.message, 'InvalidArgument'); + assert.strictEqual(err.code, 400); + }); + + it('should map missing to InvalidRequest (400)', () => { + const result = validateXAmzContentSHA256({ authorization: sigV4Auth }, body); + const err = arsenalErrorFromChecksumError(result); + assert.strictEqual(err.message, 'InvalidRequest'); + assert.strictEqual(err.code, 400); + }); + }); +}); + +describe('validateMethodChecksumNoChunking x-amz-content-sha256', () => { + const body = 'Hello, World!'; + const correctHex = crypto.createHash('sha256').update(body).digest('hex'); + const correctMd5 = crypto.createHash('md5').update(body).digest('base64'); + + it('should reject a wrong x-amz-content-sha256 with XAmzContentSHA256Mismatch', async () => { + const wrongHex = crypto.createHash('sha256').update('other').digest('hex'); + const request = { + apiMethod: 'bucketPutCors', + headers: { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, + }; + const result = await validateMethodChecksumNoChunking(request, body, new DummyRequestLogger()); + assert.strictEqual(result.message, 'XAmzContentSHA256Mismatch'); + assert.strictEqual(result.code, 400); + }); + + it('should accept a matching x-amz-content-sha256', async () => { + const request = { + apiMethod: 'bucketPutCors', + headers: { authorization: sigV4Auth, 'x-amz-content-sha256': correctHex, 'content-md5': correctMd5 }, + }; + const result = await validateMethodChecksumNoChunking(request, body, new DummyRequestLogger()); + assert.ifError(result); + }); +}); diff --git a/tests/unit/api/apiUtils/object/prepareStream.js b/tests/unit/api/apiUtils/object/prepareStream.js index d0492615e5..05bcc628c4 100644 --- a/tests/unit/api/apiUtils/object/prepareStream.js +++ b/tests/unit/api/apiUtils/object/prepareStream.js @@ -1,8 +1,10 @@ const assert = require('assert'); +const crypto = require('crypto'); const { errors } = require('arsenal'); const { prepareStream } = require('../../../../../lib/api/apiUtils/object/prepareStream'); const ChecksumTransform = require('../../../../../lib/auth/streamingV4/ChecksumTransform'); +const ContentSHA256Transform = require('../../../../../lib/auth/streamingV4/ContentSHA256Transform'); const { DummyRequestLogger } = require('../../../helpers'); const DummyRequest = require('../../../DummyRequest'); const { defaultChecksumData } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); @@ -10,6 +12,13 @@ const { defaultChecksumData } = require('../../../../../lib/api/apiUtils/integri const log = new DummyRequestLogger(); const defaultChecksums = { primary: defaultChecksumData, secondary: null }; +// A literal payload hash is only verified for SigV4 header-authenticated +// requests, so these tests carry an AWS4 Authorization header. +const sigV4Auth = 'AWS4-HMAC-SHA256 Credential=AK/20210101/us-east-1/s3/aws4_request, ' + + 'SignedHeaders=host, Signature=abc'; +const bodyData = 'the streamed body'; +const bodyHex = crypto.createHash('sha256').update(Buffer.from(bodyData)).digest('hex'); + function makeRequest(headers, body) { return new DummyRequest({ headers }, body != null ? Buffer.from(body) : undefined); } @@ -256,4 +265,70 @@ describe('prepareStream', () => { result.stream.emit('error', errors.InternalError); }); }); + + describe('default (literal sha256 payload hash)', () => { + it('should return a ContentSHA256Transform for a SigV4 header-auth request with a hex value', () => { + const request = makeRequest({ authorization: sigV4Auth, 'x-amz-content-sha256': bodyHex }); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof ChecksumTransform); + assert(result.contentSHA256Stream instanceof ContentSHA256Transform); + }); + + it('should return null contentSHA256Stream for UNSIGNED-PAYLOAD', () => { + const request = makeRequest({ authorization: sigV4Auth, 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); + assert.strictEqual(result.contentSHA256Stream, null); + }); + + it('should return null contentSHA256Stream for non-SigV4 (SigV2) auth even with a hex value', () => { + const request = makeRequest({ authorization: 'AWS AKID:sig', 'x-amz-content-sha256': bodyHex }); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); + assert.strictEqual(result.contentSHA256Stream, null); + }); + + it('should return null contentSHA256Stream when there is no authorization header', () => { + const request = makeRequest({ 'x-amz-content-sha256': bodyHex }); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); + assert.strictEqual(result.contentSHA256Stream, null); + }); + + it('should accumulate the body sha256 so validateChecksum passes when the hex matches', done => { + const request = makeRequest({ authorization: sigV4Auth, 'x-amz-content-sha256': bodyHex }, bodyData); + const result = prepareStream(request, null, defaultChecksums, log, done); + result.stream.resume(); + result.stream.on('finish', () => { + assert.strictEqual(result.contentSHA256Stream.validateChecksum(), null); + done(); + }); + result.stream.on('error', done); + }); + + it('should pipe contentSHA256Stream upstream of a secondary checksum stream', done => { + const request = makeRequest({ authorization: sigV4Auth, 'x-amz-content-sha256': bodyHex }, bodyData); + const checksums = { + primary: { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }, + secondary: { algorithm: 'crc32', isTrailer: false, expected: undefined }, + }; + const result = prepareStream(request, null, checksums, log, done); + assert(result.contentSHA256Stream instanceof ContentSHA256Transform); + assert(result.secondaryChecksumStream instanceof ChecksumTransform); + result.stream.resume(); + result.stream.on('finish', () => { + // the content stream saw the full body even with a secondary present + assert.strictEqual(result.contentSHA256Stream.validateChecksum(), null); + done(); + }); + result.stream.on('error', done); + }); + + it('should invoke errCb only once when multiple streams error', () => { + const request = makeRequest({ authorization: sigV4Auth, 'x-amz-content-sha256': bodyHex }); + let count = 0; + const result = prepareStream(request, null, defaultChecksums, log, () => { count += 1; }); + result.contentSHA256Stream.emit('error', errors.InternalError); + result.stream.emit('error', errors.InternalError); + assert.strictEqual(count, 1); + }); + }); }); diff --git a/tests/unit/api/apiUtils/object/storeObject.js b/tests/unit/api/apiUtils/object/storeObject.js index 0ddd158a3c..ffef1fd717 100644 --- a/tests/unit/api/apiUtils/object/storeObject.js +++ b/tests/unit/api/apiUtils/object/storeObject.js @@ -1,4 +1,5 @@ const assert = require('assert'); +const crypto = require('crypto'); const sinon = require('sinon'); const { errors } = require('arsenal'); @@ -13,6 +14,13 @@ const defaultChecksums = { primary: defaultChecksumData, secondary: null }; const fakeDataRetrievalInfo = { key: 'test-key', dataStoreName: 'mem' }; +// A literal payload hash is only verified for SigV4 header-authenticated +// requests, so these tests carry an AWS4 Authorization header. +const sigV4Auth = 'AWS4-HMAC-SHA256 Credential=AK/20210101/us-east-1/s3/aws4_request, ' + + 'SignedHeaders=host, Signature=abc'; +const helloWorldHex = crypto.createHash('sha256').update(Buffer.from('hello world')).digest('hex'); +const wrongHex = 'a'.repeat(64); + function makeStream(headers = {}, body = '') { return new DummyRequest({ headers }, body ? Buffer.from(body) : undefined); } @@ -273,6 +281,62 @@ describe('dataStore', () => { }); }); + describe('x-amz-content-sha256 body validation', () => { + it('should call cb with XAmzContentSHA256Mismatch and delete stored data when the hash does not match', + done => { + batchDeleteSucceeds(); + putSucceeds(); + const request = makeStream( + { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, 'hello world'); + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { + assert.strictEqual(err.message, 'XAmzContentSHA256Mismatch'); + assert(batchDeleteStub.calledOnce); + done(); + }); + }); + + it('should not delete stored data when the hash matches the body', done => { + putSucceeds(); + const request = makeStream( + { authorization: sigV4Auth, 'x-amz-content-sha256': helloWorldHex }, 'hello world'); + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { + assert.strictEqual(err, null); + assert(batchDeleteStub.notCalled); + done(); + }); + }); + + it('should validate x-amz-content-sha256 before the secondary checksum', done => { + batchDeleteSucceeds(); + putSucceeds(); + const request = makeStream( + { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, 'hello world'); + // The secondary checksum also mismatches, but the content-sha256 + // error is checked first and takes precedence. + const checksums = { + primary: { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }, + secondary: { algorithm: 'crc32', isTrailer: false, expected: 'AAAAAA==' }, + }; + dataStore({}, null, request, 0, null, {}, checksums, log, err => { + assert.strictEqual(err.message, 'XAmzContentSHA256Mismatch'); + assert(batchDeleteStub.calledOnce); + done(); + }); + }); + + it('should call cb with XAmzContentSHA256Mismatch when the hash mismatches and batchDelete also fails', + done => { + batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.InternalError)); + putSucceeds(); + const request = makeStream( + { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, 'hello world'); + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { + assert.strictEqual(err.message, 'XAmzContentSHA256Mismatch'); + done(); + }); + }); + }); + describe('dual-checksum behaviour', () => { it('should return client-facing checksum from secondary and storageChecksum from primary', done => { putSucceeds(); diff --git a/tests/unit/api/apiUtils/validateChecksumHeaders.js b/tests/unit/api/apiUtils/validateChecksumHeaders.js deleted file mode 100644 index 6b1f7dbbf6..0000000000 --- a/tests/unit/api/apiUtils/validateChecksumHeaders.js +++ /dev/null @@ -1,75 +0,0 @@ -const assert = require('assert'); - -const validateChecksumHeaders = require('../../../../lib/api/apiUtils/object/validateChecksumHeaders'); -const { unsupportedSignatureChecksums, supportedSignatureChecksums } = require('../../../../constants'); - -const passingCases = [ - { - description: 'should return null if no checksum headers are present', - headers: {}, - }, - { - description: 'should return null if UNSIGNED-PAYLOAD is used', - headers: { - 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - }, - }, - { - description: 'should return null if a sha256 checksum is used', - headers: { - 'x-amz-content-sha256': 'thisIs64CharactersLongAndThatsAllWeCheckFor1234567890abcdefghijk', - }, - }, -]; - -supportedSignatureChecksums.forEach(checksum => { - passingCases.push({ - description: `should return null if ${checksum} is used`, - headers: { - 'x-amz-content-sha256': checksum, - }, - }); -}); - -const failingCases = [ - { - description: 'should return BadRequest if a trailing checksum is used', - headers: { - 'x-amz-trailer': 'test', - }, - }, - { - description: 'should return BadRequest if an unknown algo is used', - headers: { - 'x-amz-content-sha256': 'UNSUPPORTED-CHECKSUM', - }, - }, -]; - -unsupportedSignatureChecksums.forEach(checksum => { - failingCases.push({ - description: `should return BadRequest if ${checksum} is used`, - headers: { - 'x-amz-content-sha256': checksum, - }, - }); -}); - - -describe('validateChecksumHeaders', () => { - passingCases.forEach(testCase => { - it(testCase.description, () => { - const result = validateChecksumHeaders(testCase.headers); - assert.ifError(result); - }); - }); - - failingCases.forEach(testCase => { - it(testCase.description, () => { - const result = validateChecksumHeaders(testCase.headers); - assert(result instanceof Error, 'Expected an error to be returned'); - assert.strictEqual(result.is.BadRequest, true); - assert.strictEqual(result.code, 400); - }); - }); -}); diff --git a/tests/unit/api/apiUtils/validatePayloadProtocol.js b/tests/unit/api/apiUtils/validatePayloadProtocol.js new file mode 100644 index 0000000000..7e72c88dcd --- /dev/null +++ b/tests/unit/api/apiUtils/validatePayloadProtocol.js @@ -0,0 +1,45 @@ +const assert = require('assert'); + +const validatePayloadProtocol = require('../../../../lib/api/apiUtils/object/validatePayloadProtocol'); +const { unsupportedSignatureChecksums, supportedSignatureChecksums } = require('../../../../constants'); + +// validatePayloadProtocol only validates x-amz-content-sha256 for SigV4 +// header-authenticated requests (Authorization: "AWS4-..."). +const sigV4 = 'AWS4-HMAC-SHA256 Credential=AK/20260101/us-east-1/s3/aws4_request, ' + + 'SignedHeaders=host, Signature=abc'; +const sigV2 = 'AWS AKID:signature'; +const validSha256Hex = 'a'.repeat(64); + +// build SigV4 header-auth headers, merging any extras +const v4 = extra => Object.assign({ authorization: sigV4 }, extra); + +describe('validatePayloadProtocol', () => { + it('should return null for a valid hex sha256', () => + assert.ifError(validatePayloadProtocol(v4({ 'x-amz-content-sha256': validSha256Hex })))); + + supportedSignatureChecksums.forEach(protocol => { + it(`should return null for supported protocol ${protocol}`, () => + assert.ifError(validatePayloadProtocol(v4({ 'x-amz-content-sha256': protocol })))); + }); + + it('should return null for non-SigV4 header auth, even with an invalid value', () => + assert.ifError(validatePayloadProtocol({ authorization: sigV2, 'x-amz-content-sha256': 'BAD' }))); + + unsupportedSignatureChecksums.forEach(protocol => { + it(`should return BadRequest for unsupported protocol ${protocol}`, () => { + const err = validatePayloadProtocol(v4({ 'x-amz-content-sha256': protocol })); + assert(err instanceof Error, 'expected an error'); + assert.strictEqual(err.message, 'BadRequest'); + assert.strictEqual(err.code, 400); + assert.match(err.description, /is not supported/); + }); + }); + + it('should return InvalidArgument for a non-hex x-amz-content-sha256', () => { + const err = validatePayloadProtocol(v4({ 'x-amz-content-sha256': 'BAD' })); + assert(err instanceof Error, 'expected an error'); + assert.strictEqual(err.message, 'InvalidArgument'); + assert.strictEqual(err.code, 400); + assert.match(err.description, /x-amz-content-sha256 must be/); + }); +}); diff --git a/tests/unit/auth/ContentSHA256Transform.js b/tests/unit/auth/ContentSHA256Transform.js new file mode 100644 index 0000000000..a4f17f18e0 --- /dev/null +++ b/tests/unit/auth/ContentSHA256Transform.js @@ -0,0 +1,107 @@ +const assert = require('assert'); +const crypto = require('crypto'); + +const { ChecksumError } = require('../../../lib/api/apiUtils/integrity/validateChecksums'); +const ContentSHA256Transform = require('../../../lib/auth/streamingV4/ContentSHA256Transform'); +const { DummyRequestLogger } = require('../helpers'); + +const log = new DummyRequestLogger(); +const testData = Buffer.from('hello world'); +const testDigest = crypto.createHash('sha256').update(testData).digest('hex'); +const emptyDigest = crypto.createHash('sha256').update(Buffer.alloc(0)).digest('hex'); +const wrongDigest = 'a'.repeat(64); + +// Pipe chunks through the transform, collect output, resolve on end. +function runTransform(stream, chunks) { + return new Promise((resolve, reject) => { + const output = []; + stream.on('data', chunk => output.push(chunk)); + stream.on('end', () => resolve(Buffer.concat(output))); + stream.on('error', reject); + for (const chunk of chunks) { + stream.write(chunk); + } + stream.end(); + }); +} + +// Drain the transform without collecting output, resolve on finish. +function drainTransform(stream, chunks) { + return new Promise((resolve, reject) => { + stream.resume(); + stream.on('finish', resolve); + stream.on('error', reject); + for (const chunk of chunks) { + stream.write(chunk); + } + stream.end(); + }); +} + +describe('ContentSHA256Transform', () => { + describe('pass-through and digest', () => { + it('should pass data through unchanged', async () => { + const stream = new ContentSHA256Transform(testDigest, log); + const output = await runTransform(stream, [testData]); + assert.deepStrictEqual(output, testData); + }); + + it('should compute the sha256 hex digest after the stream ends', async () => { + const stream = new ContentSHA256Transform(testDigest, log); + await drainTransform(stream, [testData]); + assert.strictEqual(stream.digest, testDigest); + }); + + it('should compute the same digest for multi-chunk input', async () => { + const half = Math.floor(testData.length / 2); + const stream = new ContentSHA256Transform(testDigest, log); + await drainTransform(stream, [testData.subarray(0, half), testData.subarray(half)]); + assert.strictEqual(stream.digest, testDigest); + }); + + it('should handle Buffer and string chunks equally', async () => { + const streamBuf = new ContentSHA256Transform(testDigest, log); + const streamStr = new ContentSHA256Transform(testDigest, log); + await drainTransform(streamBuf, [testData]); + await drainTransform(streamStr, [testData.toString()]); + assert.strictEqual(streamBuf.digest, streamStr.digest); + }); + + it('should compute the sha256 of an empty body', async () => { + const stream = new ContentSHA256Transform(emptyDigest, log); + await drainTransform(stream, []); + assert.strictEqual(stream.digest, emptyDigest); + }); + }); + + describe('validateChecksum', () => { + it('should return null when the expected digest matches the body', async () => { + const stream = new ContentSHA256Transform(testDigest, log); + await drainTransform(stream, [testData]); + assert.strictEqual(stream.validateChecksum(), null); + }); + + it('should normalize an uppercase expected digest before comparing', async () => { + const stream = new ContentSHA256Transform(testDigest.toUpperCase(), log); + await drainTransform(stream, [testData]); + assert.strictEqual(stream.validateChecksum(), null); + }); + + it('should return ContentSHA256Mismatch with calculated/expected details on mismatch', async () => { + const stream = new ContentSHA256Transform(wrongDigest, log); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.ContentSHA256Mismatch); + assert.strictEqual(result.details.calculated, testDigest); + assert.strictEqual(result.details.expected, wrongDigest); + }); + + it('should return ContentSHA256Mismatch for an empty body when a non-empty digest is expected', async () => { + const stream = new ContentSHA256Transform(testDigest, log); + await drainTransform(stream, []); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.ContentSHA256Mismatch); + assert.strictEqual(result.details.calculated, emptyDigest); + }); + }); +}); diff --git a/yarn.lock b/yarn.lock index 93e5c518f2..8ef9d5c54b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6595,9 +6595,9 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/arsenal#8.5.0": - version "8.5.0" - resolved "git+https://github.com/scality/arsenal#016072dac337eb64d602f40588cd6c22cf185625" +"arsenal@git+https://github.com/scality/arsenal#improvement/ARSN-602-add-XAmzContentSHA256Mismatch-error": + version "8.5.2" + resolved "git+https://github.com/scality/arsenal#0493faa6579763579f33835ec4899fe488b9ec09" dependencies: "@aws-sdk/client-kms" "^3.975.0" "@aws-sdk/client-s3" "^3.975.0" @@ -6607,6 +6607,10 @@ arraybuffer.prototype.slice@^1.0.4: "@azure/storage-blob" "^12.31.0" "@js-sdsl/ordered-set" "^4.4.2" "@opentelemetry/api" "^1.9.1" + "@opentelemetry/exporter-trace-otlp-http" "^0.219.0" + "@opentelemetry/resources" "^2.8.0" + "@opentelemetry/sdk-node" "^0.219.0" + "@opentelemetry/sdk-trace-base" "^2.8.0" "@scality/hdclient" "^1.3.2" "@smithy/node-http-handler" "^4.3.0" "@smithy/protocol-http" "^5.3.5" @@ -6635,13 +6639,9 @@ arraybuffer.prototype.slice@^1.0.4: sproxydclient "github:scality/sproxydclient#8.1.0" utf8 "^3.0.0" uuid "^10.0.0" - werelogs scality/werelogs#8.2.2 + werelogs scality/werelogs#8.2.4 xml2js "^0.6.2" optionalDependencies: - "@opentelemetry/exporter-trace-otlp-http" "^0.219.0" - "@opentelemetry/resources" "^2.8.0" - "@opentelemetry/sdk-node" "^0.219.0" - "@opentelemetry/sdk-trace-base" "^2.8.0" ioctl "^2.0.2" asn1@~0.2.3: @@ -12844,7 +12844,6 @@ webidl-conversions@^7.0.0: "werelogs@github:scality/werelogs#8.2.2", werelogs@scality/werelogs#8.2.2: version "8.2.2" - uid e53bef5145697bf8af940dcbe59408988d64854f resolved "https://codeload.github.com/scality/werelogs/tar.gz/e53bef5145697bf8af940dcbe59408988d64854f" dependencies: fast-safe-stringify "^2.1.1" @@ -12857,6 +12856,13 @@ werelogs@scality/werelogs#8.2.0: fast-safe-stringify "^2.1.1" safe-json-stringify "^1.2.0" +werelogs@scality/werelogs#8.2.4: + version "8.2.4" + resolved "https://codeload.github.com/scality/werelogs/tar.gz/a7bbb5917a08b035d3763b24b070d517483d6982" + dependencies: + fast-safe-stringify "^2.1.1" + safe-json-stringify "^1.2.0" + whatwg-mimetype@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf"