CLDSRV-932: validate x-amz-content-sha256

lib/api/apiUtils/integrity/validateChecksums.js

@@ -5,6 +5,7 @@ const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt');
 const { errors: ArsenalErrors, errorInstances } = require('arsenal');
 const { config } = require('../../../Config');
 const { combinePartCrcs } = require('./crcCombine');
+const { supportedSignatureChecksums, unsupportedSignatureChecksums } = require('../../../../constants');
 
 const defaultChecksumData = Object.freeze({ algorithm: 'crc64nvme', isTrailer: false, expected: undefined });
 
@@ -45,6 +46,21 @@ const errCopyChecksumAlgoNotSupported = errorInstances.InvalidRequest.customizeD
         '[CRC32, CRC32C, CRC64NVME, SHA1, SHA256]',
 );
 
+const errContentSHA256Mismatch = errorInstances.XAmzContentSHA256Mismatch;
+
+const errMissingContentSHA256 = errorInstances.InvalidRequest.customizeDescription(
+    'Missing required header for this request: x-amz-content-sha256',
+);
+
+const errInvalidContentSHA256 = errorInstances.InvalidArgument.customizeDescription(
+    'x-amz-content-sha256 must be UNSIGNED-PAYLOAD, STREAMING-UNSIGNED-PAYLOAD-TRAILER, ' +
+        'STREAMING-AWS4-HMAC-SHA256-PAYLOAD, STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER, ' +
+        'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD, STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER ' +
+        'or a valid sha256 value.',
+);
+
+const sha256HexRegex = /^[0-9a-f]{64}$/i;
+
 // Methods that validate BOTH Content-MD5 and x-amz-checksum-* against the
 // buffered request body. For these, x-amz-checksum-* is the body digest.
 const checksumedMethods = Object.freeze({
@@ -98,6 +114,9 @@ const ChecksumError = Object.freeze({
     MPUTypeWithoutAlgo: 'MPUTypeWithoutAlgo',
     MPUInvalidCombination: 'MPUInvalidCombination',
     CopyChecksumAlgoNotSupported: 'CopyChecksumAlgoNotSupported',
+    ContentSHA256Missing: 'ContentSHA256Missing',
+    ContentSHA256Invalid: 'ContentSHA256Invalid',
+    ContentSHA256Mismatch: 'ContentSHA256Mismatch',
 });
 
 const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/;
@@ -368,6 +387,90 @@ function validateContentMd5(headers, body) {
     return null;
 }
 
+// Classification of an x-amz-content-sha256 header value, returned by
+// parseContentSHA256.
+const ContentSHA256Type = Object.freeze({
+    Skip: 0, // not SigV4 header auth; do not enforce
+    Absent: 1, // SigV4 header auth but header missing
+    Unsigned: 2, // UNSIGNED-PAYLOAD
+    Streaming: 3, // a STREAMING-* token
+    HexSHA256: 4, // a hex sha256 of the payload
+    Invalid: 5, // anything else (malformed)
+});
+
+/**
+ * parseContentSHA256 - Classify x-amz-content-sha256 (the SigV4 payload hash)
+ * for a request. The header is only meaningful for SigV4 header-authenticated
+ * requests, so anything else (SigV2, presigned/query, anonymous) returns
+ * { type: ContentSHA256Type.Skip } and is not enforced. Otherwise the header
+ * value is classified.
+ *
+ * @param {object} headers - http request headers
+ * @return {object} { type, value } where type is a ContentSHA256Type and value
+ *   is the raw header value (null when not sent). Streaming results also carry
+ *   a { supported } boolean.
+ */
+function parseContentSHA256(headers) {
+    const value = headers['x-amz-content-sha256'] ?? null;
+    const authHeader = headers.authorization;
+    if (typeof authHeader !== 'string' || !authHeader.startsWith('AWS4')) {
+        return { type: ContentSHA256Type.Skip, value };
+    }
+    if (value === null) {
+        return { type: ContentSHA256Type.Absent, value };
+    }
+    if (value === 'UNSIGNED-PAYLOAD') {
+        return { type: ContentSHA256Type.Unsigned, value };
+    }
+    if (supportedSignatureChecksums.has(value)) {
+        return { type: ContentSHA256Type.Streaming, value, supported: true };
+    }
+    if (unsupportedSignatureChecksums.has(value)) {
+        return { type: ContentSHA256Type.Streaming, value, supported: false };
+    }
+    if (sha256HexRegex.test(value)) {
+        return { type: ContentSHA256Type.HexSHA256, value };
+    }
+    return { type: ContentSHA256Type.Invalid, value };
+}
+
+/**
+ * validateXAmzContentSHA256 - Validate the SHA256 of a SigV4 request.
+ *
+ * @param {object} headers - http request headers
+ * @param {Buffer} body - buffered http request body
+ * @return {object|null} - { error: ChecksumError } on missing/invalid/mismatch,
+ *   else null
+ */
+function validateXAmzContentSHA256(headers, body) {
+    const parsed = parseContentSHA256(headers);
+    switch (parsed.type) {
+        case ContentSHA256Type.Absent: // required for SigV4 header auth
+            return { error: ChecksumError.ContentSHA256Missing };
+        case ContentSHA256Type.Invalid:
+            return { error: ChecksumError.ContentSHA256Invalid, details: { value: parsed.value } };
+        case ContentSHA256Type.HexSHA256: {
+            const computed = crypto.createHash('sha256').update(body).digest('hex');
+            if (computed !== parsed.value.toLowerCase()) {
+                return {
+                    error: ChecksumError.ContentSHA256Mismatch,
+                    details: { calculated: computed, expected: parsed.value },
+                };
+            }
+            return null;
+        }
+        // Skip (non-SigV4-header auth), Unsigned and Streaming are not a literal
+        // payload hash, so there is nothing to validate here.
+        case ContentSHA256Type.Skip:
+            return null;
+        case ContentSHA256Type.Unsigned:
+            return null;
+        case ContentSHA256Type.Streaming:
+            return null;
+    }
+    return null;
+}
+
 /**
  * validateChecksumsNoChunking - Validate the checksums of a request.
  * @param {object} headers - http headers
@@ -448,6 +551,12 @@ function arsenalErrorFromChecksumError(err) {
             );
         case ChecksumError.CopyChecksumAlgoNotSupported:
             return errCopyChecksumAlgoNotSupported;
+        case ChecksumError.ContentSHA256Missing:
+            return errMissingContentSHA256;
+        case ChecksumError.ContentSHA256Invalid:
+            return errInvalidContentSHA256;
+        case ChecksumError.ContentSHA256Mismatch:
+            return errContentSHA256Mismatch;
         default:
             return ArsenalErrors.BadDigest;
     }
@@ -547,6 +656,15 @@ function md5OnlyValidationFunc(request, body, log) {
  * @return {object} - error
  */
 async function validateMethodChecksumNoChunking(request, body, log) {
+    const contentSHA256Err = validateXAmzContentSHA256(request.headers, body);
+    if (contentSHA256Err) {
+        log.debug('failed x-amz-content-sha256 validation', {
+            method: request.apiMethod,
+            ...contentSHA256Err.details,
+        });
+        return arsenalErrorFromChecksumError(contentSHA256Err);
+    }
+
     if (config.integrityChecks[request.apiMethod] === false) {
         return null;
     }
@@ -738,6 +856,10 @@ module.exports = {
     ChecksumError,
     defaultChecksumData,
     validateChecksumsNoChunking,
+    ContentSHA256Type,
+    parseContentSHA256,
+    errInvalidContentSHA256,
+    validateXAmzContentSHA256,
     validateMethodChecksumNoChunking,
     getChecksumDataFromHeaders,
     arsenalErrorFromChecksumError,

lib/api/apiUtils/object/prepareStream.js

@@ -1,6 +1,8 @@
 const V4Transform = require('../../../auth/streamingV4/V4Transform');
 const TrailingChecksumTransform = require('../../../auth/streamingV4/trailingChecksumTransform');
 const ChecksumTransform = require('../../../auth/streamingV4/ChecksumTransform');
+const ContentSHA256Transform = require('../../../auth/streamingV4/ContentSHA256Transform');
+const { parseContentSHA256, ContentSHA256Type } = require('../integrity/validateChecksums');
 const { errors, errorInstances, jsutil } = require('arsenal');
 const { unsupportedSignatureChecksums } = require('../../../../constants');
 
@@ -103,20 +105,33 @@ function prepareStream(request, streamingV4Params, checksums, log, errCb) {
                 };
             }
 
-            const onStreamError = secondary ? jsutil.once(errCb) : errCb;
+            const parsedContentSHA256 = parseContentSHA256(request.headers);
+            const shouldValidateContentSHA256 = parsedContentSHA256.type === ContentSHA256Type.HexSHA256;
+            const onStreamError = (secondary || shouldValidateContentSHA256) ? jsutil.once(errCb) : errCb;
+            let contentSHA256Stream = null;
             let secondaryChecksumStream = null;
             let stream = request;
+            if (shouldValidateContentSHA256) {
+                contentSHA256Stream = new ContentSHA256Transform(parsedContentSHA256.value, log);
+                contentSHA256Stream.on('error', onStreamError);
+                stream = stream.pipe(contentSHA256Stream);
+            }
             if (secondary) {
                 secondaryChecksumStream = new ChecksumTransform(
                     secondary.algorithm, secondary.expected,
                     secondary.isTrailer, log);
                 secondaryChecksumStream.on('error', onStreamError);
-                stream = request.pipe(secondaryChecksumStream);
+                stream = stream.pipe(secondaryChecksumStream);
             }
 
             const primaryStream = new ChecksumTransform(primary.algorithm, primary.expected, primary.isTrailer, log);
             primaryStream.on('error', onStreamError);
-            return { error: null, stream: stream.pipe(primaryStream), secondaryChecksumStream };
+            return {
+                error: null,
+                stream: stream.pipe(primaryStream),
+                secondaryChecksumStream,
+                contentSHA256Stream,
+            };
         }
     }
 }

lib/api/apiUtils/object/storeObject.js

@@ -100,10 +100,24 @@ function dataStore(objectContext, cipherBundle, stream, size, streamingV4Params,
             };
 
             // stream is always the primary (end of pipe, stored checksum).
-            // secondaryChecksumStream is upstream and only validated.
-            const { secondaryChecksumStream } = checksumedStream;
+            // secondaryChecksumStream and contentSHA256Stream are upstream and
+            // only validated.
+            const { secondaryChecksumStream, contentSHA256Stream } = checksumedStream;
 
             const doValidate = () => {
+                // Validate the SigV4 payload hash (x-amz-content-sha256) first.
+                if (contentSHA256Stream) {
+                    const contentErr = contentSHA256Stream.validateChecksum();
+                    if (contentErr) {
+                        log.debug('failed x-amz-content-sha256 validation', { error: contentErr });
+                        return data.batchDelete([dataRetrievalInfo], null, null, log, deleteErr => {
+                            if (deleteErr) {
+                                log.error('dataStore failed to delete old data', { error: deleteErr });
+                            }
+                            return cbOnce(arsenalErrorFromChecksumError(contentErr));
+                        });
+                    }
+                }
                 // Validate the secondary (checked-only) checksum first.
                 if (secondaryChecksumStream) {
                     const secondaryErr = secondaryChecksumStream.validateChecksum();

lib/api/apiUtils/object/validatePayloadProtocol.js

@@ -0,0 +1,32 @@
+const { errorInstances } = require('arsenal');
+
+const { parseContentSHA256, ContentSHA256Type, errInvalidContentSHA256 } = require('../integrity/validateChecksums');
+
+/**
+ * Validate the SigV4 payload protocol selected by x-amz-content-sha256.
+ *
+ * @param {object} headers - http request headers
+ * @return {ArsenalError|null} - error if the protocol is unsupported/malformed, else null
+ */
+function validatePayloadProtocol(headers) {
+    const parsed = parseContentSHA256(headers);
+    switch (parsed.type) {
+        case ContentSHA256Type.Skip: // not SigV4 header auth; header not meaningful
+            return null;
+        case ContentSHA256Type.Absent:
+            return null;
+        case ContentSHA256Type.Unsigned:
+            return null;
+        case ContentSHA256Type.HexSHA256:
+            return null;
+        case ContentSHA256Type.Streaming:
+            return parsed.supported
+                ? null
+                : errorInstances.BadRequest.customizeDescription(`${parsed.value} is not supported`);
+        case ContentSHA256Type.Invalid:
+            return errInvalidContentSHA256;
+    }
+    return null;
+}
+
+module.exports = validatePayloadProtocol;

lib/api/objectPut.js

@@ -18,7 +18,7 @@ const monitoring = require('../utilities/monitoringHandler');
 const { validatePutVersionId } = require('./apiUtils/object/coldStorage');
 const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders');
 const { setSSEHeaders } = require('./apiUtils/object/sseHeaders');
-const validateChecksumHeaders = require('./apiUtils/object/validateChecksumHeaders');
+const validatePayloadProtocol = require('./apiUtils/object/validatePayloadProtocol');
 
 const writeContinue = require('../utilities/writeContinue');
 const { overheadField } = require('../../constants');
@@ -119,9 +119,9 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
         ));
     }
 
-    const checksumHeaderErr = validateChecksumHeaders(headers);
-    if (checksumHeaderErr) {
-        return callback(checksumHeaderErr);
+    const payloadProtocolErr = validatePayloadProtocol(headers);
+    if (payloadProtocolErr) {
+        return callback(payloadProtocolErr);
     }
 
     log.trace('owner canonicalID to send to data', { canonicalID });

lib/api/objectPutPart.js

@@ -20,7 +20,7 @@ const { BackendInfo } = models;
 
 const writeContinue = require('../utilities/writeContinue');
 const { parseObjectEncryptionHeaders } = require('./apiUtils/bucket/bucketEncryption');
-const validateChecksumHeaders = require('./apiUtils/object/validateChecksumHeaders');
+const validatePayloadProtocol = require('./apiUtils/object/validatePayloadProtocol');
 const {
     getChecksumDataFromHeaders,
     arsenalErrorFromChecksumError,
@@ -77,9 +77,9 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
         return cb(errors.EntityTooLarge);
     }
 
-    const checksumHeaderErr = validateChecksumHeaders(request.headers);
-    if (checksumHeaderErr) {
-        return cb(checksumHeaderErr);
+    const payloadProtocolErr = validatePayloadProtocol(request.headers);
+    if (payloadProtocolErr) {
+        return cb(payloadProtocolErr);
     }
 
     // Note: Part sizes cannot be less than 5MB in size except for the last.

lib/auth/streamingV4/ContentSHA256Transform.js

@@ -0,0 +1,40 @@
+const crypto = require('crypto');
+const { Transform } = require('stream');
+const { ChecksumError } = require('../../api/apiUtils/integrity/validateChecksums');
+
+/**
+ * Computes the sha256 of the streamed body to verify it against a literal
+ * x-amz-content-sha256 header (the SigV4 payload hash) via validateChecksum().
+ */
+class ContentSHA256Transform extends Transform {
+    constructor(expectedDigest, log) {
+        super({});
+        this.log = log;
+        this.expectedDigest = (expectedDigest || '').toLowerCase();
+        this.hash = crypto.createHash('sha256');
+        this.digest = undefined;
+    }
+
+    validateChecksum() {
+        if (this.digest !== this.expectedDigest) {
+            return {
+                error: ChecksumError.ContentSHA256Mismatch,
+                details: { calculated: this.digest, expected: this.expectedDigest },
+            };
+        }
+        return null;
+    }
+
+    _transform(chunk, encoding, callback) {
+        const input = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+        this.hash.update(input);
+        callback(null, input);
+    }
+
+    _flush(callback) {
+        this.digest = this.hash.digest('hex');
+        callback();
+    }
+}
+
+module.exports = ContentSHA256Transform;