diff --git a/s3proxy/handlers/multipart/copy.py b/s3proxy/handlers/multipart/copy.py
index ef03c0c..c171e69 100644
--- a/s3proxy/handlers/multipart/copy.py
+++ b/s3proxy/handlers/multipart/copy.py
@@ -962,9 +962,13 @@ async def upload_tail() -> InternalPartMetadata | None:
await self.multipart_manager.set_deferred_copy_tail(
bucket, key, upload_id, deferred_tail_bytes
)
+ # A deferred tail is NOT part of this client part's stored bytes: it is
+ # folded into the next part's stream (take_deferred_copy_tail) or flushed
+ # at complete, and its plaintext is counted there. Counting it here too
+ # inflated total_plaintext_size by the tail size, so HEAD reported
+ # size+tail and rclone failed every >copy-cutoff copy with
+ # "corrupted on transfer: sizes differ".
total_plaintext = sum(p.plaintext_size for p in internal_parts)
- if deferred_tail_bytes:
- total_plaintext += len(deferred_tail_bytes)
total_ciphertext = sum(p.ciphertext_size for p in internal_parts)
etag = md5_task.result().hexdigest()
await self.multipart_manager.add_part(
@@ -1022,6 +1026,7 @@ async def _flush_deferred_copy_tail_for_complete(
last_pn = max(state.parts)
last_part = state.parts[last_pn]
last_part.internal_parts.append(ip)
+ last_part.plaintext_size += len(tail)
last_part.ciphertext_size += len(tail_ct)
state.deferred_copy_tail = b""
state.next_internal_part_number = internal_num + 1
diff --git a/s3proxy/handlers/multipart/lifecycle.py b/s3proxy/handlers/multipart/lifecycle.py
index e6763ae..2aab7ab 100644
--- a/s3proxy/handlers/multipart/lifecycle.py
+++ b/s3proxy/handlers/multipart/lifecycle.py
@@ -262,6 +262,18 @@ def _build_s3_parts(
total_plaintext += part_meta.plaintext_size
if part_meta.internal_parts:
+ internal_plaintext = sum(ip.plaintext_size for ip in part_meta.internal_parts)
+ if internal_plaintext != part_meta.plaintext_size:
+ logger.warning(
+ "PART_PLAINTEXT_MISMATCH",
+ bucket=bucket,
+ key=key,
+ upload_id=upload_id[:20] + "...",
+ client_part=client_part_num,
+ part_plaintext=part_meta.plaintext_size,
+ internal_plaintext=internal_plaintext,
+ drift=part_meta.plaintext_size - internal_plaintext,
+ )
sorted_internal = sorted(
part_meta.internal_parts, key=lambda x: x.internal_part_number
)
diff --git a/tests/unit/test_upload_part_copy_passthrough.py b/tests/unit/test_upload_part_copy_passthrough.py
index 50f1f6b..5630828 100644
--- a/tests/unit/test_upload_part_copy_passthrough.py
+++ b/tests/unit/test_upload_part_copy_passthrough.py
@@ -442,10 +442,13 @@ async def test_scylla_prod_shape_range_smaller_than_metadata_uses_passthrough(
updated = await manager.get_upload(BUCKET, "sst/big-Data.db.sm_manifest", upload_id)
part = updated.parts[1]
- assert part.plaintext_size == prod_range_end + 1
- assert len(part.internal_parts) == len(copy_ops)
assert len(updated.deferred_copy_tail) > 0
assert len(updated.deferred_copy_tail) < crypto.MIN_PART_SIZE
+ # The deferred tail is not stored with this part; its plaintext is counted
+ # by whichever part eventually stores it (next part or complete-time flush).
+ assert part.plaintext_size == prod_range_end + 1 - len(updated.deferred_copy_tail)
+ assert part.plaintext_size == sum(ip.plaintext_size for ip in part.internal_parts)
+ assert len(part.internal_parts) == len(copy_ops)
def test_should_defer_hybrid_tail_when_more_client_parts_follow(settings, manager):
@@ -624,6 +627,146 @@ async def test_two_part_hybrid_defer_tail_completes_fast(
await handler.handle_complete_multipart_upload(complete_req, credentials)
+async def _complete_upload(handler, dest_path, upload_id, state, credentials):
+ parts_xml = "".join(
+ f'{pn}"{state.parts[pn].etag}"'
+ for pn in sorted(state.parts)
+ )
+ complete_req = MagicMock()
+ complete_req.url.path = dest_path
+ complete_req.url.query = f"uploadId={upload_id}"
+ complete_req.headers = {}
+ complete_req.body = AsyncMock(
+ return_value=f"{parts_xml}".encode()
+ )
+ await handler.handle_complete_multipart_upload(complete_req, credentials)
+
+
+@pytest.mark.asyncio
+async def test_two_part_defer_tail_total_plaintext_not_double_counted(
+ mock_s3, settings, manager, credentials, monkeypatch
+):
+ """Regression: rclone 2-part copy (>copy-cutoff SSTable) reported size+tail.
+
+ Part 1 defers its sub-5MB tail; part 2's streaming copy folds the tail into
+ its own stream and counts it. Part 1's metadata must therefore NOT count the
+ tail too, or HEAD Content-Length comes out exactly one tail too large and
+ rclone fails the copy with "corrupted on transfer: sizes differ"
+ (398 files x +1,129,664 bytes, backup run sm_20260709080013UTC).
+ """
+ from s3proxy.handlers.multipart import copy as copy_mod
+ from s3proxy.state import load_multipart_metadata
+
+ monkeypatch.setattr(copy_mod, "HYBRID_TAIL_DEFER_MIN_CLIENT_PART", 5 * 1024 * 1024)
+ monkeypatch.setattr(crypto, "COPY_INTERNAL_PART_SIZE", crypto.MIN_PART_SIZE)
+ handler = _handler(settings, mock_s3, credentials)
+ await mock_s3.create_bucket(BUCKET)
+
+ kid, kek = settings.keyring.key_for(credentials.access_key)
+ chunk = 8 * 1024 * 1024
+ src_dek, src_plaintext, ciphertext_blob, src_meta, total, _ = _build_scylla_prod_shape_source(
+ kid, kek, num_internal_parts=12, metadata_inflate_ratio=1.0, chunk_size=chunk
+ )
+ assert total == len(src_plaintext)
+ part1_range_end = chunk * 4 + chunk // 2 # ends mid-frame -> ~4MB deferred tail
+
+ await mock_s3.put_object(BUCKET, "sst/big-Data.db", ciphertext_blob)
+ await save_multipart_metadata(mock_s3, BUCKET, "sst/big-Data.db", src_meta)
+
+ dest_key = "sst/big-Data.db.sm_20260709080013UTC"
+ resp_create = await mock_s3.create_multipart_upload(BUCKET, dest_key)
+ upload_id = resp_create["UploadId"]
+ await handler.multipart_manager.create_upload(BUCKET, dest_key, upload_id, src_dek, kid)
+
+ for part_number, rng in (
+ (1, f"bytes=0-{part1_range_end}"),
+ (2, f"bytes={part1_range_end + 1}-{total - 1}"),
+ ):
+ resp = await handler.handle_upload_part_copy(
+ _copy_part_request(
+ f"/{BUCKET}/{dest_key}",
+ f"/{BUCKET}/sst/big-Data.db",
+ upload_id,
+ part_number=part_number,
+ copy_source_range=rng,
+ ),
+ credentials,
+ )
+ await _read(resp)
+
+ state = await handler.multipart_manager.get_upload(BUCKET, dest_key, upload_id)
+ for pn, part in state.parts.items():
+ assert part.plaintext_size == sum(ip.plaintext_size for ip in part.internal_parts), (
+ f"client part {pn} plaintext must match its stored internal parts"
+ )
+ assert sum(p.plaintext_size for p in state.parts.values()) == total
+
+ await _complete_upload(handler, f"/{BUCKET}/{dest_key}", upload_id, state, credentials)
+
+ dest_meta = await load_multipart_metadata(mock_s3, BUCKET, dest_key)
+ assert dest_meta.total_plaintext_size == total, (
+ "HEAD Content-Length source: must equal the source size, not size+tail"
+ )
+
+ resp = await handler.handle_get_object(_get_request(f"/{BUCKET}/{dest_key}"), credentials)
+ assert await _read(resp) == src_plaintext
+
+
+@pytest.mark.asyncio
+async def test_defer_tail_flushed_on_complete_keeps_part_accounting(
+ mock_s3, settings, manager, credentials, monkeypatch
+):
+ """A tail flushed at complete (no part 2 consumed it) lands in part accounting once."""
+ from s3proxy.handlers.multipart import copy as copy_mod
+ from s3proxy.state import load_multipart_metadata
+
+ monkeypatch.setattr(copy_mod, "HYBRID_TAIL_DEFER_MIN_CLIENT_PART", 5 * 1024 * 1024)
+ handler = _handler(settings, mock_s3, credentials)
+ await mock_s3.create_bucket(BUCKET)
+
+ kid, kek = settings.keyring.key_for(credentials.access_key)
+ chunk = 8 * 1024 * 1024
+ src_dek, src_plaintext, ciphertext_blob, src_meta, total, _ = _build_scylla_prod_shape_source(
+ kid, kek, num_internal_parts=12, metadata_inflate_ratio=1.0, chunk_size=chunk
+ )
+ part1_range_end = chunk * 4 + chunk // 2
+ part1_span = part1_range_end + 1
+
+ await mock_s3.put_object(BUCKET, "sst/solo-Data.db", ciphertext_blob)
+ await save_multipart_metadata(mock_s3, BUCKET, "sst/solo-Data.db", src_meta)
+
+ dest_key = "sst/solo-Data.db.copy"
+ resp_create = await mock_s3.create_multipart_upload(BUCKET, dest_key)
+ upload_id = resp_create["UploadId"]
+ await handler.multipart_manager.create_upload(BUCKET, dest_key, upload_id, src_dek, kid)
+
+ resp = await handler.handle_upload_part_copy(
+ _copy_part_request(
+ f"/{BUCKET}/{dest_key}",
+ f"/{BUCKET}/sst/solo-Data.db",
+ upload_id,
+ part_number=1,
+ copy_source_range=f"bytes=0-{part1_range_end}",
+ ),
+ credentials,
+ )
+ await _read(resp)
+
+ state = await handler.multipart_manager.get_upload(BUCKET, dest_key, upload_id)
+ assert state.deferred_copy_tail
+
+ await _complete_upload(handler, f"/{BUCKET}/{dest_key}", upload_id, state, credentials)
+
+ dest_meta = await load_multipart_metadata(mock_s3, BUCKET, dest_key)
+ assert dest_meta.total_plaintext_size == part1_span
+ part = dest_meta.parts[0]
+ assert part.plaintext_size == part1_span
+ assert part.plaintext_size == sum(ip.plaintext_size for ip in part.internal_parts)
+
+ resp = await handler.handle_get_object(_get_request(f"/{BUCKET}/{dest_key}"), credentials)
+ assert await _read(resp) == src_plaintext[:part1_span]
+
+
def test_prod_shape_passthrough_eligibility_and_route(settings, manager, credentials):
"""Unit check for prod mismatch: metadata total > range, mid-frame tail still eligible."""
handler = _copy_handler(settings, manager)