2626
2727import asyncio
2828import base64
29+ import codecs
2930import logging
3031import os
3132import re
3940
4041from dataclasses import dataclass
4142from pathlib import Path , PurePosixPath
42- from typing import Iterable , Optional , Sequence
43+ from typing import Callable , Iterable , Optional , Sequence
4344
4445import edgtest
4546import edgutil
@@ -81,6 +82,10 @@ class UnsafeArchiveError(WorkspaceError):
8182 '''Raised for an artifact that is too large or has unsafe entries.'''
8283
8384
85+ class _OutputTooLarge (Exception ):
86+ '''Raised when a process's output is over its limit (it is stopped).'''
87+
88+
8489@dataclass (frozen = True )
8590class GitResult :
8691 returncode : int
@@ -140,16 +145,18 @@ def changed_paths(diff: str) -> list[str]:
140145 return paths
141146
142147
143- def changes_patterns (diff : str ,
148+ def changes_patterns (lines_iter : Iterable [ str ] ,
144149 * ,
145150 max_patterns : int = DEFAULT_MAX_PATTERNS ) -> list [str ]:
146- '''Sparse-checkout patterns for rematerializing ``diff`` (of tests/) on
147- the base tree: the test home and every affected test; at most
151+ '''Sparse-checkout patterns for rematerializing the diff (of tests/) of
152+ ``lines_iter`` (with or without their line ends, e.g. a file's) on the
153+ base tree: the test home and every affected test; at most
148154 ``max_patterns``.
149155 '''
150156 patterns = dict .fromkeys (_TEST_HOME_PATTERNS )
151157 current_suite : Optional [str ] = None
152- for line in diff .splitlines ():
158+ for line in lines_iter :
159+ line = line .rstrip ('\n ' )
153160 match = _DIFF_HEADER .match (line )
154161 if match is not None :
155162 path = PurePosixPath (match .group (1 ))
@@ -478,9 +485,13 @@ async def git(self,
478485 cwd : Optional [Path ] = None ,
479486 stdin : Optional [str ] = None ,
480487 check : bool = True ,
481- max_output : Optional [int ] = None ) -> GitResult :
482- '''Run git; with ``max_output``, its output may be at most that many
483- bytes (it is stopped, and ``WorkspaceError`` raised, beyond).
488+ max_output : Optional [int ] = None ,
489+ stdout_sink : Optional [Callable [[bytes ], None ]] = None
490+ ) -> GitResult :
491+ '''Run git; with ``stdout_sink``, its output goes there as it comes,
492+ rather than into the result, and may be at most ``max_output`` bytes
493+ (it is stopped, and ``_OutputTooLarge`` raised, beyond: the caller
494+ knows which limit that is).
484495 '''
485496 command = ['git' ]
486497 if git_dir is not None :
@@ -497,18 +508,15 @@ async def git(self,
497508 cwd = cwd ,
498509 env = self ._git_env ()
499510 )
500- if max_output is None :
511+ if stdout_sink is None :
512+ assert max_output is None
501513 stdout , stderr = await process .communicate (
502514 stdin .encode () if stdin is not None else None
503515 )
504516 else :
505- assert stdin is None
506- bounded = await _communicate_bounded (process , max_output )
507- if bounded is None :
508- raise WorkspaceError (
509- f"git { args [0 ]} 's output is over { max_output } bytes"
510- )
511- stdout , stderr = bounded
517+ assert stdin is None and max_output is not None
518+ stderr = await _communicate_bounded (process , max_output , stdout_sink )
519+ stdout = b''
512520 assert process .returncode is not None
513521 result = GitResult (
514522 process .returncode ,
@@ -609,15 +617,42 @@ async def diff(self,
609617 repository : str ,
610618 base : str ,
611619 head : str ,
612- paths : Sequence [str ]) -> str :
620+ paths : Sequence [str ],
621+ destination : Path ) -> None :
622+ '''Write the diff of ``paths`` from ``base`` to ``head`` to (the new
623+ file) ``destination``, as it comes (it may be large): UTF-8, with what
624+ isn't replaced (U+FFFD).
625+ '''
626+ decoder = codecs .getincrementaldecoder ('utf-8' )(errors = 'replace' )
613627 async with self .lock (repository ):
614- result = await self .git (
615- 'diff' , '--no-renames' , '--no-color' , '--no-ext-diff' , '--no-textconv' ,
616- base , head , '--' , * paths ,
617- git_dir = self .clone_dir (repository ),
618- max_output = self ._max_diff_bytes
619- )
620- return result .stdout
628+ with open (destination , 'xb' ) as handle :
629+
630+ # Decoded and encoded again so that what isn't UTF-8 is replaced:
631+ # the builder's edg-rematerialize-test-run (and changes_patterns)
632+ # read it as UTF-8 text, and the Changes section's fingerprint
633+ # (the file's) stays what it was when the diff was decoded whole.
634+ # Incrementally, so a character split between chunks survives.
635+
636+ def write (chunk : bytes ) -> None :
637+ handle .write (decoder .decode (chunk ).encode ())
638+
639+ try :
640+ await self .git (
641+ 'diff' , '--no-renames' , '--no-color' , '--no-ext-diff' ,
642+ '--no-textconv' , base , head , '--' , * paths ,
643+ git_dir = self .clone_dir (repository ),
644+ max_output = self ._max_diff_bytes ,
645+ stdout_sink = write
646+ )
647+ except _OutputTooLarge :
648+ raise WorkspaceError (
649+ f"the diff is over { self ._max_diff_bytes } bytes "
650+ f"({ ENV_PREFIX } MAX_DIFF_BYTES)"
651+ ) from None
652+
653+ # Finish decoding: bytes the decoder still holds (a character the
654+ # output ended in the middle of) are written too, replaced.
655+ handle .write (decoder .decode (b'' , final = True ).encode ())
621656
622657 async def checkout (self ,
623658 info : PullRequestInfo ,
@@ -702,31 +737,31 @@ async def collect_garbage(self, repository: str, open_count: int) -> None:
702737
703738
704739async def _communicate_bounded (process : asyncio .subprocess .Process ,
705- limit : int ) -> Optional [tuple [bytes , bytes ]]:
706- '''``process``'s output (stdout, stderr), once it exits; ``None``, having
707- stopped it, should its stdout be over ``limit`` bytes. Should this be
708- cancelled (or fail), it is stopped too, rather than left writing to a
709- pipe no one reads.
740+ limit : int ,
741+ sink : Callable [[bytes ], None ]) -> bytes :
742+ '''Pass ``process``'s stdout to ``sink`` as it comes, in chunks (none held
743+ here); return its stderr once it exits. Should its stdout be over
744+ ``limit`` bytes, it is stopped, and ``_OutputTooLarge`` raised. Should
745+ this be cancelled (or fail), it is stopped too, rather than left writing
746+ to a pipe no one reads.
710747 '''
711748 assert process .stdout is not None and process .stderr is not None
712- errors = asyncio .create_task (process .stderr .read ())
713- chunks : list [bytes ] = []
749+ reading_stderr = asyncio .create_task (process .stderr .read ())
714750 size = 0
715751 try :
716752 while chunk := await process .stdout .read (64 * 1024 ):
717753 size += len (chunk )
718754 if size > limit :
719- await edgutil .async_terminate_within (process , _TERMINATE_ALLOWANCE )
720- return None
721- chunks .append (chunk )
722- output = b'' .join (chunks ), await errors
755+ raise _OutputTooLarge ()
756+ sink (chunk )
757+ stderr = await reading_stderr
723758 await process .wait ()
724- return output
759+ return stderr
725760 except BaseException :
726761 await edgutil .async_terminate_within (process , _TERMINATE_ALLOWANCE )
727762 raise
728763 finally :
729- errors .cancel ()
764+ reading_stderr .cancel ()
730765
731766
732767def basic_auth_header (user : str , password : str ) -> str :
0 commit comments