Skip to content

builtins: Audit bytes arguments #7631

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
srittau merged 3 commits into from
Apr 16, 2022
Merged

builtins: Audit bytes arguments #7631

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8df4803
builtins: Audit bytes arguments
JelleZijlstra Apr 16, 2022
446691f
Update stdlib/builtins.pyi
JelleZijlstra Apr 16, 2022
a7b6ab8
type ignore
JelleZijlstra Apr 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 74 additions & 61 deletions stdlib/builtins.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ _NegativeInteger = Literal[-1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -1

class int:
@overload
def __new__(cls: type[Self], __x: str | bytes | SupportsInt | SupportsIndex | SupportsTrunc = ...) -> Self: ...
def __new__(cls: type[Self], __x: str | ReadableBuffer | SupportsInt | SupportsIndex | SupportsTrunc = ...) -> Self: ...
@overload
def __new__(cls: type[Self], __x: str | bytes | bytearray, base: SupportsIndex) -> Self: ...
Copy link
Member Author

@JelleZijlstra JelleZijlstra Apr 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

>>> int(memoryview(b"0xdeadbeef"), 16)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: int() can't convert non-string with explicit base
>>> int(memoryview(b"123"))
123

Showing that the first overload accepts buffers but the second doesn't.

Copy link
Collaborator

@hauntsaninja hauntsaninja Apr 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://github.com/bradfitz/deadbeef

if sys.version_info >= (3, 8):
Expand All @@ -223,7 +223,7 @@ class int:
@classmethod
def from_bytes(
cls: type[Self],
bytes: Iterable[SupportsIndex] | SupportsBytes, # TODO buffer object argument
bytes: Iterable[SupportsIndex] | SupportsBytes | ReadableBuffer,
Copy link
Member Author

@JelleZijlstra JelleZijlstra Apr 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

>>> int.from_bytes([1, 2, 3])
66051
>>> int.from_bytes(memoryview(b"123"))
3224115

byteorder: Literal["little", "big"],
*,
signed: bool = ...,
Expand Down Expand Up @@ -289,7 +289,7 @@ class int:
def __index__(self) -> int: ...

class float:
def __new__(cls: type[Self], x: SupportsFloat | SupportsIndex | str | bytes | bytearray = ...) -> Self: ...
def __new__(cls: type[Self], x: SupportsFloat | SupportsIndex | str | ReadableBuffer = ...) -> Self: ...
def as_integer_ratio(self) -> tuple[int, int]: ...
def hex(self) -> str: ...
def is_integer(self) -> bool: ...
Expand Down Expand Up @@ -383,7 +383,7 @@ class str(Sequence[str]):
@overload
def __new__(cls: type[Self], object: object = ...) -> Self: ...
@overload
def __new__(cls: type[Self], object: bytes, encoding: str = ..., errors: str = ...) -> Self: ...
def __new__(cls: type[Self], object: ReadableBuffer, encoding: str = ..., errors: str = ...) -> Self: ...
def capitalize(self) -> str: ...
def casefold(self) -> str: ...
def center(self, __width: SupportsIndex, __fillchar: str = ...) -> str: ...
Expand Down Expand Up @@ -480,27 +480,30 @@ class bytes(ByteString):
def capitalize(self) -> bytes: ...
def center(self, __width: SupportsIndex, __fillchar: bytes = ...) -> bytes: ...
def count(
self, __sub: bytes | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
self, __sub: ReadableBuffer | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
) -> int: ...
def decode(self, encoding: str = ..., errors: str = ...) -> str: ...
def endswith(
self, __suffix: bytes | tuple[bytes, ...], __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
self,
__suffix: ReadableBuffer | tuple[ReadableBuffer, ...],
__start: SupportsIndex | None = ...,
__end: SupportsIndex | None = ...,
) -> bool: ...
if sys.version_info >= (3, 8):
def expandtabs(self, tabsize: SupportsIndex = ...) -> bytes: ...
else:
def expandtabs(self, tabsize: int = ...) -> bytes: ...

def find(
self, __sub: bytes | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
self, __sub: ReadableBuffer | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
) -> int: ...
if sys.version_info >= (3, 8):
def hex(self, sep: str | bytes = ..., bytes_per_sep: SupportsIndex = ...) -> str: ...
Copy link
Member Author

@JelleZijlstra JelleZijlstra Apr 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

>>> b"xy".hex(memoryview(b"x"))
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: sep must be str or bytes.
>>> b"xy".hex(bytearray(b"x"))
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: sep must be str or bytes.

else:
def hex(self) -> str: ...

def index(
self, __sub: bytes | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
self, __sub: ReadableBuffer | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
) -> int: ...
def isalnum(self) -> bool: ...
def isalpha(self) -> bool: ...
Expand All @@ -512,49 +515,52 @@ class bytes(ByteString):
def isspace(self) -> bool: ...
def istitle(self) -> bool: ...
def isupper(self) -> bool: ...
def join(self, __iterable_of_bytes: Iterable[ByteString | memoryview]) -> bytes: ...
def ljust(self, __width: SupportsIndex, __fillchar: bytes = ...) -> bytes: ...
def join(self, __iterable_of_bytes: Iterable[ReadableBuffer]) -> bytes: ...
def ljust(self, __width: SupportsIndex, __fillchar: bytes | bytearray = ...) -> bytes: ...
Copy link
Collaborator

@srittau srittau Apr 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unfortunately, this will also accept memoryview at the moment, but having it more explicit can't hurt.

Copy link
Member Author

@JelleZijlstra JelleZijlstra Apr 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's a mypy bug :)

Copy link
Collaborator

@srittau srittau Apr 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's working as documented. In the past when reviewing I've always asked people to remove bytearray from argument types due to that.

def lower(self) -> bytes: ...
def lstrip(self, __bytes: bytes | None = ...) -> bytes: ...
def partition(self, __sep: bytes) -> tuple[bytes, bytes, bytes]: ...
def replace(self, __old: bytes, __new: bytes, __count: SupportsIndex = ...) -> bytes: ...
def lstrip(self, __bytes: ReadableBuffer | None = ...) -> bytes: ...
def partition(self, __sep: ReadableBuffer) -> tuple[bytes, bytes, bytes]: ...
def replace(self, __old: ReadableBuffer, __new: ReadableBuffer, __count: SupportsIndex = ...) -> bytes: ...
if sys.version_info >= (3, 9):
def removeprefix(self, __prefix: bytes) -> bytes: ...
def removesuffix(self, __suffix: bytes) -> bytes: ...
def removeprefix(self, __prefix: ReadableBuffer) -> bytes: ...
def removesuffix(self, __suffix: ReadableBuffer) -> bytes: ...

def rfind(
self, __sub: bytes | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
self, __sub: ReadableBuffer | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
) -> int: ...
def rindex(
self, __sub: bytes | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
self, __sub: ReadableBuffer | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
) -> int: ...
def rjust(self, __width: SupportsIndex, __fillchar: bytes = ...) -> bytes: ...
def rpartition(self, __sep: bytes) -> tuple[bytes, bytes, bytes]: ...
def rsplit(self, sep: bytes | None = ..., maxsplit: SupportsIndex = ...) -> list[bytes]: ...
def rstrip(self, __bytes: bytes | None = ...) -> bytes: ...
def split(self, sep: bytes | None = ..., maxsplit: SupportsIndex = ...) -> list[bytes]: ...
def rjust(self, __width: SupportsIndex, __fillchar: bytes | bytearray = ...) -> bytes: ...
def rpartition(self, __sep: ReadableBuffer) -> tuple[bytes, bytes, bytes]: ...
def rsplit(self, sep: ReadableBuffer | None = ..., maxsplit: SupportsIndex = ...) -> list[bytes]: ...
def rstrip(self, __bytes: ReadableBuffer | None = ...) -> bytes: ...
def split(self, sep: ReadableBuffer | None = ..., maxsplit: SupportsIndex = ...) -> list[bytes]: ...
def splitlines(self, keepends: bool = ...) -> list[bytes]: ...
def startswith(
self, __prefix: bytes | tuple[bytes, ...], __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
self,
__prefix: ReadableBuffer | tuple[ReadableBuffer, ...],
__start: SupportsIndex | None = ...,
__end: SupportsIndex | None = ...,
) -> bool: ...
def strip(self, __bytes: bytes | None = ...) -> bytes: ...
def strip(self, __bytes: ReadableBuffer | None = ...) -> bytes: ...
def swapcase(self) -> bytes: ...
def title(self) -> bytes: ...
def translate(self, __table: bytes | None, delete: bytes = ...) -> bytes: ...
def translate(self, __table: ReadableBuffer | None, delete: bytes = ...) -> bytes: ...
def upper(self) -> bytes: ...
def zfill(self, __width: SupportsIndex) -> bytes: ...
@classmethod
def fromhex(cls: type[Self], __s: str) -> Self: ...
@staticmethod
def maketrans(__frm: bytes, __to: bytes) -> bytes: ...
def maketrans(__frm: ReadableBuffer, __to: ReadableBuffer) -> bytes: ...
def __len__(self) -> int: ...
def __iter__(self) -> Iterator[int]: ...
def __hash__(self) -> int: ...
@overload
def __getitem__(self, __i: SupportsIndex) -> int: ...
@overload
def __getitem__(self, __s: slice) -> bytes: ...
def __add__(self, __s: bytes) -> bytes: ...
def __add__(self, __s: ReadableBuffer) -> bytes: ...
Copy link
Member Author

@JelleZijlstra JelleZijlstra Apr 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

>>> b"x" + memoryview(b"y")
b'xy'

def __mul__(self, __n: SupportsIndex) -> bytes: ...
def __rmul__(self, __n: SupportsIndex) -> bytes: ...
def __mod__(self, __value: Any) -> bytes: ...
Expand Down Expand Up @@ -583,12 +589,15 @@ class bytearray(MutableSequence[int], ByteString):
def capitalize(self) -> bytearray: ...
def center(self, __width: SupportsIndex, __fillchar: bytes = ...) -> bytearray: ...
def count(
self, __sub: bytes | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
self, __sub: ReadableBuffer | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
) -> int: ...
def copy(self) -> bytearray: ...
def decode(self, encoding: str = ..., errors: str = ...) -> str: ...
def endswith(
self, __suffix: bytes | tuple[bytes, ...], __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
self,
__suffix: ReadableBuffer | tuple[ReadableBuffer, ...],
__start: SupportsIndex | None = ...,
__end: SupportsIndex | None = ...,
) -> bool: ...
if sys.version_info >= (3, 8):
def expandtabs(self, tabsize: SupportsIndex = ...) -> bytearray: ...
Expand All @@ -597,15 +606,15 @@ class bytearray(MutableSequence[int], ByteString):

def extend(self, __iterable_of_ints: Iterable[SupportsIndex]) -> None: ...
def find(
self, __sub: bytes | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
self, __sub: ReadableBuffer | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
) -> int: ...
if sys.version_info >= (3, 8):
def hex(self, sep: str | bytes = ..., bytes_per_sep: SupportsIndex = ...) -> str: ...
else:
def hex(self) -> str: ...

def index(
self, __sub: bytes | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
self, __sub: ReadableBuffer | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
) -> int: ...
def insert(self, __index: SupportsIndex, __item: SupportsIndex) -> None: ...
def isalnum(self) -> bool: ...
Expand All @@ -618,43 +627,46 @@ class bytearray(MutableSequence[int], ByteString):
def isspace(self) -> bool: ...
def istitle(self) -> bool: ...
def isupper(self) -> bool: ...
def join(self, __iterable_of_bytes: Iterable[ByteString | memoryview]) -> bytearray: ...
def ljust(self, __width: SupportsIndex, __fillchar: bytes = ...) -> bytearray: ...
def join(self, __iterable_of_bytes: Iterable[ReadableBuffer]) -> bytearray: ...
def ljust(self, __width: SupportsIndex, __fillchar: bytes | bytearray = ...) -> bytearray: ...
def lower(self) -> bytearray: ...
def lstrip(self, __bytes: bytes | None = ...) -> bytearray: ...
def partition(self, __sep: bytes) -> tuple[bytearray, bytearray, bytearray]: ...
def lstrip(self, __bytes: ReadableBuffer | None = ...) -> bytearray: ...
def partition(self, __sep: ReadableBuffer) -> tuple[bytearray, bytearray, bytearray]: ...
def pop(self, __index: int = ...) -> int: ...
def remove(self, __value: int) -> None: ...
if sys.version_info >= (3, 9):
def removeprefix(self, __prefix: bytes) -> bytearray: ...
def removesuffix(self, __suffix: bytes) -> bytearray: ...
def removeprefix(self, __prefix: ReadableBuffer) -> bytearray: ...
def removesuffix(self, __suffix: ReadableBuffer) -> bytearray: ...

def replace(self, __old: bytes, __new: bytes, __count: SupportsIndex = ...) -> bytearray: ...
def replace(self, __old: ReadableBuffer, __new: ReadableBuffer, __count: SupportsIndex = ...) -> bytearray: ...
def rfind(
self, __sub: bytes | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
self, __sub: ReadableBuffer | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
) -> int: ...
def rindex(
self, __sub: bytes | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
self, __sub: ReadableBuffer | SupportsIndex, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
) -> int: ...
def rjust(self, __width: SupportsIndex, __fillchar: bytes = ...) -> bytearray: ...
def rpartition(self, __sep: bytes) -> tuple[bytearray, bytearray, bytearray]: ...
def rsplit(self, sep: bytes | None = ..., maxsplit: SupportsIndex = ...) -> list[bytearray]: ...
def rstrip(self, __bytes: bytes | None = ...) -> bytearray: ...
def split(self, sep: bytes | None = ..., maxsplit: SupportsIndex = ...) -> list[bytearray]: ...
def rjust(self, __width: SupportsIndex, __fillchar: bytes | bytearray = ...) -> bytearray: ...
def rpartition(self, __sep: ReadableBuffer) -> tuple[bytearray, bytearray, bytearray]: ...
def rsplit(self, sep: ReadableBuffer | None = ..., maxsplit: SupportsIndex = ...) -> list[bytearray]: ...
def rstrip(self, __bytes: ReadableBuffer | None = ...) -> bytearray: ...
def split(self, sep: ReadableBuffer | None = ..., maxsplit: SupportsIndex = ...) -> list[bytearray]: ...
def splitlines(self, keepends: bool = ...) -> list[bytearray]: ...
def startswith(
self, __prefix: bytes | tuple[bytes, ...], __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...
self,
__prefix: ReadableBuffer | tuple[ReadableBuffer, ...],
__start: SupportsIndex | None = ...,
__end: SupportsIndex | None = ...,
) -> bool: ...
def strip(self, __bytes: bytes | None = ...) -> bytearray: ...
def strip(self, __bytes: ReadableBuffer | None = ...) -> bytearray: ...
def swapcase(self) -> bytearray: ...
def title(self) -> bytearray: ...
def translate(self, __table: bytes | None, delete: bytes = ...) -> bytearray: ...
def translate(self, __table: ReadableBuffer | None, delete: bytes = ...) -> bytearray: ...
def upper(self) -> bytearray: ...
def zfill(self, __width: SupportsIndex) -> bytearray: ...
@classmethod
def fromhex(cls: type[Self], __string: str) -> Self: ...
@staticmethod
def maketrans(__frm: bytes, __to: bytes) -> bytes: ...
def maketrans(__frm: ReadableBuffer, __to: ReadableBuffer) -> bytes: ...
def __len__(self) -> int: ...
def __iter__(self) -> Iterator[int]: ...
__hash__: ClassVar[None] # type: ignore[assignment]
Expand All @@ -667,14 +679,15 @@ class bytearray(MutableSequence[int], ByteString):
@overload
def __setitem__(self, __s: slice, __x: Iterable[SupportsIndex] | bytes) -> None: ...
def __delitem__(self, __i: SupportsIndex | slice) -> None: ...
def __add__(self, __s: bytes) -> bytearray: ...
def __iadd__(self: Self, __s: Iterable[int]) -> Self: ...
Copy link
Member Author

@JelleZijlstra JelleZijlstra Apr 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was wrong; ba += [1, 2, 3] fails

def __add__(self, __s: ReadableBuffer) -> bytearray: ...
# The superclass wants us to accept Iterable[int], but that fails at runtime.
def __iadd__(self: Self, __s: ReadableBuffer) -> Self: ... # type: ignore[override]
def __mul__(self, __n: SupportsIndex) -> bytearray: ...
def __rmul__(self, __n: SupportsIndex) -> bytearray: ...
def __imul__(self: Self, __n: SupportsIndex) -> Self: ...
def __mod__(self, __value: Any) -> bytes: ...
# Incompatible with Sequence.__contains__
def __contains__(self, __o: SupportsIndex | bytes) -> bool: ... # type: ignore[override]
def __contains__(self, __o: SupportsIndex | ReadableBuffer) -> bool: ... # type: ignore[override]
def __eq__(self, __x: object) -> bool: ...
def __ne__(self, __x: object) -> bool: ...
def __lt__(self, __x: bytes) -> bool: ...
Expand Down Expand Up @@ -723,7 +736,7 @@ class memoryview(Sized, Sequence[int]):
def __iter__(self) -> Iterator[int]: ...
def __len__(self) -> int: ...
@overload
def __setitem__(self, __s: slice, __o: bytes) -> None: ...
def __setitem__(self, __s: slice, __o: ReadableBuffer) -> None: ...
@overload
def __setitem__(self, __i: SupportsIndex, __o: SupportsIndex) -> None: ...
if sys.version_info >= (3, 8):
Expand Down Expand Up @@ -1087,8 +1100,8 @@ if sys.version_info >= (3, 10):
# TODO: `compile` has a more precise return type in reality; work on a way of expressing that?
if sys.version_info >= (3, 8):
def compile(
source: str | bytes | AST,
filename: str | bytes | _PathLike[Any],
source: str | ReadableBuffer | AST,
filename: str | ReadableBuffer | _PathLike[Any],
mode: str,
flags: int = ...,
dont_inherit: int = ...,
Expand All @@ -1099,8 +1112,8 @@ if sys.version_info >= (3, 8):

else:
def compile(
source: str | bytes | AST,
filename: str | bytes | _PathLike[Any],
source: str | ReadableBuffer | AST,
filename: str | ReadableBuffer | _PathLike[Any],
mode: str,
flags: int = ...,
dont_inherit: int = ...,
Expand All @@ -1119,12 +1132,12 @@ def divmod(__x: _T_contra, __y: SupportsRDivMod[_T_contra, _T_co]) -> _T_co: ...
# The `globals` argument to `eval` has to be `dict[str, Any]` rather than `dict[str, object]` due to invariance.
# (The `globals` argument has to be a "real dict", rather than any old mapping, unlike the `locals` argument.)
def eval(
__source: str | bytes | CodeType, __globals: dict[str, Any] | None = ..., __locals: Mapping[str, object] | None = ...
__source: str | ReadableBuffer | CodeType, __globals: dict[str, Any] | None = ..., __locals: Mapping[str, object] | None = ...
) -> Any: ...

# Comment above regarding `eval` applies to `exec` as well
def exec(
__source: str | bytes | CodeType, __globals: dict[str, Any] | None = ..., __locals: Mapping[str, object] | None = ...
__source: str | ReadableBuffer | CodeType, __globals: dict[str, Any] | None = ..., __locals: Mapping[str, object] | None = ...
) -> None: ...
def exit(code: object = ...) -> NoReturn: ...

Expand Down Expand Up @@ -1352,7 +1365,7 @@ def open(
closefd: bool = ...,
opener: _Opener | None = ...,
) -> IO[Any]: ...
def ord(__c: str | bytes) -> int: ...
def ord(__c: str | bytes | bytearray) -> int: ...
Copy link
Member Author

@JelleZijlstra JelleZijlstra Apr 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

>>> ord(memoryview(b"x"))
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: ord() expected string of length 1, but memoryview found


class _SupportsWriteAndFlush(SupportsWrite[_T_contra], Protocol[_T_contra]):
def flush(self) -> None: ...
Expand Down Expand Up @@ -1713,7 +1726,7 @@ class UnicodeDecodeError(UnicodeError):
start: int
end: int
reason: str
def __init__(self, __encoding: str, __object: bytes, __start: int, __end: int, __reason: str) -> None: ...
def __init__(self, __encoding: str, __object: ReadableBuffer, __start: int, __end: int, __reason: str) -> None: ...

class UnicodeEncodeError(UnicodeError):
encoding: str
Expand Down