Skip to content

Commit

Permalink
Fix docstring, type annotation for private method (openai#19)
Browse files Browse the repository at this point in the history
  • Loading branch information
arvid220u authored Jan 19, 2023
1 parent 40d9b1f commit cf385ca
Showing 1 changed file with 2 additions and 2 deletions.
4 changes: 2 additions & 2 deletions tiktoken/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ def encode_ordinary_batch(self, text: list[str], *, num_threads: int = 8) -> lis
This is equivalent to `encode_batch(text, disallowed_special=())` (but slightly faster).
```
>>> enc.encode_batch(["hello world", "goodbye world"])
>>> enc.encode_ordinary_batch(["hello world", "goodbye world"])
[[31373, 995], [11274, 16390, 995]]
```
"""
Expand Down Expand Up @@ -285,7 +285,7 @@ def _encode_single_piece(self, text_or_bytes: Union[str, bytes]) -> list[int]:
text_or_bytes = text_or_bytes.encode("utf-8")
return self._core_bpe.encode_single_piece(text_or_bytes)

def _encode_only_native_bpe(self, text: str) -> list[str]:
def _encode_only_native_bpe(self, text: str) -> list[int]:
"""Encodes a string into tokens, but do regex splitting in Python."""
_unused_pat = regex.compile(self._pat_str)
ret = []
Expand Down

0 comments on commit cf385ca

Please sign in to comment.