Skip to content

Commit

Permalink
Updated magnus
Browse files Browse the repository at this point in the history
  • Loading branch information
@ankane
ankane committed Jun 30, 2024
1 parent c828ffe commit 7c71990
Show file tree
Hide file tree
Showing 6 changed files with 26 additions and 26 deletions.
4 changes: 2 additions & 2 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion ext/tokenizers/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ publish = false
crate-type = ["cdylib"]

[dependencies]
magnus = "0.6"
magnus = "0.7"
onig = { version = "6", default-features = false }
serde = { version = "1", features = ["rc", "derive"] }

Expand Down
4 changes: 2 additions & 2 deletions ext/tokenizers/src/normalizers.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,8 +222,8 @@ pub struct RbSequence {}
impl RbSequence {
fn new(normalizers: RArray) -> RbResult<RbNormalizer> {
let mut sequence = Vec::with_capacity(normalizers.len());
for n in normalizers.each() {
let normalizer: &RbNormalizer = TryConvert::try_convert(n?)?;
for n in normalizers.into_iter() {
let normalizer: &RbNormalizer = TryConvert::try_convert(n)?;
match &normalizer.normalizer {
RbNormalizerTypeWrapper::Sequence(inner) => sequence.extend(inner.iter().cloned()),
RbNormalizerTypeWrapper::Single(inner) => sequence.push(inner.clone()),
Expand Down
4 changes: 2 additions & 2 deletions ext/tokenizers/src/pre_tokenizers.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,8 +258,8 @@ pub struct RbSequence {}
impl RbSequence {
fn new(pre_tokenizers: RArray) -> RbResult<RbPreTokenizer> {
let mut sequence = Vec::with_capacity(pre_tokenizers.len());
for n in pre_tokenizers.each() {
let pretokenizer: &RbPreTokenizer = TryConvert::try_convert(n?)?;
for n in pre_tokenizers.into_iter() {
let pretokenizer: &RbPreTokenizer = TryConvert::try_convert(n)?;
match &pretokenizer.pretok {
RbPreTokenizerTypeWrapper::Sequence(inner) => {
sequence.extend(inner.iter().cloned())
Expand Down
6 changes: 3 additions & 3 deletions ext/tokenizers/src/tokenizer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,12 +282,12 @@ impl RbTokenizer {
add_special_tokens: bool,
) -> RbResult<RArray> {
let input: Vec<tk::EncodeInput> = input
.each()
.into_iter()
.map(|o| {
let input: tk::EncodeInput = if is_pretokenized {
PreTokenizedEncodeInput::try_convert(o?)?.into()
PreTokenizedEncodeInput::try_convert(o)?.into()
} else {
TextEncodeInput::try_convert(o?)?.into()
TextEncodeInput::try_convert(o)?.into()
};
Ok(input)
})
Expand Down
32 changes: 16 additions & 16 deletions ext/tokenizers/src/trainers.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,9 +110,9 @@ impl RbTrainer {
BpeTrainer,
special_tokens,
special_tokens
.each()
.into_iter()
.map(|token| {
if let Ok(content) = String::try_convert(token?) {
if let Ok(content) = String::try_convert(token) {
Ok(RbAddedToken::from(content, Some(true)).get_token())
} else {
todo!()
Expand Down Expand Up @@ -197,9 +197,9 @@ impl RbTrainer {
UnigramTrainer,
special_tokens,
special_tokens
.each()
.into_iter()
.map(|token| {
if let Ok(content) = String::try_convert(token?) {
if let Ok(content) = String::try_convert(token) {
Ok(RbAddedToken::from(content, Some(true)).get_token())
} else {
todo!()
Expand Down Expand Up @@ -268,9 +268,9 @@ impl RbTrainer {
WordLevelTrainer,
special_tokens,
special_tokens
.each()
.into_iter()
.map(|token| {
if let Ok(content) = String::try_convert(token?) {
if let Ok(content) = String::try_convert(token) {
Ok(RbAddedToken::from(content, Some(true)).get_token())
} else {
todo!()
Expand Down Expand Up @@ -322,9 +322,9 @@ impl RbTrainer {
WordPieceTrainer,
@set_special_tokens,
special_tokens
.each()
.into_iter()
.map(|token| {
if let Ok(content) = String::try_convert(token?) {
if let Ok(content) = String::try_convert(token) {
Ok(RbAddedToken::from(content, Some(true)).get_token())
} else {
todo!()
Expand Down Expand Up @@ -398,9 +398,9 @@ impl RbBpeTrainer {
if !value.is_nil() {
builder = builder.special_tokens(
RArray::try_convert(value)?
.each()
.into_iter()
.map(|token| {
if let Ok(content) = String::try_convert(token?) {
if let Ok(content) = String::try_convert(token) {
Ok(RbAddedToken::from(content, Some(true)).get_token())
} else {
todo!()
Expand Down Expand Up @@ -466,9 +466,9 @@ impl RbUnigramTrainer {
if !value.is_nil() {
builder.special_tokens(
RArray::try_convert(value)?
.each()
.into_iter()
.map(|token| {
if let Ok(content) = String::try_convert(token?) {
if let Ok(content) = String::try_convert(token) {
Ok(RbAddedToken::from(content, Some(true)).get_token())
} else {
todo!()
Expand Down Expand Up @@ -540,9 +540,9 @@ impl RbWordLevelTrainer {
if !value.is_nil() {
builder.special_tokens(
RArray::try_convert(value)?
.each()
.into_iter()
.map(|token| {
if let Ok(content) = String::try_convert(token?) {
if let Ok(content) = String::try_convert(token) {
Ok(RbAddedToken::from(content, Some(true)).get_token())
} else {
todo!()
Expand Down Expand Up @@ -581,9 +581,9 @@ impl RbWordPieceTrainer {
if !value.is_nil() {
builder = builder.special_tokens(
RArray::try_convert(value)?
.each()
.into_iter()
.map(|token| {
if let Ok(content) = String::try_convert(token?) {
if let Ok(content) = String::try_convert(token) {
Ok(RbAddedToken::from(content, Some(true)).get_token())
} else {
todo!()
Expand Down

0 comments on commit 7c71990

Please sign in to comment.