Skip to content

Commit

Permalink
feat(query): Simplified version Pattern S/L/D/G for number to_char (d…
Browse files Browse the repository at this point in the history 
…atabendlabs#16569)

* f32 basic locale

Signed-off-by: coldWater <[email protected]>

* fix

Signed-off-by: coldWater <[email protected]>

* FmtCacheEntry

Signed-off-by: coldWater <[email protected]>

* fix

Signed-off-by: coldWater <[email protected]>

* fix

Signed-off-by: coldWater <[email protected]>

* fix

Signed-off-by: coldWater <[email protected]>

---------

Signed-off-by: coldWater <[email protected]>
  • Loading branch information
@forsaken628
forsaken628 authored Oct 9, 2024
1 parent 040e8fa commit 25c2e4d
Show file tree
Hide file tree
Showing 4 changed files with 225 additions and 83 deletions.
206 changes: 154 additions & 52 deletions src/common/io/src/number.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.

use std::str::FromStr;

use databend_common_exception::ErrorCode;
use databend_common_exception::Result;
use enumflags2::bitflags;
Expand Down Expand Up @@ -259,26 +261,23 @@ impl NumDesc {
if self.flag.contains(NumFlag::LSign) {
return Err("cannot use \"S\" twice");
}
self.need_locale = true;
self.flag.insert(NumFlag::LSign);

if self
.flag
.intersects(NumFlag::Plus | NumFlag::Minus | NumFlag::Bracket)
{
return Err("cannot use \"S\" and \"PL\"/\"MI\"/\"SG\"/\"PR\" together");
}

if self.flag.contains(NumFlag::Decimal) {
if !self.flag.contains(NumFlag::Decimal) {
self.lsign = Some(NumLSign::Pre);
self.pre_lsign_num = self.pre;
self.need_locale = true;
self.flag.insert(NumFlag::LSign);
return Ok(());
}

if self.lsign.is_none() {
} else {
self.lsign = Some(NumLSign::Post);
self.need_locale = true;
self.flag.insert(NumFlag::LSign);
}

Ok(())
}

Expand Down Expand Up @@ -430,6 +429,14 @@ impl NumDesc {
}

fn f64_to_num_part(&mut self, value: f64) -> Result<NumPart> {
self.float_to_num_part(value, 15)
}

fn f32_to_num_part(&mut self, value: f32) -> Result<NumPart> {
self.float_to_num_part(value as f64, 6)
}

fn float_to_num_part(&mut self, value: f64, float_digits: usize) -> Result<NumPart> {
if self.flag.contains(NumFlag::Roman) {
return Err(ErrorCode::Unimplemented("to_char RN (Roman numeral)"));
}
Expand Down Expand Up @@ -463,12 +470,11 @@ impl NumDesc {
let orgnum = format!("{:.0}", value.abs());
let numstr_pre_len = orgnum.len();

const FLT_DIG: usize = 6;
// adjust post digits to fit max float digits
if numstr_pre_len >= FLT_DIG {
if numstr_pre_len >= float_digits {
self.post = 0;
} else if numstr_pre_len + self.post > FLT_DIG {
self.post = FLT_DIG - numstr_pre_len;
} else if numstr_pre_len + self.post > float_digits {
self.post = float_digits - numstr_pre_len;
}
let orgnum = format!("{:.*}", self.post, value.abs());

Expand Down Expand Up @@ -570,10 +576,6 @@ struct NumProc {
num_curr: usize, // current position in number
out_pre_spaces: usize, // spaces before first digit

_read_dec: bool, // to_number - was read dec. point
_read_post: usize, // to_number - number of dec. digit
_read_pre: usize, // to_number - number non-dec. digit

number: Vec<char>,
number_p: usize,

Expand All @@ -584,8 +586,8 @@ struct NumProc {
decimal: String,
loc_negative_sign: String,
loc_positive_sign: String,
_loc_thousands_sep: String,
_loc_currency_symbol: String,
loc_thousands_sep: String,
loc_currency_symbol: String,
}

impl NumProc {
Expand Down Expand Up @@ -728,6 +730,58 @@ impl NumProc {
fn last_relevant_is_dot(&self) -> bool {
self.last_relevant.is_some_and(|(c, _)| c == '.')
}

fn prepare_locale(&mut self) {
// todo: True localization will be the next step
self.loc_negative_sign = "-".to_string();
self.loc_positive_sign = "+".to_string();
self.loc_thousands_sep = ",".to_string();
self.loc_currency_symbol = "$".to_string();
}
}

pub struct FmtCacheEntry {
format: Vec<FormatNode>,
_str: String,
desc: NumDesc,
}

impl FromStr for FmtCacheEntry {
type Err = ErrorCode;

fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
let mut desc = NumDesc::default();
let format = parse_format(s, &NUM_KEYWORDS, Some(&mut desc))?;
Ok(FmtCacheEntry {
format,
_str: s.to_string(),
desc,
})
}
}

impl FmtCacheEntry {
pub fn process_i64(&self, value: i64) -> Result<String> {
let desc = self.desc.clone();
let num_part = desc.i64_to_num_part(value)?;
self.process(desc, num_part)
}

pub fn process_f64(&self, value: f64) -> Result<String> {
let mut desc = self.desc.clone();
let num_part = desc.f64_to_num_part(value)?;
self.process(desc, num_part)
}

pub fn process_f32(&self, value: f32) -> Result<String> {
let mut desc = self.desc.clone();
let num_part = desc.f32_to_num_part(value)?;
self.process(desc, num_part)
}

fn process(&self, desc: NumDesc, num_part: NumPart) -> Result<String> {
num_processor(&self.format, desc, num_part)
}
}

fn num_processor(nodes: &[FormatNode], desc: NumDesc, num_part: NumPart) -> Result<String> {
Expand All @@ -744,18 +798,15 @@ fn num_processor(nodes: &[FormatNode], desc: NumDesc, num_part: NumPart) -> Resu
num_in: false,
num_curr: 0,
out_pre_spaces,
_read_dec: false,
_read_post: 0,
_read_pre: 0,
number: number.chars().collect(),
number_p: 0,
inout: String::new(),
last_relevant: None,
decimal: ".".to_string(),
loc_negative_sign: String::new(),
loc_positive_sign: String::new(),
_loc_thousands_sep: String::new(),
_loc_currency_symbol: String::new(),
loc_thousands_sep: String::new(),
loc_currency_symbol: String::new(),
};

if np.desc.zero_start > 0 {
Expand Down Expand Up @@ -829,15 +880,18 @@ fn num_processor(nodes: &[FormatNode], desc: NumDesc, num_part: NumPart) -> Resu

// Locale
if np.desc.need_locale {
// NUM_prepare_locale(Np);
return Err(ErrorCode::Unimplemented("to_char uses locale S/L/D/G"));
np.prepare_locale();
}

// Processor direct cycle
for n in nodes.iter() {
match n {
// Format pictures actions
FormatNode::Action(key) => match key.id {
// Note: The locale sign is anchored to number and we
// write it when we work with first or last number
// (Tk0/Tk9).
NumPoz::TkS => (),
id @ (NumPoz::Tk9 | NumPoz::Tk0 | NumPoz::TkDec | NumPoz::TkD) => {
np.numpart_to_char(id)
}
Expand All @@ -852,6 +906,23 @@ fn num_processor(nodes: &[FormatNode], desc: NumDesc, num_part: NumPart) -> Resu
}
}

NumPoz::TkG => {
if np.num_in {
np.inout.push_str(&np.loc_thousands_sep);
continue;
}
if np.desc.flag.contains(NumFlag::FillMode) {
continue;
}
let sep = " ".repeat(np.loc_thousands_sep.len());
np.inout.push_str(&sep);
continue;
}

NumPoz::TkL => {
np.inout.push_str(&np.loc_currency_symbol);
}

NumPoz::TkMI => {
if np.sign {
if !np.desc.flag.contains(NumFlag::FillMode) {
Expand Down Expand Up @@ -890,30 +961,22 @@ fn num_processor(nodes: &[FormatNode], desc: NumDesc, num_part: NumPart) -> Resu
Ok(np.inout)
}

pub fn i64_to_char(value: i64, fmt: &str) -> Result<String> {
// TODO: We should cache FormatNode
let mut desc = NumDesc::default();
let nodes = parse_format(fmt, &NUM_KEYWORDS, Some(&mut desc))?;

let num_part = desc.i64_to_num_part(value)?;

num_processor(&nodes, desc, num_part)
}

pub fn f64_to_char(value: f64, fmt: &str) -> Result<String> {
// TODO: We should cache FormatNode
let mut desc = NumDesc::default();
let nodes = parse_format(fmt, &NUM_KEYWORDS, Some(&mut desc))?;

let num_part = desc.f64_to_num_part(value)?;

num_processor(&nodes, desc, num_part)
}

#[cfg(test)]
mod tests {
use super::*;

fn i64_to_char(value: i64, fmt: &str) -> Result<String> {
fmt.parse::<FmtCacheEntry>()?.process_i64(value)
}

fn f64_to_char(value: f64, fmt: &str) -> Result<String> {
fmt.parse::<FmtCacheEntry>()?.process_f64(value)
}

fn f32_to_char(value: f32, fmt: &str) -> Result<String> {
fmt.parse::<FmtCacheEntry>()?.process_f32(value)
}

#[test]
fn test_i64() -> Result<()> {
assert_eq!(" 123", i64_to_char(123, "999")?);
Expand Down Expand Up @@ -986,13 +1049,32 @@ mod tests {
assert_eq!("485 ", i64_to_char(485, "999MI")?);
assert_eq!("485", i64_to_char(485, "FM999MI")?);

// assert_eq!(" 1 485", i64_to_char(1485, "9G999")?);
assert_eq!(" 1,485", i64_to_char(1485, "9G999")?);
assert_eq!("-1,485", i64_to_char(-1485, "9G999")?);

assert_eq!("1,485", i64_to_char(1485, "FM9G999")?);
assert_eq!("-1,485", i64_to_char(-1485, "FM9G999")?);

assert_eq!(" 12,345,678", i64_to_char(12345678, "999G999G999")?);
assert_eq!(" -12,345,678", i64_to_char(-12345678, "999G999G999")?);

assert_eq!("12,345,678", i64_to_char(12345678, "FM999G999G999")?);
assert_eq!("-12,345,678", i64_to_char(-12345678, "FM999G999G999")?);

assert_eq!("$ 485", i64_to_char(485, "L999")?);
assert_eq!("$-485", i64_to_char(-485, "L999")?);

assert_eq!("485+", i64_to_char(485, "999S")?);
assert_eq!("485-", i64_to_char(-485, "999S")?);

assert_eq!("+485", i64_to_char(485, "S999")?);
assert_eq!("-485", i64_to_char(-485, "S999")?);

Ok(())
}

#[test]
fn test_f64() -> Result<()> {
fn test_float() -> Result<()> {
assert_eq!(" 12.34", f64_to_char(12.34, "99.99")?);
assert_eq!("-12.34", f64_to_char(-12.34, "99.99")?);
assert_eq!(" .10", f64_to_char(0.1, "99.99")?);
Expand All @@ -1014,11 +1096,31 @@ mod tests {
f64_to_char(485.8, "\"Pre:\"999\" Post:\" .999")?
);

// assert_eq!(" 148,500", f64_to_char(148.5, "999D999")?);
// assert_eq!(" 3 148,500", f64_to_char(3148.5, "9G999D999")?);
// assert_eq!("485-", f64_to_char(-485, "999S")?);
assert_eq!(" 148.500", f64_to_char(148.5, "999D999")?);

// assert_eq!("DM 485", f64_to_char(485, "L999")?);
assert_eq!(" 0003148.50", f32_to_char(3148.5, "0009999.999")?);
assert_eq!(" 3,148.50", f32_to_char(3148.5, "9G999D999")?);

assert_eq!(
" 1234567040",
f32_to_char(1.234_567e9, "99999999999D999999")?
);
assert_eq!(
" 1234567",
f32_to_char(1234567.0, "99999999999D999999")?
);
assert_eq!(
" 1234.57",
f32_to_char(1.234_567e3, "99999999999D999999")?
);
assert_eq!(
" 1.23457",
f32_to_char(1.234_567, "99999999999D999999")?
);
assert_eq!(
" .00123",
f32_to_char(1.234_567e-3, "99999999999D999999")?
);

// assert_eq!(" CDLXXXV", f64_to_char(485, "RN")?);
// assert_eq!("CDLXXXV", f64_to_char(485, "FMRN")?);
Expand Down
Loading

0 comments on commit 25c2e4d

Please sign in to comment.