diff --git a/Cargo.lock b/Cargo.lock index 0fd38907873..6a79215144a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1235,6 +1235,7 @@ dependencies = [ "rand_distr", "rand_pcg", "serde", + "tinystr", "writeable", "zerovec", ] diff --git a/components/datetime/src/external_loaders.rs b/components/datetime/src/external_loaders.rs index 53e5c060f0c..ae2f7057dc4 100644 --- a/components/datetime/src/external_loaders.rs +++ b/components/datetime/src/external_loaders.rs @@ -113,7 +113,9 @@ pub(crate) struct ExternalLoaderUnstable<'a, P: ?Sized>(pub &'a P); impl

FixedDecimalFormatterLoader for ExternalLoaderUnstable<'_, P> where - P: ?Sized + DataProvider, + P: ?Sized + + DataProvider + + DataProvider, { #[inline] fn load( diff --git a/components/datetime/src/format/neo.rs b/components/datetime/src/format/neo.rs index 1d691647657..4a9ea7db227 100644 --- a/components/datetime/src/format/neo.rs +++ b/components/datetime/src/format/neo.rs @@ -30,7 +30,7 @@ use icu_calendar::types::FormattingEra; use icu_calendar::types::MonthCode; use icu_decimal::options::FixedDecimalFormatterOptions; use icu_decimal::options::GroupingStrategy; -use icu_decimal::provider::DecimalSymbolsV2Marker; +use icu_decimal::provider::{DecimalDigitsV1Marker, DecimalSymbolsV2Marker}; use icu_decimal::FixedDecimalFormatter; use icu_provider::marker::NeverMarker; use icu_provider::prelude::*; @@ -262,7 +262,7 @@ where size_test!( TypedDateTimeNames, typed_date_time_names_size, - 336 + 352 ); /// A low-level type that formats datetime patterns with localized names. @@ -625,7 +625,7 @@ impl TypedDateTimeNames { #[doc = icu_provider::gen_any_buffer_unstable_docs!(UNSTABLE, Self::try_new)] pub fn try_new_unstable

(provider: &P, locale: &DataLocale) -> Result where - P: DataProvider + ?Sized, + P: DataProvider + DataProvider + ?Sized, { let mut names = Self { locale: locale.clone(), @@ -1418,7 +1418,7 @@ impl TypedDateTimeNames { #[inline] pub fn load_fixed_decimal_formatter

(&mut self, provider: &P) -> Result<&mut Self, DataError> where - P: DataProvider + ?Sized, + P: DataProvider + DataProvider + ?Sized, { self.inner .load_fixed_decimal_formatter(&ExternalLoaderUnstable(provider), &self.locale)?; @@ -1498,6 +1498,7 @@ impl TypedDateTimeNames { + DataProvider + DataProvider + DataProvider + + DataProvider + ?Sized, { let locale = &self.locale; diff --git a/components/datetime/src/neo.rs b/components/datetime/src/neo.rs index cde40cb7f01..a0a5a558063 100644 --- a/components/datetime/src/neo.rs +++ b/components/datetime/src/neo.rs @@ -132,7 +132,7 @@ macro_rules! gen_any_buffer_constructors_with_external_loader { // } // } -size_test!(FixedCalendarDateTimeFormatter, typed_neo_year_month_day_formatter_size, 336); +size_test!(FixedCalendarDateTimeFormatter, typed_neo_year_month_day_formatter_size, 352); /// [`FixedCalendarDateTimeFormatter`] is a formatter capable of formatting dates and/or times from /// a calendar selected at compile time. @@ -352,7 +352,7 @@ where size_test!( DateTimeFormatter, neo_year_month_day_formatter_size, - 392 + 408 ); /// [`DateTimeFormatter`] is a formatter capable of formatting dates and/or times from diff --git a/components/datetime/src/scaffold/fieldset_traits.rs b/components/datetime/src/scaffold/fieldset_traits.rs index 316d6b20a1b..53b52dfe657 100644 --- a/components/datetime/src/scaffold/fieldset_traits.rs +++ b/components/datetime/src/scaffold/fieldset_traits.rs @@ -18,7 +18,7 @@ use icu_calendar::{ }, Date, Iso, Time, }; -use icu_decimal::provider::DecimalSymbolsV2Marker; +use icu_decimal::provider::{DecimalDigitsV1Marker, DecimalSymbolsV2Marker}; use icu_provider::{marker::NeverMarker, prelude::*}; use icu_timezone::scaffold::IntoOption; use icu_timezone::{TimeZoneBcp47Id, UtcOffset, ZoneVariant}; @@ -368,10 +368,13 @@ where /// Trait to consolidate data provider markers external to this crate /// for datetime formatting with a fixed calendar. -pub trait AllFixedCalendarExternalDataMarkers: DataProvider {} +pub trait AllFixedCalendarExternalDataMarkers: + DataProvider + DataProvider +{ +} impl AllFixedCalendarExternalDataMarkers for T where - T: ?Sized + DataProvider + T: ?Sized + DataProvider + DataProvider { } @@ -385,6 +388,7 @@ pub trait AllAnyCalendarExternalDataMarkers: + DataProvider + DataProvider + DataProvider + + DataProvider { } @@ -397,6 +401,7 @@ impl AllAnyCalendarExternalDataMarkers for T where + DataProvider + DataProvider + DataProvider + + DataProvider { } diff --git a/components/decimal/Cargo.toml b/components/decimal/Cargo.toml index a1bfe9445b4..2fd71cc1edc 100644 --- a/components/decimal/Cargo.toml +++ b/components/decimal/Cargo.toml @@ -29,6 +29,7 @@ databake = { workspace = true, features = ["derive"], optional = true} serde = { workspace = true, features = ["derive", "alloc"], optional = true } icu_decimal_data = { workspace = true, optional = true } +tinystr = { workspace = true } [dev-dependencies] icu = { path = "../../components/icu", default-features = false } @@ -46,8 +47,8 @@ criterion = { workspace = true } [features] default = ["compiled_data"] std = ["fixed_decimal/std", "icu_locale_core/std", "icu_provider/std"] -serde = ["dep:serde", "icu_provider/serde", "zerovec/serde"] -datagen = ["serde", "dep:databake", "zerovec/databake"] +serde = ["dep:serde", "icu_provider/serde", "zerovec/serde", "tinystr/serde"] +datagen = ["serde", "dep:databake", "zerovec/databake", "tinystr/databake"] bench = ["serde"] compiled_data = ["dep:icu_decimal_data"] diff --git a/components/decimal/benches/fixed_decimal_format.rs b/components/decimal/benches/fixed_decimal_format.rs index fd32e9a9c0f..003c8d69f66 100644 --- a/components/decimal/benches/fixed_decimal_format.rs +++ b/components/decimal/benches/fixed_decimal_format.rs @@ -9,11 +9,7 @@ use rand_pcg::Lcg64Xsh32; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use fixed_decimal::FixedDecimal; -use icu_decimal::provider::{Baked, DecimalSymbolsV2Marker}; use icu_decimal::FixedDecimalFormatter; -use icu_locale_core::locale; -use icu_provider::prelude::*; -use icu_provider_adapters::fixed::FixedProvider; fn triangular_nums(range: f64) -> Vec { // Use Lcg64Xsh32, a small, fast PRNG. @@ -28,24 +24,13 @@ fn triangular_nums(range: f64) -> Vec { fn overview_bench(c: &mut Criterion) { let nums = triangular_nums(1e9); - let data = Baked - .load(DataRequest { - id: DataIdentifierBorrowed::for_locale(&locale!("en-US").into()), - ..Default::default() - }) - .unwrap() - .payload; - let provider = FixedProvider::::from_payload(data); + c.bench_function("icu_decimal/overview", |b| { b.iter(|| { // This benchmark demonstrates the performance of the format function on 1000 numbers // ranging from -1e9 to 1e9. - let fdf = FixedDecimalFormatter::try_new_unstable( - &provider, - &Default::default(), - Default::default(), - ) - .unwrap(); + let fdf = + FixedDecimalFormatter::try_new(&Default::default(), Default::default()).unwrap(); for &num in &nums { let fd = FixedDecimal::from(black_box(num)); fdf.format_to_string(&fd); diff --git a/components/decimal/src/format.rs b/components/decimal/src/format.rs index dca8c2a4aee..3dd22104ecf 100644 --- a/components/decimal/src/format.rs +++ b/components/decimal/src/format.rs @@ -18,6 +18,7 @@ pub struct FormattedFixedDecimal<'l> { pub(crate) value: &'l FixedDecimal, pub(crate) options: &'l FixedDecimalFormatterOptions, pub(crate) symbols: &'l DecimalSymbolsV2<'l>, + pub(crate) digits: &'l DecimalDigitsV1, } impl FormattedFixedDecimal<'_> { @@ -47,7 +48,7 @@ impl Writeable for FormattedFixedDecimal<'_> { sink.write_str(self.symbols.decimal_separator())?; } #[allow(clippy::indexing_slicing)] // digit_at in 0..=9 - sink.write_char(self.symbols.digits[self.value.digit_at(m) as usize])?; + sink.write_char(self.digits.digits[self.value.digit_at(m) as usize])?; if grouper::check( upper_magnitude, m, diff --git a/components/decimal/src/grouper.rs b/components/decimal/src/grouper.rs index c7e7d3369d7..73fdf21ae45 100644 --- a/components/decimal/src/grouper.rs +++ b/components/decimal/src/grouper.rs @@ -59,6 +59,7 @@ fn test_grouper() { use fixed_decimal::FixedDecimal; use icu_provider::prelude::*; use icu_provider_adapters::fixed::FixedProvider; + use icu_provider_adapters::fork::ForkByMarkerProvider; use writeable::assert_writeable_eq; let western_sizes = GroupingSizesV1 { @@ -154,12 +155,18 @@ fn test_grouper() { for cas in &cases { for i in 0..4 { let dec = FixedDecimal::from(1).multiplied_pow10((i as i16) + 3); - let provider = FixedProvider::::from_owned( + let provider_symbols = FixedProvider::::from_owned( crate::provider::DecimalSymbolsV2 { grouping_sizes: cas.sizes, ..DecimalSymbolsV2::new_en_for_testing() }, ); + let provider_digits = FixedProvider::::from_owned( + crate::provider::DecimalDigitsV1 { + digits: ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'], + }, + ); + let provider = ForkByMarkerProvider::new(provider_symbols, provider_digits); let options = options::FixedDecimalFormatterOptions { grouping_strategy: cas.strategy, ..Default::default() diff --git a/components/decimal/src/lib.rs b/components/decimal/src/lib.rs index b6e546292be..240841255f5 100644 --- a/components/decimal/src/lib.rs +++ b/components/decimal/src/lib.rs @@ -100,11 +100,12 @@ pub use format::FormattedFixedDecimal; use alloc::string::String; use fixed_decimal::FixedDecimal; +use icu_locale_core::locale; use icu_provider::prelude::*; use size_test_macro::size_test; use writeable::Writeable; -size_test!(FixedDecimalFormatter, fixed_decimal_formatter_size, 88); +size_test!(FixedDecimalFormatter, fixed_decimal_formatter_size, 104); /// A formatter for [`FixedDecimal`], rendering decimal digits in an i18n-friendly way. /// @@ -123,6 +124,7 @@ size_test!(FixedDecimalFormatter, fixed_decimal_formatter_size, 88); pub struct FixedDecimalFormatter { options: options::FixedDecimalFormatterOptions, symbols: DataPayload, + digits: DataPayload, } impl AsRef for FixedDecimalFormatter { @@ -138,23 +140,39 @@ impl FixedDecimalFormatter { ); #[doc = icu_provider::gen_any_buffer_unstable_docs!(UNSTABLE, Self::try_new)] - pub fn try_new_unstable + ?Sized>( + pub fn try_new_unstable< + D: DataProvider + + DataProvider + + ?Sized, + >( provider: &D, locale: &DataLocale, options: options::FixedDecimalFormatterOptions, ) -> Result { - let symbols = provider + let symbols: DataPayload = provider + .load(DataRequest { + id: DataIdentifierBorrowed::for_locale(locale), + ..Default::default() + })? + .payload; + let numsys = locale + .get_single_unicode_ext("nu") + .unwrap_or(&symbols.get().numsys); + let digits = provider .load(DataRequest { id: DataIdentifierBorrowed::for_marker_attributes_and_locale( - DataMarkerAttributes::from_str_or_panic( - locale.get_single_unicode_ext("nu").unwrap_or_default(), - ), - locale, + DataMarkerAttributes::from_str_or_panic(numsys), + &locale!("und").into(), ), ..Default::default() })? .payload; - Ok(Self { options, symbols }) + + Ok(Self { + options, + symbols, + digits, + }) } /// Formats a [`FixedDecimal`], returning a [`FormattedFixedDecimal`]. @@ -163,6 +181,7 @@ impl FixedDecimalFormatter { value, options: &self.options, symbols: self.symbols.get(), + digits: self.digits.get(), } } diff --git a/components/decimal/src/provider.rs b/components/decimal/src/provider.rs index 4bfee526045..d855fee925f 100644 --- a/components/decimal/src/provider.rs +++ b/components/decimal/src/provider.rs @@ -18,6 +18,7 @@ use alloc::borrow::Cow; use icu_provider::prelude::*; +use tinystr::TinyStr8; use zerovec::VarZeroCow; #[cfg(feature = "compiled_data")] @@ -41,11 +42,12 @@ const _: () = { } make_provider!(Baked); impl_decimal_symbols_v2_marker!(Baked); + impl_decimal_digits_v1_marker!(Baked); }; #[cfg(feature = "datagen")] /// The latest minimum set of markers required by this component. -pub const MARKERS: &[DataMarkerInfo] = &[DecimalSymbolsV2Marker::INFO]; +pub const MARKERS: &[DataMarkerInfo] = &[DecimalSymbolsV2Marker::INFO, DecimalDigitsV1Marker::INFO]; /// A collection of settings expressing where to put grouping separators in a decimal number. /// For example, `1,000,000` has two grouping separators, positioned along every 3 digits. @@ -136,6 +138,24 @@ pub struct DecimalSymbolsV2<'data> { /// Settings used to determine where to place groups in the integer part of the number. pub grouping_sizes: GroupingSizesV1, + /// The numbering system to use. + pub numsys: TinyStr8, +} + +/// The digits for a given numbering system. This data ought to be stored in the `und` locale with an auxiliary key +/// set to the numbering system code. +/// +///

+/// 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways, +/// including in SemVer minor releases. While the serde representation of data structs is guaranteed +/// to be stable, their Rust representation might not be. Use with caution. +///
+#[icu_provider::data_struct(DecimalDigitsV1Marker = "decimal/digits@1")] +#[derive(Debug, PartialEq, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize))] +#[cfg_attr(feature = "datagen", derive(serde::Serialize, databake::Bake))] +#[cfg_attr(feature = "datagen", databake(path = icu_decimal::provider))] +pub struct DecimalDigitsV1 { /// Digit characters for the current numbering system. In most systems, these digits are /// contiguous, but in some systems, such as *hanidec*, they are not contiguous. pub digits: [char; 10], @@ -185,7 +205,7 @@ impl DecimalSymbolsV2<'static> { secondary: 3, min_grouping: 1, }, - digits: ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'], + numsys: tinystr::tinystr!(8, "latn"), } } } diff --git a/components/experimental/src/compactdecimal/formatter.rs b/components/experimental/src/compactdecimal/formatter.rs index d14806fad7b..9ee9e7fd5c6 100644 --- a/components/experimental/src/compactdecimal/formatter.rs +++ b/components/experimental/src/compactdecimal/formatter.rs @@ -126,6 +126,7 @@ impl CompactDecimalFormatter { where D: DataProvider + DataProvider + + DataProvider + DataProvider + ?Sized, { @@ -212,6 +213,7 @@ impl CompactDecimalFormatter { where D: DataProvider + DataProvider + + DataProvider + DataProvider + ?Sized, { diff --git a/components/experimental/src/dimension/currency/formatter.rs b/components/experimental/src/dimension/currency/formatter.rs index 797d31587a2..67ea87bd227 100644 --- a/components/experimental/src/dimension/currency/formatter.rs +++ b/components/experimental/src/dimension/currency/formatter.rs @@ -81,7 +81,8 @@ impl CurrencyFormatter { where D: ?Sized + DataProvider - + DataProvider, + + DataProvider + + DataProvider, { let fixed_decimal_formatter = FixedDecimalFormatter::try_new_unstable( provider, diff --git a/components/experimental/src/dimension/currency/long_formatter.rs b/components/experimental/src/dimension/currency/long_formatter.rs index 6f92480a76f..7005f899888 100644 --- a/components/experimental/src/dimension/currency/long_formatter.rs +++ b/components/experimental/src/dimension/currency/long_formatter.rs @@ -102,6 +102,7 @@ impl LongCurrencyFormatter { + DataProvider + DataProvider + DataProvider + + DataProvider + DataProvider, { let fixed_decimal_formatter = FixedDecimalFormatter::try_new_unstable( diff --git a/components/experimental/src/dimension/percent/formatter.rs b/components/experimental/src/dimension/percent/formatter.rs index 1d516cb49f1..7c21798f632 100644 --- a/components/experimental/src/dimension/percent/formatter.rs +++ b/components/experimental/src/dimension/percent/formatter.rs @@ -73,7 +73,8 @@ impl PercentFormatter { where D: ?Sized + DataProvider - + DataProvider, + + DataProvider + + DataProvider, { let fixed_decimal_formatter = FixedDecimalFormatter::try_new_unstable( provider, diff --git a/components/experimental/src/dimension/units/formatter.rs b/components/experimental/src/dimension/units/formatter.rs index 05895758b09..ebb1d5c7d4c 100644 --- a/components/experimental/src/dimension/units/formatter.rs +++ b/components/experimental/src/dimension/units/formatter.rs @@ -119,6 +119,7 @@ impl UnitsFormatter { D: ?Sized + DataProvider + DataProvider + + DataProvider + DataProvider, { let fixed_decimal_formatter = FixedDecimalFormatter::try_new_unstable( diff --git a/components/experimental/src/duration/formatter.rs b/components/experimental/src/duration/formatter.rs index bb75da081da..148ddf33610 100644 --- a/components/experimental/src/duration/formatter.rs +++ b/components/experimental/src/duration/formatter.rs @@ -13,7 +13,7 @@ use super::validated_options::Unit; use super::{provider, Duration}; pub use super::validated_options::ValidatedDurationFormatterOptions; -use icu_decimal::provider::DecimalSymbolsV2Marker; +use icu_decimal::provider::{DecimalDigitsV1Marker, DecimalSymbolsV2Marker}; use icu_decimal::FixedDecimalFormatter; use icu_list::{ListFormatter, ListLength}; use icu_provider::prelude::*; @@ -114,6 +114,7 @@ impl DurationUnitFormatter { D: ?Sized + DataProvider + DataProvider + + DataProvider + DataProvider, >( provider: &D, @@ -210,6 +211,7 @@ impl DurationFormatter { D: DataProvider + DataProvider + DataProvider + + DataProvider + DataProvider + DataProvider + ?Sized, diff --git a/components/experimental/src/relativetime/relativetime.rs b/components/experimental/src/relativetime/relativetime.rs index 9f5c3d05b81..8e9f6654a56 100644 --- a/components/experimental/src/relativetime/relativetime.rs +++ b/components/experimental/src/relativetime/relativetime.rs @@ -4,7 +4,8 @@ use fixed_decimal::{FixedDecimal, Sign}; use icu_decimal::{ - options::FixedDecimalFormatterOptions, provider::DecimalSymbolsV2Marker, FixedDecimalFormatter, + options::FixedDecimalFormatterOptions, provider::DecimalDigitsV1Marker, + provider::DecimalSymbolsV2Marker, FixedDecimalFormatter, }; use icu_plurals::{provider::CardinalV1Marker, PluralRules}; use icu_provider::marker::ErasedMarker; @@ -165,7 +166,7 @@ macro_rules! constructor { where D: DataProvider + DataProvider<$marker> - + DataProvider + + DataProvider + DataProvider + ?Sized, { let temp_loc = locale.clone().into_locale(); diff --git a/ffi/capi/src/decimal.rs b/ffi/capi/src/decimal.rs index aa05031dd4b..dd890860761 100644 --- a/ffi/capi/src/decimal.rs +++ b/ffi/capi/src/decimal.rs @@ -12,6 +12,7 @@ pub mod ffi { errors::ffi::DataError, fixed_decimal::ffi::FixedDecimal, locale_core::ffi::Locale, provider::ffi::DataProvider, }; + use icu_decimal::options::FixedDecimalFormatterOptions; use writeable::Writeable; @@ -42,7 +43,7 @@ pub mod ffi { ) -> Result, DataError> { let locale = locale.to_datalocale(); - let mut options = icu_decimal::options::FixedDecimalFormatterOptions::default(); + let mut options = FixedDecimalFormatterOptions::default(); options.grouping_strategy = grouping_strategy .map(Into::into) .unwrap_or(options.grouping_strategy); @@ -73,6 +74,9 @@ pub mod ffi { grouping_strategy: Option, ) -> Result, DataError> { use alloc::borrow::Cow; + use icu_provider::any::AsDowncastingAnyProvider; + use icu_provider_adapters::{fixed::FixedProvider, fork::ForkByMarkerProvider}; + use tinystr::tinystr; use zerovec::VarZeroCow; fn str_to_cow(s: &'_ diplomat_runtime::DiplomatStr) -> Cow<'_, str> { if s.is_empty() { @@ -85,7 +89,8 @@ pub mod ffi { } use icu_decimal::provider::{ - DecimalSymbolStrsBuilder, DecimalSymbolsV2, GroupingSizesV1, + DecimalDigitsV1, DecimalDigitsV1Marker, DecimalSymbolStrsBuilder, DecimalSymbolsV2, + DecimalSymbolsV2Marker, GroupingSizesV1, }; let mut new_digits = ['\0'; 10]; for (old, new) in digits @@ -112,17 +117,22 @@ pub mod ffi { min_grouping: min_group_size, }; - let mut options = icu_decimal::options::FixedDecimalFormatterOptions::default(); + let mut options = FixedDecimalFormatterOptions::default(); options.grouping_strategy = grouping_strategy .map(Into::into) .unwrap_or(options.grouping_strategy); + let provider_symbols = + FixedProvider::::from_owned(DecimalSymbolsV2 { + strings: VarZeroCow::from_encodeable(&strings), + grouping_sizes, + numsys: tinystr!(8, "zyyy"), + }); + let provider_digits = + FixedProvider::::from_owned(DecimalDigitsV1 { digits }); + let provider = ForkByMarkerProvider::new(provider_symbols, provider_digits); Ok(Box::new(FixedDecimalFormatter( icu_decimal::FixedDecimalFormatter::try_new_unstable( - &icu_provider_adapters::fixed::FixedProvider::from_owned(DecimalSymbolsV2 { - strings: VarZeroCow::from_encodeable(&strings), - grouping_sizes, - digits, - }), + &provider.as_downcasting(), &Default::default(), options, )?, diff --git a/provider/registry/src/lib.rs b/provider/registry/src/lib.rs index 83350cb76cd..32f829f8502 100644 --- a/provider/registry/src/lib.rs +++ b/provider/registry/src/lib.rs @@ -45,6 +45,7 @@ macro_rules! registry( icu::datetime::provider::time_zones::MetazoneSpecificNamesShortV1Marker = "time_zone/specific_short@1", icu::datetime::provider::time_zones::TimeZoneEssentialsV1Marker = "time_zone/essentials@1", icu::timezone::provider::ZoneOffsetPeriodV1Marker = "time_zone/offset_period@1", + icu::decimal::provider::DecimalDigitsV1Marker = "decimal/digits@1", icu::decimal::provider::DecimalSymbolsV2Marker = "decimal/symbols@2", icu::list::provider::AndListV2Marker = "list/and@2", icu::list::provider::OrListV2Marker = "list/or@2", diff --git a/provider/source/src/decimal/compact.rs b/provider/source/src/decimal/compact.rs index 7a92ebc10b2..156a64a898a 100644 --- a/provider/source/src/decimal/compact.rs +++ b/provider/source/src/decimal/compact.rs @@ -102,13 +102,13 @@ impl DataProvider for SourceDataProvider { impl IterableDataProviderCached for SourceDataProvider { fn iter_ids_cached(&self) -> Result>, DataError> { - self.iter_ids_for_numbers() + self.iter_ids_for_numbers_with_locales() } } impl IterableDataProviderCached for SourceDataProvider { fn iter_ids_cached(&self) -> Result>, DataError> { - self.iter_ids_for_numbers() + self.iter_ids_for_numbers_with_locales() } } diff --git a/provider/source/src/decimal/digits.rs b/provider/source/src/decimal/digits.rs new file mode 100644 index 00000000000..c9cd6b8871a --- /dev/null +++ b/provider/source/src/decimal/digits.rs @@ -0,0 +1,36 @@ +// This file is part of ICU4X. For terms of use, please see the file +// called LICENSE at the top level of the ICU4X source tree +// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). + +use crate::IterableDataProviderCached; +use crate::SourceDataProvider; +use icu::decimal::provider::*; +use icu_provider::prelude::*; +use std::collections::HashSet; + +impl DataProvider for SourceDataProvider { + fn load(&self, req: DataRequest) -> Result, DataError> { + self.check_req::(req)?; + + let nsname = req.id.marker_attributes.as_str(); + + if nsname.is_empty() { + panic!("Found empty numbering system") + } + + let result = DecimalDigitsV1 { + digits: self.get_digits_for_numbering_system(nsname)?, + }; + + Ok(DataResponse { + metadata: Default::default(), + payload: DataPayload::from_owned(result), + }) + } +} + +impl IterableDataProviderCached for SourceDataProvider { + fn iter_ids_cached(&self) -> Result>, DataError> { + self.iter_ids_for_used_numbers() + } +} diff --git a/provider/source/src/decimal/mod.rs b/provider/source/src/decimal/mod.rs index 458c60e008f..789330c399e 100644 --- a/provider/source/src/decimal/mod.rs +++ b/provider/source/src/decimal/mod.rs @@ -6,6 +6,7 @@ use std::collections::HashSet; use crate::cldr_serde; use crate::SourceDataProvider; +use icu_locale_core::locale; use icu_provider::prelude::*; #[cfg(feature = "experimental")] @@ -15,6 +16,8 @@ mod compact_decimal_pattern; pub(crate) mod decimal_pattern; mod symbols; +mod digits; + impl SourceDataProvider { /// Returns the digits for the given numbering system name. fn get_digits_for_numbering_system(&self, nsname: &str) -> Result<[char; 10], DataError> { @@ -48,9 +51,11 @@ impl SourceDataProvider { }) } - fn get_supported_numsys_for_langid_without_default( + /// Get all numbering systems supported by a langid, potentially excluding the default one + fn get_supported_numsys_for_langid( &self, locale: &DataLocale, + exclude_default: bool, ) -> Result>, DataError> { let resource: &cldr_serde::numbers::Resource = self .cldr()? @@ -63,12 +68,17 @@ impl SourceDataProvider { .numsys_data .symbols .keys() - .filter(|nsname| **nsname != numbers.default_numbering_system) + .filter(|nsname| !exclude_default || **nsname != numbers.default_numbering_system) .filter_map(|nsname| Some(DataMarkerAttributes::try_from_str(nsname).ok()?.to_owned())) .collect()) } - fn iter_ids_for_numbers(&self) -> Result>, DataError> { + /// Produce DataIdentifier's for all locale-numbering system pairs in the form / + /// This also includes a bare + #[cfg(feature = "experimental")] + fn iter_ids_for_numbers_with_locales( + &self, + ) -> Result>, DataError> { Ok(self .cldr()? .numbers() @@ -76,7 +86,7 @@ impl SourceDataProvider { .flat_map(|locale| { let data_locale = locale.clone(); let last = data_locale.clone(); - self.get_supported_numsys_for_langid_without_default(&locale) + self.get_supported_numsys_for_langid(&locale, true) .expect("All languages from list_locales should be present") .into_iter() .map(move |nsname| { @@ -90,4 +100,49 @@ impl SourceDataProvider { }) .collect()) } + + /// Produce DataIdentifier's for all *used* numbering systems in the form und/ + fn iter_ids_for_used_numbers(&self) -> Result>, DataError> { + Ok(self + .cldr()? + .numbers() + .list_locales()? + .flat_map(|locale| { + self.get_supported_numsys_for_langid(&locale, false) + .expect("All languages from list_locales should be present") + .into_iter() + .map(move |nsname| { + DataIdentifierBorrowed::for_marker_attributes_and_locale( + DataMarkerAttributes::try_from_str(&nsname).unwrap(), + &locale!("und").into(), + ) + .into_owned() + }) + }) + .collect()) + } + + /// Produce DataIdentifier's for all digit-based numbering systems in the form und/ + #[allow(unused)] // TODO configurable + fn iter_all_number_ids(&self) -> Result>, DataError> { + use cldr_serde::numbering_systems::NumberingSystemType; + let resource: &cldr_serde::numbering_systems::Resource = self + .cldr()? + .core() + .read_and_parse("supplemental/numberingSystems.json")?; + + Ok(resource + .supplemental + .numbering_systems + .iter() + .filter(|(_nsname, data)| data.nstype == NumberingSystemType::Numeric) + .map(|(nsname, _data)| { + DataIdentifierBorrowed::for_marker_attributes_and_locale( + DataMarkerAttributes::try_from_str(nsname).unwrap(), + &locale!("und").into(), + ) + .into_owned() + }) + .collect()) + } } diff --git a/provider/source/src/decimal/symbols.rs b/provider/source/src/decimal/symbols.rs index 2d1a748e2a8..b07dc79b041 100644 --- a/provider/source/src/decimal/symbols.rs +++ b/provider/source/src/decimal/symbols.rs @@ -29,15 +29,13 @@ impl DataProvider for SourceDataProvider { &numbers.default_numbering_system }; - let mut result = + let result = DecimalSymbolsV2::try_from(NumbersWithNumsys(numbers, nsname)).map_err(|s| { DataError::custom("Could not create decimal symbols") .with_display_context(&s) .with_display_context(nsname) })?; - result.digits = self.get_digits_for_numbering_system(nsname)?; - Ok(DataResponse { metadata: Default::default(), payload: DataPayload::from_owned(result), @@ -47,7 +45,12 @@ impl DataProvider for SourceDataProvider { impl IterableDataProviderCached for SourceDataProvider { fn iter_ids_cached(&self) -> Result>, DataError> { - self.iter_ids_for_numbers() + Ok(self + .cldr()? + .numbers() + .list_locales()? + .map(|loc| DataIdentifierCow::from_locale(loc.clone())) + .collect()) } } @@ -87,6 +90,10 @@ impl TryFrom> for DecimalSymbolsV2<'static> { decimal_separator: Cow::Owned(symbols.decimal.clone()), grouping_separator: Cow::Owned(symbols.group.clone()), }; + let numsys = nsname + .parse() + .map_err(|_| format!("Numbering system {nsname} should not be more than 8 bytes!"))?; + Ok(Self { strings: VarZeroCow::from_encodeable(&strings), grouping_sizes: GroupingSizesV1 { @@ -94,7 +101,7 @@ impl TryFrom> for DecimalSymbolsV2<'static> { secondary: parsed_pattern.positive.secondary_grouping, min_grouping: numbers.minimum_grouping_digits, }, - digits: Default::default(), // to be filled in + numsys, }) } } @@ -111,7 +118,6 @@ fn test_basic() { ..Default::default() }) .unwrap(); - assert_eq!(ar_decimal.payload.get().decimal_separator(), "Ù«"); - assert_eq!(ar_decimal.payload.get().digits[0], 'Ù '); + assert_eq!(ar_decimal.payload.get().numsys, "arab"); }