diff --git a/bindings/node/Cargo.toml b/bindings/node/Cargo.toml
index cf1e51e99..43e86ed84 100644
--- a/bindings/node/Cargo.toml
+++ b/bindings/node/Cargo.toml
@@ -12,6 +12,7 @@ crate-type = ["cdylib"]
 [dependencies]
 napi        = "2"
 napi-derive = "2"
+rustc-hash = "2.1.1"
 serde       = { version = "1.0.163", features = ["derive"] }
 tokenizers  = { path = "../../tokenizers/" }
 
diff --git a/bindings/node/src/models.rs b/bindings/node/src/models.rs
index a4138b91f..f66962742 100644
--- a/bindings/node/src/models.rs
+++ b/bindings/node/src/models.rs
@@ -3,8 +3,8 @@ use crate::tasks::models::{BPEFromFilesTask, WordLevelFromFilesTask, WordPieceFr
 use crate::trainers::Trainer;
 use napi::bindgen_prelude::*;
 use napi_derive::napi;
+use rustc_hash::FxHashMap;
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, RwLock};
 use tokenizers as tk;
@@ -95,7 +95,7 @@ impl tk::Model for Model {
     self.model.as_ref()?.read().unwrap().id_to_token(id)
   }
 
-  fn get_vocab(&self) -> HashMap<String, u32> {
+  fn get_vocab(&self) -> FxHashMap<String, u32> {
     self
       .model
       .as_ref()
diff --git a/bindings/node/src/tokenizer.rs b/bindings/node/src/tokenizer.rs
index 4acbcac83..a99ac0313 100644
--- a/bindings/node/src/tokenizer.rs
+++ b/bindings/node/src/tokenizer.rs
@@ -6,7 +6,7 @@ use crate::pre_tokenizers::PreTokenizer;
 use crate::processors::Processor;
 use crate::tasks::tokenizer::{DecodeBatchTask, DecodeTask, EncodeBatchTask, EncodeTask};
 use crate::trainers::Trainer;
-use std::collections::HashMap;
+use rustc_hash::FxHashMap;
 use tokenizers::Model as ModelTrait;
 
 use napi::bindgen_prelude::*;
@@ -433,7 +433,7 @@ impl Tokenizer {
   }
 
   #[napi]
-  pub fn get_vocab(&self, with_added_tokens: Option<bool>) -> HashMap<String, u32> {
+  pub fn get_vocab(&self, with_added_tokens: Option<bool>) -> FxHashMap<String, u32> {
     let with_added_tokens = with_added_tokens.unwrap_or(true);
     self.tokenizer.read().unwrap().get_vocab(with_added_tokens)
   }
diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
index 6e8b0c34c..ac8041e6e 100644
--- a/bindings/python/Cargo.toml
+++ b/bindings/python/Cargo.toml
@@ -18,6 +18,7 @@ pyo3 = { version = "0.23", features = ["abi3", "abi3-py39", "py-clone"] }
 numpy = "0.23"
 ndarray = "0.16"
 itertools = "0.12"
+rustc-hash = "2.1.1"
 
 [dependencies.tokenizers]
 path = "../../tokenizers"
diff --git a/bindings/python/src/models.rs b/bindings/python/src/models.rs
index 2f4dba825..4d6e084f3 100644
--- a/bindings/python/src/models.rs
+++ b/bindings/python/src/models.rs
@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use rustc_hash::FxHashMap;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, RwLock};
 
@@ -70,7 +70,7 @@ impl Model for PyModel {
         self.model.read().unwrap().id_to_token(id)
     }
 
-    fn get_vocab(&self) -> HashMap<String, u32> {
+    fn get_vocab(&self) -> FxHashMap<String, u32> {
         self.model.read().unwrap().get_vocab()
     }
 
diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs
index 73a0dbbe8..cb4f6ad47 100644
--- a/bindings/python/src/tokenizer.rs
+++ b/bindings/python/src/tokenizer.rs
@@ -1,5 +1,5 @@
+use rustc_hash::{FxHashMap, FxHasher};
 use serde::Serialize;
-use std::collections::{hash_map::DefaultHasher, HashMap};
 use std::hash::{Hash, Hasher};
 
 use numpy::{npyffi, PyArray1, PyArrayMethods};
@@ -255,7 +255,7 @@ impl PyAddedToken {
     }
 
     fn __hash__(&self) -> u64 {
-        let mut hasher = DefaultHasher::new();
+        let mut hasher = FxHasher::default();
         self.get_token().hash(&mut hasher);
         hasher.finish()
     }
@@ -675,7 +675,7 @@ impl PyTokenizer {
     ///     :obj:`Dict[str, int]`: The vocabulary
     #[pyo3(signature = (with_added_tokens = true))]
     #[pyo3(text_signature = "(self, with_added_tokens=True)")]
-    fn get_vocab(&self, with_added_tokens: bool) -> HashMap<String, u32> {
+    fn get_vocab(&self, with_added_tokens: bool) -> FxHashMap<String, u32> {
         self.tokenizer.get_vocab(with_added_tokens)
     }
 
diff --git a/tokenizers/Cargo.toml b/tokenizers/Cargo.toml
index db56865d2..154b7a698 100644
--- a/tokenizers/Cargo.toml
+++ b/tokenizers/Cargo.toml
@@ -67,6 +67,7 @@ fancy-regex = { version = "0.14", optional = true}
 getrandom = { version = "0.2.10" }
 esaxx-rs = { version = "0.1.10", default-features = false, features=[]}
 monostate = "0.1.12"
+rustc-hash = "2.1.1"
 
 [features]
 default = ["progressbar", "onig", "esaxx_fast"]
diff --git a/tokenizers/src/models/bpe/model.rs b/tokenizers/src/models/bpe/model.rs
index 217c37e90..db9cfaebd 100644
--- a/tokenizers/src/models/bpe/model.rs
+++ b/tokenizers/src/models/bpe/model.rs
@@ -2,19 +2,22 @@ use super::{super::OrderedVocabIter, trainer::BpeTrainer, Error, Pair, Word};
 use crate::tokenizer::{Model, Result, Token};
 use crate::utils::cache::{Cache, DEFAULT_CACHE_CAPACITY, MAX_LENGTH};
 use crate::utils::iter::ResultShunt;
+use rustc_hash::FxHashMap;
 use serde_json::Value;
 use std::borrow::Cow;
+use std::collections::HashMap;
+use std::hash::BuildHasher;
+use std::iter::FromIterator;
 use std::{
-    collections::HashMap,
     fs::File,
     io::prelude::*,
     io::{BufRead, BufReader},
     path::{Path, PathBuf},
 };
 
-pub type Vocab = HashMap<String, u32>;
-type VocabR = HashMap<u32, String>;
-pub type MergeMap = HashMap<Pair, (u32, u32)>;
+pub type Vocab = FxHashMap<String, u32>;
+type VocabR = FxHashMap<u32, String>;
+pub type MergeMap = FxHashMap<Pair, (u32, u32)>;
 pub type Merges = Vec<(String, String)>;
 
 struct Config {
@@ -41,7 +44,7 @@ impl Default for BpeBuilder {
         Self {
             config: Config {
                 files: None,
-                vocab: HashMap::new(),
+                vocab: FxHashMap::default(),
                 merges: vec![],
                 cache_capacity: DEFAULT_CACHE_CAPACITY,
                 dropout: None,
@@ -71,8 +74,12 @@ impl BpeBuilder {
 
     /// Set the vocab (token -> ID) and merges mappings.
     #[must_use]
-    pub fn vocab_and_merges(mut self, vocab: Vocab, merges: Merges) -> Self {
-        self.config.vocab = vocab;
+    pub fn vocab_and_merges<S: BuildHasher>(
+        mut self,
+        vocab: HashMap<String, u32, S>,
+        merges: Merges,
+    ) -> Self {
+        self.config.vocab = FxHashMap::from_iter(vocab);
         self.config.merges = merges;
         self
     }
@@ -324,7 +331,7 @@ impl BPE {
         let mut buffer = String::new();
         vocab_file.read_to_string(&mut buffer)?;
         let json: Value = serde_json::from_str(&buffer)?;
-        let mut vocab = HashMap::new();
+        let mut vocab = FxHashMap::default();
         match json {
             Value::Object(m) => {
                 for (token, id) in m {
@@ -493,7 +500,7 @@ impl BPE {
 impl Model for BPE {
     type Trainer = BpeTrainer;
 
-    fn get_vocab(&self) -> HashMap<String, u32> {
+    fn get_vocab(&self) -> FxHashMap<String, u32> {
         self.vocab.clone()
     }
 
diff --git a/tokenizers/src/models/bpe/serialization.rs b/tokenizers/src/models/bpe/serialization.rs
index 98cc15102..b443889c8 100644
--- a/tokenizers/src/models/bpe/serialization.rs
+++ b/tokenizers/src/models/bpe/serialization.rs
@@ -1,10 +1,10 @@
 use super::{super::OrderedVocabIter, convert_merges_to_hashmap, BpeBuilder, Pair, BPE};
+use rustc_hash::FxHashMap;
 use serde::{
     de::{Error, MapAccess, Visitor},
     ser::SerializeStruct,
     Deserialize, Deserializer, Serialize, Serializer,
 };
-use std::collections::HashMap;
 
 impl Serialize for BPE {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
@@ -80,7 +80,7 @@ impl<'de> Visitor<'de> for BPEVisitor {
         V: MapAccess<'de>,
     {
         let mut builder = BpeBuilder::new();
-        let mut vocab: Option<HashMap<String, u32>> = None;
+        let mut vocab: Option<FxHashMap<String, u32>> = None;
 
         #[derive(Debug, Deserialize)]
         #[serde(untagged)]
diff --git a/tokenizers/src/models/bpe/trainer.rs b/tokenizers/src/models/bpe/trainer.rs
index a1a0aba76..7d7fe2a2f 100644
--- a/tokenizers/src/models/bpe/trainer.rs
+++ b/tokenizers/src/models/bpe/trainer.rs
@@ -4,15 +4,19 @@ use super::{Pair, WithFirstLastIterator, Word, BPE};
 use crate::parallelism::*;
 use crate::tokenizer::{AddedToken, Result, Trainer};
 use crate::utils::progress::{ProgressBar, ProgressStyle};
+use rustc_hash::FxHashMap;
+use rustc_hash::FxHashSet;
 use serde::{Deserialize, Serialize};
 use std::cmp::Ordering;
 use std::collections::{BinaryHeap, HashMap, HashSet};
+use std::hash::BuildHasher;
+use std::iter::FromIterator;
 
 #[derive(Debug, Eq)]
 struct Merge {
     pair: Pair,
     count: u64,
-    pos: HashSet<usize>,
+    pos: FxHashSet<usize>,
 }
 impl PartialEq for Merge {
     fn eq(&self, other: &Self) -> bool {
@@ -41,7 +45,7 @@ struct Config {
     show_progress: bool,
     special_tokens: Vec<AddedToken>,
     limit_alphabet: Option<usize>,
-    initial_alphabet: HashSet<char>,
+    initial_alphabet: FxHashSet<char>,
     continuing_subword_prefix: Option<String>,
     end_of_word_suffix: Option<String>,
     max_token_length: Option<usize>,
@@ -62,7 +66,7 @@ impl Default for BpeTrainerBuilder {
                 show_progress: true,
                 special_tokens: vec![],
                 limit_alphabet: None,
-                initial_alphabet: HashSet::new(),
+                initial_alphabet: FxHashSet::default(),
                 continuing_subword_prefix: None,
                 end_of_word_suffix: None,
                 max_token_length: None,
@@ -114,8 +118,8 @@ impl BpeTrainerBuilder {
 
     /// Set the initial alphabet
     #[must_use]
-    pub fn initial_alphabet(mut self, alphabet: HashSet<char>) -> Self {
-        self.config.initial_alphabet = alphabet;
+    pub fn initial_alphabet<S: BuildHasher>(mut self, alphabet: HashSet<char, S>) -> Self {
+        self.config.initial_alphabet = FxHashSet::from_iter(alphabet);
         self
     }
 
@@ -151,7 +155,7 @@ impl BpeTrainerBuilder {
             continuing_subword_prefix: self.config.continuing_subword_prefix,
             end_of_word_suffix: self.config.end_of_word_suffix,
             max_token_length: self.config.max_token_length,
-            words: HashMap::new(),
+            words: FxHashMap::default(),
         }
     }
 }
@@ -187,7 +191,7 @@ pub struct BpeTrainer {
     pub limit_alphabet: Option<usize>,
     /// The initial alphabet we want absolutely to include. This allows to cover
     /// some characters that are not necessarily in the training set
-    pub initial_alphabet: HashSet<char>,
+    pub initial_alphabet: FxHashSet<char>,
     /// An optional prefix to use on any subword that exist only behind another one
     pub continuing_subword_prefix: Option<String>,
     /// An optional suffix to caracterize and end-of-word subword
@@ -195,7 +199,7 @@ pub struct BpeTrainer {
     /// An optional parameter to limit the max length of any single token
     pub max_token_length: Option<usize>,
 
-    words: HashMap<String, u64>,
+    words: FxHashMap<String, u64>,
 }
 
 impl Default for BpeTrainer {
@@ -251,7 +255,11 @@ impl BpeTrainer {
     }
 
     /// Add the provided special tokens to the initial vocabulary
-    fn add_special_tokens(&self, w2id: &mut HashMap<String, u32>, id2w: &mut Vec<String>) {
+    fn add_special_tokens<S: BuildHasher>(
+        &self,
+        w2id: &mut HashMap<String, u32, S>,
+        id2w: &mut Vec<String>,
+    ) {
         for token in &self.special_tokens {
             if !w2id.contains_key(&token.content) {
                 id2w.push(token.content.to_owned());
@@ -261,14 +269,14 @@ impl BpeTrainer {
     }
 
     /// Compute the initial alphabet and limit it if relevant
-    fn compute_alphabet(
+    fn compute_alphabet<S1: BuildHasher, S2: BuildHasher>(
         &self,
-        wc: &HashMap<String, u64>,
-        w2id: &mut HashMap<String, u32>,
+        wc: &HashMap<String, u64, S1>,
+        w2id: &mut HashMap<String, u32, S2>,
         id2w: &mut Vec<String>,
     ) {
         // Compute the alphabet from seen words
-        let mut alphabet: HashMap<char, usize> = HashMap::new();
+        let mut alphabet: FxHashMap<char, usize> = FxHashMap::default();
         for (word, count) in wc {
             for c in word.chars() {
                 alphabet
@@ -320,10 +328,10 @@ impl BpeTrainer {
     }
 
     /// Tokenize words and add subwords to the vocabulary when relevant
-    fn tokenize_words(
+    fn tokenize_words<S1: BuildHasher, S2: BuildHasher>(
         &self,
-        wc: &HashMap<String, u64>,
-        w2id: &mut HashMap<String, u32>,
+        wc: &HashMap<String, u64, S1>,
+        w2id: &mut HashMap<String, u32, S2>,
         id2w: &mut Vec<String>,
         p: &Option<ProgressBar>,
     ) -> (Vec<Word>, Vec<u64>) {
@@ -375,13 +383,13 @@ impl BpeTrainer {
         words: &[Word],
         counts: &[u64],
         p: &Option<ProgressBar>,
-    ) -> (HashMap<Pair, i32>, HashMap<Pair, HashSet<usize>>) {
+    ) -> (FxHashMap<Pair, i32>, FxHashMap<Pair, FxHashSet<usize>>) {
         words
             .maybe_par_iter()
             .enumerate()
             .map(|(i, word)| {
-                let mut pair_counts = HashMap::new();
-                let mut where_to_update: HashMap<Pair, HashSet<usize>> = HashMap::new();
+                let mut pair_counts = FxHashMap::default();
+                let mut where_to_update: FxHashMap<Pair, FxHashSet<usize>> = FxHashMap::default();
 
                 for window in word.get_chars().windows(2) {
                     let cur_pair: Pair = (window[0], window[1]);
@@ -399,7 +407,7 @@ impl BpeTrainer {
                             h.insert(i);
                         })
                         .or_insert_with(|| {
-                            let mut h = HashSet::new();
+                            let mut h = FxHashSet::default();
                             h.insert(i);
                             h
                         });
@@ -413,7 +421,7 @@ impl BpeTrainer {
                 (pair_counts, where_to_update)
             })
             .reduce(
-                || (HashMap::new(), HashMap::new()),
+                || (FxHashMap::default(), FxHashMap::default()),
                 |(mut pair_counts, mut where_to_update), (pc, wtu)| {
                     for (k, v) in pc {
                         pair_counts.entry(k).and_modify(|c| *c += v).or_insert(v);
@@ -429,12 +437,13 @@ impl BpeTrainer {
             )
     }
 
-    pub fn do_train(
+    pub fn do_train<S: BuildHasher>(
         &self,
-        word_counts: &HashMap<String, u64>,
+        word_counts: &HashMap<String, u64, S>,
         model: &mut BPE,
     ) -> Result<Vec<AddedToken>> {
-        let mut word_to_id: HashMap<String, u32> = HashMap::with_capacity(self.vocab_size);
+        let mut word_to_id: FxHashMap<String, u32> =
+            FxHashMap::with_capacity_and_hasher(self.vocab_size, Default::default());
         let mut id_to_word: Vec<String> = Vec::with_capacity(self.vocab_size);
         let max_token_length: usize = self.max_token_length.unwrap_or(usize::MAX);
 
@@ -532,7 +541,7 @@ impl BpeTrainer {
             // Merge the new pair in every words
             // Safety: This is just a type assertion, the code below may no longer be safe
             // if the type of `pos` changes
-            let pos: &HashSet<usize> = &top.pos;
+            let pos: &FxHashSet<usize> = &top.pos;
 
             let words_len = words.len();
             struct WordPtr(*mut Word);
@@ -577,7 +586,7 @@ impl BpeTrainer {
                             h.insert(iw);
                         })
                         .or_insert_with(|| {
-                            let mut h = HashSet::new();
+                            let mut h = FxHashSet::default();
                             h.insert(iw);
                             h
                         });
@@ -647,18 +656,18 @@ impl Trainer for BpeTrainer {
         S: AsRef<str> + Send,
         F: Fn(&str) -> Result<Vec<String>> + Sync,
     {
-        let words: Result<HashMap<String, u64>> = iterator
+        let words: Result<FxHashMap<String, u64>> = iterator
             .maybe_par_bridge()
             .map(|sequence| {
                 let words = process(sequence.as_ref())?;
-                let mut map = HashMap::new();
+                let mut map = FxHashMap::default();
                 for word in words {
                     map.entry(word).and_modify(|c| *c += 1).or_insert(1);
                 }
                 Ok(map)
             })
             .reduce(
-                || Ok(HashMap::new()),
+                || Ok(FxHashMap::default()),
                 |acc, ws| {
                     let mut acc = acc?;
                     for (k, v) in ws? {
@@ -675,9 +684,10 @@ impl Trainer for BpeTrainer {
 
 #[cfg(test)]
 mod tests {
-    use super::{BpeTrainer, Pair, BPE};
     use std::collections::HashMap;
 
+    use super::{BpeTrainer, Pair, BPE};
+
     #[test]
     fn test_train() {
         let word_counts: HashMap<String, u64> = [
@@ -735,7 +745,12 @@ mod tests {
         .iter()
         .cloned()
         .collect();
-        assert_eq!(model.vocab, expected_vocab);
+
+        let mut lhs = model.vocab.into_iter().collect::<Vec<_>>();
+        let mut rhs = expected_vocab.into_iter().collect::<Vec<_>>();
+        lhs.sort_unstable();
+        rhs.sort_unstable();
+        assert_eq!(lhs, rhs);
 
         // The keys in `merges` are pairs of symbols, the values are tuples of (rank, id),
         // where 'rank' determines the order in which this merge will be applied during
@@ -749,7 +764,12 @@ mod tests {
         .iter()
         .cloned()
         .collect();
-        assert_eq!(model.merges, expected_merges);
+
+        let mut lhs = model.merges.into_iter().collect::<Vec<_>>();
+        let mut rhs = expected_merges.into_iter().collect::<Vec<_>>();
+        lhs.sort_unstable();
+        rhs.sort_unstable();
+        assert_eq!(lhs, rhs);
     }
     #[test]
     fn bpe_test_max_token_length_16() {
@@ -823,7 +843,7 @@ mod tests {
             .build();
         let mut model = BPE::default();
         trainer.do_train(&long_word_counts, &mut model).unwrap();
-        let trained_vocab: HashMap<String, u32> = model.get_vocab();
+        let trained_vocab = model.get_vocab();
         let expected_vocab: HashMap<String, u32> = [
             ("短", 12),
             ("n", 6),
@@ -860,6 +880,11 @@ mod tests {
         .cloned()
         .map(|(k, v)| (k.to_string(), v))
         .collect();
-        assert_eq!(trained_vocab, expected_vocab)
+
+        let mut lhs = trained_vocab.into_iter().collect::<Vec<_>>();
+        let mut rhs = expected_vocab.into_iter().collect::<Vec<_>>();
+        lhs.sort_unstable();
+        rhs.sort_unstable();
+        assert_eq!(lhs, rhs)
     }
 }
diff --git a/tokenizers/src/models/bpe/word.rs b/tokenizers/src/models/bpe/word.rs
index 93b3d9c37..24eb0e730 100644
--- a/tokenizers/src/models/bpe/word.rs
+++ b/tokenizers/src/models/bpe/word.rs
@@ -2,6 +2,7 @@ use super::Pair;
 use rand::{thread_rng, Rng};
 use std::cmp::Ordering;
 use std::collections::{BinaryHeap, HashMap};
+use std::hash::BuildHasher;
 
 #[derive(Debug, Eq)]
 struct Merge {
@@ -158,7 +159,11 @@ impl Word {
         changes
     }
 
-    pub(super) fn merge_all(&mut self, merges: &HashMap<Pair, (u32, u32)>, dropout: Option<f32>) {
+    pub(super) fn merge_all<S: BuildHasher>(
+        &mut self,
+        merges: &HashMap<Pair, (u32, u32), S>,
+        dropout: Option<f32>,
+    ) {
         let mut queue = BinaryHeap::with_capacity(self.symbols.len());
         let mut skip = Vec::with_capacity(queue.len());
 
diff --git a/tokenizers/src/models/mod.rs b/tokenizers/src/models/mod.rs
index 3a3a91adc..0fd750447 100644
--- a/tokenizers/src/models/mod.rs
+++ b/tokenizers/src/models/mod.rs
@@ -5,7 +5,7 @@ pub mod unigram;
 pub mod wordlevel;
 pub mod wordpiece;
 
-use std::collections::HashMap;
+use rustc_hash::FxHashMap;
 use std::path::{Path, PathBuf};
 
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -19,11 +19,11 @@ use crate::{AddedToken, Model, Result, Token, Trainer};
 /// Wraps a vocab mapping (ID -> token) to a struct that will be serialized in order
 /// of token ID, smallest to largest.
 struct OrderedVocabIter<'a> {
-    vocab_r: &'a HashMap<u32, String>,
+    vocab_r: &'a FxHashMap<u32, String>,
 }
 
 impl<'a> OrderedVocabIter<'a> {
-    fn new(vocab_r: &'a HashMap<u32, String>) -> Self {
+    fn new(vocab_r: &'a FxHashMap<u32, String>) -> Self {
         Self { vocab_r }
     }
 }
@@ -35,7 +35,7 @@ impl Serialize for OrderedVocabIter<'_> {
     {
         // There could be holes so max + 1 is more correct than vocab_r.len()
         let mut holes = vec![];
-        let result = if let Some(max) = self.vocab_r.iter().map(|(key, _)| key).max() {
+        let result = if let Some(max) = self.vocab_r.keys().max() {
             let iter = (0..*max + 1).filter_map(|i| {
                 if let Some(token) = self.vocab_r.get(&i) {
                     Some((token, i))
@@ -170,7 +170,7 @@ impl Model for ModelWrapper {
         }
     }
 
-    fn get_vocab(&self) -> HashMap<String, u32> {
+    fn get_vocab(&self) -> FxHashMap<String, u32> {
         match self {
             Self::WordLevel(t) => t.get_vocab(),
             Self::WordPiece(t) => t.get_vocab(),
@@ -287,6 +287,8 @@ impl_enum_from!(WordLevelTrainer, TrainerWrapper, WordLevelTrainer);
 
 #[cfg(test)]
 mod tests {
+    use std::iter::FromIterator;
+
     use super::*;
     use crate::models::bpe::{BpeBuilder, Vocab};
 
@@ -301,8 +303,8 @@ mod tests {
 
     #[test]
     fn incomplete_ordered_vocab() {
-        let vocab_r: HashMap<u32, String> =
-            HashMap::from([(0, "Hi".to_string()), (2, "There".to_string())]);
+        let vocab_r: FxHashMap<u32, String> =
+            FxHashMap::from_iter([(0, "Hi".to_string()), (2, "There".to_string())]);
 
         let ordered = OrderedVocabIter::new(&vocab_r);
 
diff --git a/tokenizers/src/models/unigram/model.rs b/tokenizers/src/models/unigram/model.rs
index da4d631ce..1c3f14234 100644
--- a/tokenizers/src/models/unigram/model.rs
+++ b/tokenizers/src/models/unigram/model.rs
@@ -6,12 +6,12 @@ use super::{
 use crate::tokenizer::{Model, Result, Token};
 use crate::utils::cache::{Cache, MAX_LENGTH};
 
-use std::collections::HashMap;
+use rustc_hash::FxHashMap;
 use std::convert::TryInto;
 use std::fs::read_to_string;
 use std::path::{Path, PathBuf};
 
-type TokenMap = HashMap<String, u32>;
+type TokenMap = FxHashMap<String, u32>;
 type Vocab = Vec<(String, f64)>;
 
 /// A `Unigram` model to encode sentences.
@@ -98,7 +98,7 @@ impl Unigram {
         byte_fallback: bool,
     ) -> Result<Self> {
         let n = vocab.len();
-        let mut token_to_ids: TokenMap = HashMap::new();
+        let mut token_to_ids: TokenMap = FxHashMap::default();
         let mut builder = TrieBuilder::default();
 
         if let Some(unk_id) = unk_id {
@@ -415,7 +415,7 @@ impl<'a> Iterator for UnigramIterator<'a> {
 impl Model for Unigram {
     type Trainer = UnigramTrainer;
 
-    fn get_vocab(&self) -> HashMap<String, u32> {
+    fn get_vocab(&self) -> FxHashMap<String, u32> {
         self.token_to_ids.clone()
     }
 
diff --git a/tokenizers/src/models/unigram/trainer.rs b/tokenizers/src/models/unigram/trainer.rs
index 5d178e77b..ebd4749fa 100644
--- a/tokenizers/src/models/unigram/trainer.rs
+++ b/tokenizers/src/models/unigram/trainer.rs
@@ -3,10 +3,13 @@ use crate::tokenizer::{AddedToken, Result, Trainer};
 use crate::utils::parallelism::*;
 use crate::utils::progress::{ProgressBar, ProgressStyle};
 use log::debug;
+use rustc_hash::FxHashMap;
+use rustc_hash::FxHashSet;
 use serde::{Deserialize, Serialize};
 use std::cmp::Reverse;
-use std::collections::{HashMap, HashSet};
+use std::collections::HashSet;
 use std::convert::TryInto;
+use std::hash::BuildHasher;
 
 // A token and a score
 type SentencePiece = (String, f64);
@@ -57,8 +60,8 @@ pub struct UnigramTrainer {
     pub shrinking_factor: f64,
     #[builder(default = "vec![]")]
     pub special_tokens: Vec<AddedToken>,
-    #[builder(default = "HashSet::new()")]
-    pub initial_alphabet: HashSet<char>,
+    #[builder(default = "FxHashSet::default()")]
+    pub initial_alphabet: FxHashSet<char>,
 
     #[builder(default = "None")]
     pub unk_token: Option<String>,
@@ -67,8 +70,8 @@ pub struct UnigramTrainer {
     pub max_piece_length: usize,
     #[builder(default = "1_000_000")]
     seed_size: usize,
-    #[builder(default = "HashMap::new()")]
-    words: HashMap<String, u32>,
+    #[builder(default = "FxHashMap::default()")]
+    words: FxHashMap<String, u32>,
 }
 
 impl Default for UnigramTrainer {
@@ -110,17 +113,21 @@ impl UnigramTrainer {
         true
     }
 
-    fn finalize(&self, model: Unigram, required_chars: HashSet<String>) -> Result<Unigram> {
+    fn finalize<S: BuildHasher>(
+        &self,
+        model: Unigram,
+        required_chars: HashSet<String, S>,
+    ) -> Result<Unigram> {
         let mut min_score_penalty = 0.0;
         let min_score_penalty_delta = 0.0001;
 
         let mut pieces: Vec<(String, f64)> = vec![];
-        let mut inserted: HashSet<String> = HashSet::new();
+        let mut inserted: FxHashSet<String> = FxHashSet::default();
 
         // We don't want to include the <UNK> that was used to train
         inserted.insert("<UNK>".into());
 
-        let existing_pieces: HashMap<String, f64> = model.iter().cloned().collect();
+        let existing_pieces: FxHashMap<String, f64> = model.iter().cloned().collect();
         for c in required_chars {
             if let Some(t) = existing_pieces.get(&c) {
                 inserted.insert(c.clone());
@@ -185,7 +192,7 @@ impl UnigramTrainer {
         )
     }
 
-    fn required_chars(&self, word_counts: &[Sentence]) -> HashSet<String> {
+    fn required_chars(&self, word_counts: &[Sentence]) -> FxHashSet<String> {
         word_counts
             .iter()
             .flat_map(|(s, _count)| s.chars())
@@ -205,7 +212,7 @@ impl UnigramTrainer {
             .sum::<usize>()
             + sentences.len();
         let mut flat_string = String::with_capacity(total);
-        let mut all_chars: HashMap<char, u32> = HashMap::new();
+        let mut all_chars: FxHashMap<char, u32> = FxHashMap::default();
         let c_sentence_boundary = '\0';
         let k_sentence_boundary = '\0'.to_string();
         for (string, n) in sentences {
@@ -631,18 +638,18 @@ impl Trainer for UnigramTrainer {
         S: AsRef<str> + Send,
         F: Fn(&str) -> Result<Vec<String>> + Sync,
     {
-        let words: Result<HashMap<String, u32>> = iterator
+        let words: Result<FxHashMap<String, u32>> = iterator
             .maybe_par_bridge()
             .map(|sequence| {
                 let words = process(sequence.as_ref())?;
-                let mut map = HashMap::new();
+                let mut map = FxHashMap::default();
                 for word in words {
                     map.entry(word).and_modify(|c| *c += 1).or_insert(1);
                 }
                 Ok(map)
             })
             .reduce(
-                || Ok(HashMap::new()),
+                || Ok(FxHashMap::default()),
                 |acc, ws| {
                     let mut acc = acc?;
                     for (k, v) in ws? {
@@ -661,7 +668,7 @@ impl Trainer for UnigramTrainer {
 mod tests {
     use super::*;
     use assert_approx_eq::assert_approx_eq;
-    use std::iter::FromIterator;
+    use std::{collections::HashSet, iter::FromIterator};
 
     #[test]
     fn test_unigram_chars() {
@@ -722,13 +729,18 @@ mod tests {
 
         let sentences = vec![("こんにちは友達".to_string(), 1)];
         let required_chars = trainer.required_chars(&sentences);
-        assert_eq!(
-            required_chars,
-            vec!["こ", "ん", "に", "ち", "は", "友", "達", "a", "b", "c", "d", "e", "f"]
-                .into_iter()
-                .map(|s| s.to_owned())
-                .collect::<HashSet<_>>()
-        );
+
+        let mut lhs = required_chars.into_iter().collect::<Vec<_>>();
+        let mut rhs = vec![
+            "こ", "ん", "に", "ち", "は", "友", "達", "a", "b", "c", "d", "e", "f",
+        ]
+        .into_iter()
+        .collect::<Vec<_>>();
+
+        lhs.sort_unstable();
+        rhs.sort_unstable();
+
+        assert_eq!(lhs, rhs);
     }
 
     #[test]
diff --git a/tokenizers/src/models/unigram/trie.rs b/tokenizers/src/models/unigram/trie.rs
index 2f94b1766..70f5333d4 100644
--- a/tokenizers/src/models/unigram/trie.rs
+++ b/tokenizers/src/models/unigram/trie.rs
@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use rustc_hash::FxHashMap;
 use std::hash::Hash;
 
 #[derive(Default)]
@@ -78,14 +78,14 @@ impl<Label> Default for Trie<Label> {
 #[derive(Clone)]
 pub struct Node<Label> {
     is_leaf: bool,
-    children: HashMap<Label, Node<Label>>,
+    children: FxHashMap<Label, Node<Label>>,
 }
 
 impl<Label> Default for Node<Label> {
     fn default() -> Self {
         Self {
             is_leaf: false,
-            children: HashMap::new(),
+            children: FxHashMap::default(),
         }
     }
 }
diff --git a/tokenizers/src/models/wordlevel/mod.rs b/tokenizers/src/models/wordlevel/mod.rs
index 545db13a7..cb7d1d705 100644
--- a/tokenizers/src/models/wordlevel/mod.rs
+++ b/tokenizers/src/models/wordlevel/mod.rs
@@ -1,9 +1,12 @@
 use super::OrderedVocabIter;
 use crate::tokenizer::{Model, Result, Token};
+use rustc_hash::FxHashMap;
 use serde_json::Value;
 use std::collections::HashMap;
 use std::fs::File;
+use std::hash::BuildHasher;
 use std::io::{BufReader, Read, Write};
+use std::iter::FromIterator;
 use std::path::{Path, PathBuf};
 
 mod serialization;
@@ -12,7 +15,7 @@ mod trainer;
 // Re-export
 pub use trainer::*;
 
-type Vocab = HashMap<String, u32>;
+type Vocab = FxHashMap<String, u32>;
 
 #[derive(thiserror::Error, Debug)]
 pub enum Error {
@@ -24,7 +27,7 @@ pub enum Error {
 
 struct Config {
     files: Option<String>,
-    vocab: HashMap<String, u32>,
+    vocab: FxHashMap<String, u32>,
     unk_token: String,
 }
 
@@ -39,7 +42,7 @@ impl Default for WordLevelBuilder {
         Self {
             config: Config {
                 files: None,
-                vocab: HashMap::new(),
+                vocab: FxHashMap::default(),
                 unk_token: String::from("<unk>"),
             },
         }
@@ -61,8 +64,8 @@ impl WordLevelBuilder {
 
     /// Set the vocab (token -> ID) mapping.
     #[must_use]
-    pub fn vocab(mut self, vocab: HashMap<String, u32>) -> Self {
-        self.config.vocab = vocab;
+    pub fn vocab<S: BuildHasher>(mut self, vocab: HashMap<String, u32, S>) -> Self {
+        self.config.vocab = FxHashMap::from_iter(vocab);
         self
     }
 
@@ -96,8 +99,8 @@ impl WordLevelBuilder {
 
 #[derive(PartialEq, Clone, Eq)]
 pub struct WordLevel {
-    vocab: HashMap<String, u32>,
-    vocab_r: HashMap<u32, String>,
+    vocab: FxHashMap<String, u32>,
+    vocab_r: FxHashMap<u32, String>,
     pub unk_token: String,
 }
 
@@ -119,7 +122,7 @@ impl WordLevel {
         let vocab_file = File::open(vocab_path)?;
         let mut vocab_file = BufReader::new(vocab_file);
         let mut buffer = String::new();
-        let mut vocab = HashMap::new();
+        let mut vocab = FxHashMap::default();
 
         vocab_file.read_to_string(&mut buffer)?;
         let json: Value = serde_json::from_str(&buffer)?;
@@ -148,8 +151,8 @@ impl WordLevel {
 impl Default for WordLevel {
     fn default() -> Self {
         Self {
-            vocab: HashMap::new(),
-            vocab_r: HashMap::new(),
+            vocab: FxHashMap::default(),
+            vocab_r: FxHashMap::default(),
             unk_token: String::from("<unk>"),
         }
     }
@@ -184,7 +187,7 @@ impl Model for WordLevel {
         self.vocab_r.get(&id).cloned()
     }
 
-    fn get_vocab(&self) -> HashMap<String, u32> {
+    fn get_vocab(&self) -> FxHashMap<String, u32> {
         self.vocab.clone()
     }
 
diff --git a/tokenizers/src/models/wordlevel/serialization.rs b/tokenizers/src/models/wordlevel/serialization.rs
index a077a4999..5827254f0 100644
--- a/tokenizers/src/models/wordlevel/serialization.rs
+++ b/tokenizers/src/models/wordlevel/serialization.rs
@@ -1,10 +1,10 @@
 use super::{super::OrderedVocabIter, WordLevel, WordLevelBuilder};
+use rustc_hash::FxHashSet;
 use serde::{
     de::{MapAccess, Visitor},
     ser::SerializeStruct,
     Deserialize, Deserializer, Serialize, Serializer,
 };
-use std::collections::HashSet;
 
 impl Serialize for WordLevel {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
@@ -52,10 +52,10 @@ impl<'de> Visitor<'de> for WordLevelVisitor {
             "vocab",
         ]
         .into_iter()
-        .collect::<HashSet<_>>();
+        .collect::<FxHashSet<_>>();
         while let Some(key) = map.next_key::<String>()? {
             match key.as_ref() {
-                "vocab" => builder = builder.vocab(map.next_value()?),
+                "vocab" => builder = builder.vocab::<rustc_hash::FxBuildHasher>(map.next_value()?),
                 "unk_token" => builder = builder.unk_token(map.next_value()?),
                 "type" => match map.next_value()? {
                     "WordLevel" => {}
diff --git a/tokenizers/src/models/wordlevel/trainer.rs b/tokenizers/src/models/wordlevel/trainer.rs
index c52ad08d7..a0a3f96bf 100644
--- a/tokenizers/src/models/wordlevel/trainer.rs
+++ b/tokenizers/src/models/wordlevel/trainer.rs
@@ -1,9 +1,11 @@
 use super::WordLevel;
 use crate::utils::parallelism::*;
 use crate::{AddedToken, Result, Trainer};
+use rustc_hash::{FxBuildHasher, FxHashMap};
 use serde::{Deserialize, Serialize};
 use std::cmp::Ordering;
 use std::collections::HashMap;
+use std::hash::BuildHasher;
 
 #[non_exhaustive]
 #[derive(Debug, Clone, Builder, Serialize, Deserialize)]
@@ -22,7 +24,7 @@ pub struct WordLevelTrainer {
     pub special_tokens: Vec<AddedToken>,
 
     #[builder(default, private)]
-    words: HashMap<String, u64>,
+    words: FxHashMap<String, u64>,
 }
 
 impl Default for WordLevelTrainer {
@@ -36,9 +38,9 @@ impl WordLevelTrainer {
         WordLevelTrainerBuilder::default()
     }
 
-    fn do_train(
+    fn do_train<S: BuildHasher>(
         &self,
-        word_counts: &HashMap<String, u64>,
+        word_counts: &HashMap<String, u64, S>,
         model: &mut WordLevel,
     ) -> Result<Vec<AddedToken>> {
         let mut ordered_counts = word_counts.iter().collect::<Vec<_>>();
@@ -56,7 +58,7 @@ impl WordLevelTrainer {
         ordered_counts.sort_by(cmp);
 
         let word_level = WordLevel::builder()
-            .vocab(
+            .vocab::<FxBuildHasher>(
                 self.special_tokens
                     .iter()
                     .map(|token| token.content.clone())
@@ -100,18 +102,18 @@ impl Trainer for WordLevelTrainer {
         S: AsRef<str> + Send,
         F: Fn(&str) -> Result<Vec<String>> + Sync,
     {
-        let words: Result<HashMap<String, u64>> = iterator
+        let words: Result<FxHashMap<String, u64>> = iterator
             .maybe_par_bridge()
             .map(|sequence| {
                 let words = process(sequence.as_ref())?;
-                let mut map = HashMap::new();
+                let mut map = FxHashMap::default();
                 for word in words {
                     map.entry(word).and_modify(|c| *c += 1).or_insert(1);
                 }
                 Ok(map)
             })
             .reduce(
-                || Ok(HashMap::new()),
+                || Ok(FxHashMap::default()),
                 |acc, ws| {
                     let mut acc = acc?;
                     for (k, v) in ws? {
@@ -128,6 +130,8 @@ impl Trainer for WordLevelTrainer {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashMap;
+
     use super::*;
 
     #[test]
@@ -161,7 +165,12 @@ mod tests {
         .iter()
         .cloned()
         .collect();
-        assert_eq!(model.vocab, expected_vocab);
+
+        let mut lhs = model.vocab.into_iter().collect::<Vec<_>>();
+        let mut rhs = expected_vocab.into_iter().collect::<Vec<_>>();
+        lhs.sort_unstable();
+        rhs.sort_unstable();
+        assert_eq!(lhs, rhs);
 
         // If we specify a min_frequency
         trainer.min_frequency = 15;
@@ -177,6 +186,10 @@ mod tests {
         .cloned()
         .collect();
 
-        assert_eq!(model.vocab, expected_vocab);
+        let mut lhs = model.vocab.into_iter().collect::<Vec<_>>();
+        let mut rhs = expected_vocab.into_iter().collect::<Vec<_>>();
+        lhs.sort_unstable();
+        rhs.sort_unstable();
+        assert_eq!(lhs, rhs);
     }
 }
diff --git a/tokenizers/src/models/wordpiece/mod.rs b/tokenizers/src/models/wordpiece/mod.rs
index 0c63405c1..b1d16bd9a 100644
--- a/tokenizers/src/models/wordpiece/mod.rs
+++ b/tokenizers/src/models/wordpiece/mod.rs
@@ -3,9 +3,9 @@
 
 use crate::models::bpe::BPE;
 use crate::tokenizer::{Model, Result, Token};
+use rustc_hash::FxHashMap;
 use std::{
     borrow::Cow,
-    collections::HashMap,
     fs::File,
     io::prelude::*,
     io::{BufRead, BufReader},
@@ -22,8 +22,8 @@ pub enum Error {
     MissingUnkToken,
 }
 
-type Vocab = HashMap<String, u32>;
-type VocabR = HashMap<u32, String>;
+type Vocab = FxHashMap<String, u32>;
+type VocabR = FxHashMap<u32, String>;
 
 struct Config {
     files: Option<String>,
@@ -43,7 +43,7 @@ impl Default for WordPieceBuilder {
         Self {
             config: Config {
                 files: None,
-                vocab: HashMap::new(),
+                vocab: FxHashMap::default(),
                 unk_token: String::from("[UNK]"),
                 continuing_subword_prefix: String::from("##"),
                 max_input_chars_per_word: 100,
@@ -142,8 +142,8 @@ impl std::fmt::Debug for WordPiece {
 impl Default for WordPiece {
     fn default() -> Self {
         Self {
-            vocab: HashMap::new(),
-            vocab_r: HashMap::new(),
+            vocab: FxHashMap::default(),
+            vocab_r: FxHashMap::default(),
             unk_token: String::from("[UNK]"),
             continuing_subword_prefix: String::from("##"),
             max_input_chars_per_word: 100,
@@ -162,7 +162,7 @@ impl WordPiece {
         let file = File::open(vocab)?;
         let file = BufReader::new(file);
 
-        let mut vocab = HashMap::new();
+        let mut vocab = FxHashMap::default();
         for (index, line) in file.lines().enumerate() {
             let line = line?;
             vocab.insert(line.trim_end().to_owned(), index as u32);
@@ -192,7 +192,7 @@ impl WordPiece {
 impl Model for WordPiece {
     type Trainer = WordPieceTrainer;
 
-    fn get_vocab(&self) -> HashMap<String, u32> {
+    fn get_vocab(&self) -> FxHashMap<String, u32> {
         self.vocab.clone()
     }
 
diff --git a/tokenizers/src/models/wordpiece/serialization.rs b/tokenizers/src/models/wordpiece/serialization.rs
index a50274e81..6883f0a21 100644
--- a/tokenizers/src/models/wordpiece/serialization.rs
+++ b/tokenizers/src/models/wordpiece/serialization.rs
@@ -1,10 +1,10 @@
 use super::{super::OrderedVocabIter, WordPiece, WordPieceBuilder};
+use rustc_hash::FxHashSet;
 use serde::{
     de::{MapAccess, Visitor},
     ser::SerializeStruct,
     Deserialize, Deserializer, Serialize, Serializer,
 };
-use std::collections::HashSet;
 
 impl Serialize for WordPiece {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
@@ -67,7 +67,7 @@ impl<'de> Visitor<'de> for WordPieceVisitor {
             "vocab",
         ]
         .into_iter()
-        .collect::<HashSet<_>>();
+        .collect::<FxHashSet<_>>();
 
         while let Some(key) = map.next_key::<String>()? {
             match key.as_ref() {
diff --git a/tokenizers/src/models/wordpiece/trainer.rs b/tokenizers/src/models/wordpiece/trainer.rs
index 58a5abc8f..586321e13 100644
--- a/tokenizers/src/models/wordpiece/trainer.rs
+++ b/tokenizers/src/models/wordpiece/trainer.rs
@@ -1,8 +1,12 @@
+use std::collections::HashSet;
+use std::hash::BuildHasher;
+use std::iter::FromIterator;
+
 use super::WordPiece;
 use crate::models::bpe::{BpeTrainer, BpeTrainerBuilder, BPE};
 use crate::tokenizer::{AddedToken, Result, Trainer};
+use rustc_hash::FxHashSet;
 use serde::{Deserialize, Serialize};
-use std::collections::HashSet;
 
 /// A `WordPieceTrainerBuilder` can be used to create a `WordPieceTrainer` with a custom
 /// configuration.
@@ -61,7 +65,7 @@ impl WordPieceTrainerBuilder {
 
     /// Set the initial alphabet
     #[must_use]
-    pub fn initial_alphabet(mut self, alphabet: HashSet<char>) -> Self {
+    pub fn initial_alphabet<S: BuildHasher>(mut self, alphabet: HashSet<char, S>) -> Self {
         self.bpe_trainer_builder = self.bpe_trainer_builder.initial_alphabet(alphabet);
         self
     }
@@ -134,12 +138,12 @@ impl WordPieceTrainer {
         self.bpe_trainer.limit_alphabet = limit;
     }
 
-    pub fn initial_alphabet(&self) -> &HashSet<char> {
+    pub fn initial_alphabet(&self) -> &FxHashSet<char> {
         &self.bpe_trainer.initial_alphabet
     }
 
-    pub fn set_initial_alphabet(&mut self, alphabet: HashSet<char>) {
-        self.bpe_trainer.initial_alphabet = alphabet;
+    pub fn set_initial_alphabet<S: BuildHasher>(&mut self, alphabet: HashSet<char, S>) {
+        self.bpe_trainer.initial_alphabet = FxHashSet::from_iter(alphabet);
     }
 
     pub fn continuing_subword_prefix(&self) -> &Option<String> {
diff --git a/tokenizers/src/normalizers/byte_level.rs b/tokenizers/src/normalizers/byte_level.rs
index ae47de5ac..47f611022 100644
--- a/tokenizers/src/normalizers/byte_level.rs
+++ b/tokenizers/src/normalizers/byte_level.rs
@@ -1,14 +1,15 @@
 use crate::processors::byte_level::bytes_char;
 use crate::tokenizer::{NormalizedString, Normalizer, Result};
 use crate::utils::macro_rules_attribute;
-use std::collections::{HashMap, HashSet};
+use rustc_hash::FxHashMap;
+use rustc_hash::FxHashSet;
 use std::sync::LazyLock;
 
 #[derive(Clone, Debug)]
 #[macro_rules_attribute(impl_serde_type!)]
 pub struct ByteLevel;
 
-static BYTES_CHAR: LazyLock<HashMap<u8, char>> = LazyLock::new(bytes_char);
+static BYTES_CHAR: LazyLock<FxHashMap<u8, char>> = LazyLock::new(bytes_char);
 
 impl Default for ByteLevel {
     fn default() -> Self {
@@ -21,7 +22,7 @@ impl ByteLevel {
         Self {}
     }
 
-    pub fn alphabet() -> HashSet<char> {
+    pub fn alphabet() -> FxHashSet<char> {
         BYTES_CHAR.values().copied().collect()
     }
 }
diff --git a/tokenizers/src/pre_tokenizers/byte_level.rs b/tokenizers/src/pre_tokenizers/byte_level.rs
index e761cbc96..ee1ee5de4 100644
--- a/tokenizers/src/pre_tokenizers/byte_level.rs
+++ b/tokenizers/src/pre_tokenizers/byte_level.rs
@@ -1,4 +1,5 @@
-use std::collections::{HashMap, HashSet};
+use rustc_hash::FxHashMap;
+use rustc_hash::FxHashSet;
 use std::sync::LazyLock;
 
 use crate::utils::SysRegex;
@@ -12,7 +13,7 @@ use crate::utils::macro_rules_attribute;
 
 /// Converts bytes to unicode characters.
 /// See https://github.com/openai/gpt-2/blob/master/src/encoder.py#L9
-pub(crate) fn bytes_char() -> HashMap<u8, char> {
+pub(crate) fn bytes_char() -> FxHashMap<u8, char> {
     let mut bs: Vec<u8> = vec![];
     bs.extend(b'!'..=b'~');
     bs.extend(b'\xA1'..=b'\xAC');
@@ -44,8 +45,8 @@ static RE: LazyLock<SysRegex> = LazyLock::new(|| {
     SysRegex::new(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+")
         .unwrap()
 });
-static BYTES_CHAR: LazyLock<HashMap<u8, char>> = LazyLock::new(bytes_char);
-static CHAR_BYTES: LazyLock<HashMap<char, u8>> =
+static BYTES_CHAR: LazyLock<FxHashMap<u8, char>> = LazyLock::new(bytes_char);
+static CHAR_BYTES: LazyLock<FxHashMap<char, u8>> =
     LazyLock::new(|| bytes_char().into_iter().map(|(c, b)| (b, c)).collect());
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
@@ -90,7 +91,7 @@ impl ByteLevel {
         }
     }
 
-    pub fn alphabet() -> HashSet<char> {
+    pub fn alphabet() -> FxHashSet<char> {
         BYTES_CHAR.values().copied().collect()
     }
 
@@ -243,7 +244,7 @@ mod tests {
         Decoder, Encoding, OffsetReferential, OffsetType, PostProcessor, PreTokenizedString,
         PreTokenizer,
     };
-    use std::iter::FromIterator;
+    use std::collections::HashMap;
 
     #[test]
     fn pre_tokenization() {
@@ -503,7 +504,7 @@ mod tests {
             vec![],
             vec![],
             vec![],
-            HashMap::from_iter(vec![(0, 0..5)]),
+            HashMap::from([(0, 0..5)]),
         );
 
         let bytelevel = ByteLevel::default().trim_offsets(true);
@@ -543,7 +544,7 @@ mod tests {
             vec![],
             vec![],
             vec![],
-            HashMap::from_iter(vec![(0, 0..5), (1, 5..10)]),
+            HashMap::from([(0, 0..5), (1, 5..10)]),
         );
         assert_eq!(
             pair_expected,
diff --git a/tokenizers/src/processors/bert.rs b/tokenizers/src/processors/bert.rs
index 179391122..6aa2cfe97 100644
--- a/tokenizers/src/processors/bert.rs
+++ b/tokenizers/src/processors/bert.rs
@@ -1,6 +1,6 @@
 use crate::tokenizer::{Encoding, PostProcessor, Result};
+use rustc_hash::FxHashMap;
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
 use std::iter::FromIterator;
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
@@ -78,7 +78,7 @@ impl PostProcessor for BertProcessing {
 
                     // For compatibility with `TemplateProcessing`, the sequence_ranges shouldn't contain
                     // the special tokens.
-                    let sequence_ranges = HashMap::from_iter(vec![(0, 1..ids.len() - 1)]);
+                    let sequence_ranges = FxHashMap::from_iter(vec![(0, 1..ids.len() - 1)]);
                     Encoding::new(
                         ids,
                         type_ids,
@@ -111,7 +111,7 @@ impl PostProcessor for BertProcessing {
                                 // For compatibility with `TemplateProcessing`, the sequence_ranges shouldn't
                                 // contain the special tokens.
                                 let sequence_ranges =
-                                    HashMap::from_iter(vec![(0, 1..ids.len() - 1)]);
+                                    FxHashMap::from_iter(vec![(0, 1..ids.len() - 1)]);
                                 Encoding::new(
                                     ids,
                                     type_ids,
@@ -139,7 +139,8 @@ impl PostProcessor for BertProcessing {
 
                     // For compatibility with `TemplateProcessing`, the sequence_ranges shouldn't contain
                     // the special tokens.
-                    let pair_sequence_ranges = HashMap::from_iter(vec![(1, 0..pair_ids.len() - 1)]);
+                    let pair_sequence_ranges =
+                        FxHashMap::from_iter(vec![(1, 0..pair_ids.len() - 1)]);
                     Encoding::new(
                         pair_ids,
                         pair_type_ids,
@@ -165,7 +166,7 @@ impl PostProcessor for BertProcessing {
                                 // For compatibility with `TemplateProcessing`, the sequence_ranges
                                 // shouldn't contain the special tokens.
                                 let pair_sequence_ranges =
-                                    HashMap::from_iter(vec![(1, 0..pair_ids.len() - 1)]);
+                                    FxHashMap::from_iter(vec![(1, 0..pair_ids.len() - 1)]);
                                 Encoding::new(
                                     pair_ids,
                                     pair_type_ids,
@@ -191,6 +192,8 @@ impl PostProcessor for BertProcessing {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashMap;
+
     use super::*;
 
     #[test]
@@ -236,7 +239,7 @@ mod tests {
                 vec![1, 0, 0, 1],
                 vec![1, 1, 1, 1],
                 vec![],
-                HashMap::from_iter(vec![(0, 1..3)]),
+                HashMap::from([(0, 1..3)]),
             )
         );
         assert_eq!(single_encoding.token_to_sequence(2), Some(0));
@@ -262,7 +265,7 @@ mod tests {
                 vec![1, 0, 0, 1, 0, 1],
                 vec![1, 1, 1, 1, 1, 1],
                 vec![],
-                HashMap::from_iter(vec![(0, 1..3), (1, 4..5)]),
+                HashMap::from([(0, 1..3), (1, 4..5)]),
             )
         );
         assert_eq!(pair_encoding.token_to_sequence(2), Some(0));
@@ -283,7 +286,7 @@ mod tests {
                 vec![0, 0, 0],
                 vec![1, 1, 1],
                 vec![],
-                HashMap::from_iter(vec![(0, 0..2), (1, 2..3)]),
+                HashMap::from([(0, 0..2), (1, 2..3)]),
             )
         );
         assert_eq!(pair_encoding.token_to_sequence(0), Some(0));
diff --git a/tokenizers/src/processors/roberta.rs b/tokenizers/src/processors/roberta.rs
index 5bbc4ea63..9dda972f8 100644
--- a/tokenizers/src/processors/roberta.rs
+++ b/tokenizers/src/processors/roberta.rs
@@ -1,7 +1,7 @@
 use crate::processors::byte_level::process_offsets;
 use crate::tokenizer::{Encoding, PostProcessor, Result};
+use rustc_hash::FxHashMap;
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
 use std::iter::FromIterator;
 
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
@@ -108,7 +108,7 @@ impl PostProcessor for RobertaProcessing {
 
                     // For compatibility with `TemplateProcessing`, the sequence_ranges shouldn't contain
                     // the special tokens.
-                    let sequence_ranges = HashMap::from_iter(vec![(0, 1..ids.len() - 1)]);
+                    let sequence_ranges = FxHashMap::from_iter(vec![(0, 1..ids.len() - 1)]);
                     Encoding::new(
                         ids,
                         type_ids,
@@ -141,7 +141,7 @@ impl PostProcessor for RobertaProcessing {
                                 // For compatibility with `TemplateProcessing`, the sequence_ranges shouldn't
                                 // contain the special tokens.
                                 let sequence_ranges =
-                                    HashMap::from_iter(vec![(0, 1..ids.len() - 1)]);
+                                    FxHashMap::from_iter(vec![(0, 1..ids.len() - 1)]);
                                 Encoding::new(
                                     ids,
                                     type_ids,
@@ -174,7 +174,8 @@ impl PostProcessor for RobertaProcessing {
 
                     // For compatibility with `TemplateProcessing`, the sequence_ranges shouldn't contain
                     // the special tokens.
-                    let pair_sequence_ranges = HashMap::from_iter(vec![(1, 1..pair_ids.len() - 1)]);
+                    let pair_sequence_ranges =
+                        FxHashMap::from_iter(vec![(1, 1..pair_ids.len() - 1)]);
                     Encoding::new(
                         pair_ids,
                         pair_type_ids,
@@ -208,7 +209,7 @@ impl PostProcessor for RobertaProcessing {
                                 // For compatibility with `TemplateProcessing`, the sequence_ranges
                                 // shouldn't contain the special tokens.
                                 let pair_sequence_ranges =
-                                    HashMap::from_iter(vec![(1, 1..pair_ids.len() - 1)]);
+                                    FxHashMap::from_iter(vec![(1, 1..pair_ids.len() - 1)]);
                                 Encoding::new(
                                     pair_ids,
                                     pair_type_ids,
@@ -234,6 +235,8 @@ impl PostProcessor for RobertaProcessing {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashMap;
+
     use super::*;
 
     #[test]
@@ -281,7 +284,7 @@ mod tests {
                 vec![1, 0, 0, 1],
                 vec![1, 1, 1, 1],
                 vec![],
-                HashMap::from_iter(vec![(0, 1..3)]),
+                HashMap::from([(0, 1..3)]),
             )
         );
         assert_eq!(single_encoding.token_to_sequence(2), Some(0));
@@ -308,7 +311,7 @@ mod tests {
                 vec![1, 0, 0, 1, 1, 0, 1],
                 vec![1, 1, 1, 1, 1, 1, 1],
                 vec![],
-                HashMap::from_iter(vec![(0, 1..3), (1, 5..6)]),
+                HashMap::from([(0, 1..3), (1, 5..6)]),
             )
         );
         assert_eq!(pair_encoding.token_to_sequence(2), Some(0));
@@ -330,7 +333,7 @@ mod tests {
                 vec![0, 0, 0],
                 vec![1, 1, 1],
                 vec![],
-                HashMap::from_iter(vec![(0, 0..2), (1, 2..3)]),
+                HashMap::from([(0, 0..2), (1, 2..3)]),
             )
         );
         assert_eq!(pair_encoding.token_to_sequence(0), Some(0));
diff --git a/tokenizers/src/processors/sequence.rs b/tokenizers/src/processors/sequence.rs
index 5cfb3eb51..ff7b17c8d 100644
--- a/tokenizers/src/processors/sequence.rs
+++ b/tokenizers/src/processors/sequence.rs
@@ -74,7 +74,6 @@ mod tests {
     use crate::processors::{ByteLevel, PostProcessorWrapper};
     use crate::tokenizer::{Encoding, PostProcessor};
     use std::collections::HashMap;
-    use std::iter::FromIterator;
 
     #[test]
     fn process_chain() {
@@ -113,7 +112,7 @@ mod tests {
             vec![],
             vec![],
             vec![],
-            HashMap::from_iter(vec![(0, 0..5)]),
+            HashMap::from([(0, 0..5)]),
         );
 
         assert_eq!(
@@ -156,7 +155,7 @@ mod tests {
             vec![],
             vec![],
             vec![],
-            HashMap::from_iter(vec![(0, 0..5), (1, 5..10)]),
+            HashMap::from([(0, 0..5), (1, 5..10)]),
         );
         assert_eq!(
             pair_expected,
diff --git a/tokenizers/src/processors/template.rs b/tokenizers/src/processors/template.rs
index 6c9cf9a74..8a898ea5f 100644
--- a/tokenizers/src/processors/template.rs
+++ b/tokenizers/src/processors/template.rs
@@ -58,9 +58,13 @@
 //!
 use crate::{Encoding, PostProcessor, Result};
 use itertools::Itertools;
+use rustc_hash::FxHashMap;
+use rustc_hash::FxHashSet;
 use serde::{Deserialize, Serialize};
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
 use std::convert::{TryFrom, TryInto};
+use std::hash::BuildHasher;
+use std::iter::FromIterator;
 use std::result::Result as StdResult;
 
 /// Represents any sequences received as input of the PostProcessor
@@ -293,7 +297,7 @@ impl TryFrom<&str> for Template {
 #[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, Eq)]
 #[serde(transparent)]
 pub struct Tokens(
-    #[serde(serialize_with = "crate::utils::ordered_map")] pub HashMap<String, SpecialToken>,
+    #[serde(serialize_with = "crate::utils::ordered_map")] pub FxHashMap<String, SpecialToken>,
 );
 
 impl<T: Into<SpecialToken>> From<Vec<T>> for Tokens {
@@ -309,9 +313,9 @@ impl<T: Into<SpecialToken>> From<Vec<T>> for Tokens {
     }
 }
 
-impl From<HashMap<String, SpecialToken>> for Tokens {
-    fn from(v: HashMap<String, SpecialToken>) -> Self {
-        Self(v)
+impl<S: BuildHasher> From<HashMap<String, SpecialToken, S>> for Tokens {
+    fn from(v: HashMap<String, SpecialToken, S>) -> Self {
+        Self(FxHashMap::from_iter(v))
     }
 }
 
@@ -502,7 +506,7 @@ impl TemplateProcessingBuilder {
         };
 
         let empty = [];
-        let missing: HashSet<&str> = self
+        let missing: FxHashSet<&str> = self
             .single
             .as_ref()
             .map_or(empty.iter(), |s| s.0.iter())
@@ -511,7 +515,7 @@ impl TemplateProcessingBuilder {
                 Piece::Sequence { .. } => None,
                 Piece::SpecialToken { id, .. } => check(id.as_ref()),
             })
-            .collect::<HashSet<_>>();
+            .collect::<FxHashSet<_>>();
 
         if missing.is_empty() {
             Ok(())
@@ -578,7 +582,7 @@ impl TemplateProcessing {
                                 // overflowing
                                 vec![],
                                 // sequence_range
-                                HashMap::new(),
+                                FxHashMap::default(),
                             );
                             Some(encoding)
                         } else {
@@ -688,8 +692,8 @@ impl PostProcessor for TemplateProcessing {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::collections::HashMap;
     use std::convert::TryInto;
-    use std::iter::FromIterator;
 
     #[test]
     fn piece_serde() {
@@ -917,7 +921,7 @@ mod tests {
                 vec![1, 0, 0, 1],
                 vec![1, 1, 1, 1],
                 vec![],
-                HashMap::from_iter(vec![(0, 1..3)]),
+                HashMap::from([(0, 1..3)]),
             )
         );
         assert_eq!(single_encoding.token_to_sequence(2), Some(0));
@@ -941,7 +945,7 @@ mod tests {
                 vec![1, 0, 0, 1, 0, 1],
                 vec![1, 1, 1, 1, 1, 1],
                 vec![],
-                HashMap::from_iter(vec![(0, 1..3), (1, 4..5)]),
+                HashMap::from([(0, 1..3), (1, 4..5)]),
             )
         );
         assert_eq!(pair_encoding.token_to_sequence(2), Some(0));
@@ -1003,9 +1007,9 @@ mod tests {
                     vec![1, 0, 1],
                     vec![1, 1, 1],
                     vec![],
-                    HashMap::from_iter(vec![(0, 1..2)]),
+                    HashMap::from([(0, 1..2)]),
                 )],
-                HashMap::from_iter(vec![(0, 1..3)]),
+                HashMap::from([(0, 1..3)]),
             )
         );
         assert_eq!(single_encoding.token_to_sequence(2), Some(0));
@@ -1061,9 +1065,9 @@ mod tests {
                             vec![1, 0, 1, 0, 1],
                             vec![1, 1, 1, 1, 1],
                             vec![],
-                            HashMap::from_iter(vec![(0, 1..2), (1, 3..4)]),
+                            HashMap::from([(0, 1..2), (1, 3..4)]),
                         ),],
-                        HashMap::from_iter(vec![(1, 3..5), (0, 1..2)]),
+                        HashMap::from([(1, 3..5), (0, 1..2)]),
                     ),
                     Encoding::new(
                         vec![1, 13, 0, 17, 0],
@@ -1080,7 +1084,7 @@ mod tests {
                         vec![1, 0, 1, 0, 1],
                         vec![1, 1, 1, 1, 1],
                         vec![],
-                        HashMap::from_iter(vec![(0, 1..2), (1, 3..4)]),
+                        HashMap::from([(0, 1..2), (1, 3..4)]),
                     ),
                     Encoding::new(
                         vec![1, 12, 14, 0, 17, 0],
@@ -1112,12 +1116,12 @@ mod tests {
                             vec![1, 0, 1, 0, 1],
                             vec![1, 1, 1, 1, 1],
                             vec![],
-                            HashMap::from_iter(vec![(0, 1..2), (1, 3..4)]),
+                            HashMap::from([(0, 1..2), (1, 3..4)]),
                         ),],
-                        HashMap::from_iter(vec![(0, 1..3), (1, 4..5)]),
+                        HashMap::from([(0, 1..3), (1, 4..5)]),
                     )
                 ],
-                HashMap::from_iter(vec![(0, 1..3), (1, 4..6)]),
+                HashMap::from([(0, 1..3), (1, 4..6)]),
             )
         );
         assert_eq!(pair_encoding.token_to_sequence(2), Some(0));
diff --git a/tokenizers/src/tokenizer/added_vocabulary.rs b/tokenizers/src/tokenizer/added_vocabulary.rs
index f988477be..18ff91359 100644
--- a/tokenizers/src/tokenizer/added_vocabulary.rs
+++ b/tokenizers/src/tokenizer/added_vocabulary.rs
@@ -3,8 +3,9 @@ use super::{
 };
 use aho_corasick::{AhoCorasick, AhoCorasickBuilder, MatchKind};
 use regex::Regex;
+use rustc_hash::FxHashMap;
+use rustc_hash::FxHashSet;
 use serde::{ser::SerializeSeq, Deserialize, Serialize, Serializer};
-use std::collections::{HashMap, HashSet};
 use std::sync::LazyLock;
 
 /// Represent a token added by the user on top of the existing Model vocabulary.
@@ -141,10 +142,10 @@ fn space_rightmost_at_start(sentence: &str) -> usize {
 pub struct AddedVocabulary {
     /// Contains the mapping from String (token content) to ID. This map contains both special
     /// tokens and classic added tokens that were added to the this vocabulary.
-    added_tokens_map: HashMap<String, u32>,
+    added_tokens_map: FxHashMap<String, u32>,
     /// Contains the mapping from ID to AddedToken for all the added tokens, both special
     /// and classic.
-    added_tokens_map_r: HashMap<u32, AddedToken>,
+    added_tokens_map_r: FxHashMap<u32, AddedToken>,
 
     /// Contains only the classic AddedToken, in the specific order the user gave them.
     added_tokens: Vec<AddedToken>,
@@ -153,7 +154,7 @@ pub struct AddedVocabulary {
 
     /// A Set, containing all the special token for easy access while decoding. This let's
     /// us remove them easily with an O(1) complexity.
-    special_tokens_set: HashSet<String>,
+    special_tokens_set: FxHashSet<String>,
 
     /// A RegexSet containing all the non-normalized patterns used to split on AddedTokens
     split_trie: MatchingSet,
@@ -175,11 +176,11 @@ impl AddedVocabulary {
             .build::<_, &&[u8]>([])
             .expect("The normalized trie should build correctly");
         Self {
-            added_tokens_map: HashMap::new(),
-            added_tokens_map_r: HashMap::new(),
+            added_tokens_map: FxHashMap::default(),
+            added_tokens_map_r: FxHashMap::default(),
             added_tokens: vec![],
             special_tokens: vec![],
-            special_tokens_set: HashSet::new(),
+            special_tokens_set: FxHashSet::default(),
             split_trie: (trie, vec![]),
             split_normalized_trie: (normalized_trie, vec![]),
             encode_special_tokens: false,
@@ -197,12 +198,12 @@ impl AddedVocabulary {
     }
 
     /// Get the additional vocabulary
-    pub fn get_vocab(&self) -> &HashMap<String, u32> {
+    pub fn get_vocab(&self) -> &FxHashMap<String, u32> {
         &self.added_tokens_map
     }
 
     /// Get the additional vocabulary with the AddedTokens
-    pub fn get_added_tokens_decoder(&self) -> &HashMap<u32, AddedToken> {
+    pub fn get_added_tokens_decoder(&self) -> &FxHashMap<u32, AddedToken> {
         &self.added_tokens_map_r
     }
 
@@ -546,19 +547,20 @@ mod tests {
     use crate::normalizers::utils::Lowercase;
     use crate::normalizers::NormalizerWrapper;
     use crate::{OffsetReferential, OffsetType, Result, Token, Trainer};
+    use std::iter::FromIterator;
     use std::path::{Path, PathBuf};
 
     #[derive(Serialize, Deserialize)]
     struct ModelMock {
-        vocab: HashMap<String, u32>,
-        vocab_r: HashMap<u32, String>,
+        vocab: FxHashMap<String, u32>,
+        vocab_r: FxHashMap<u32, String>,
     }
     impl ModelMock {
         pub fn new<I>(iter: I) -> Self
         where
             I: IntoIterator<Item = &'static (&'static str, u32)>,
         {
-            let vocab: HashMap<String, u32> = iter
+            let vocab: FxHashMap<String, u32> = iter
                 .into_iter()
                 .map(|&(tok, id)| (tok.to_string(), id))
                 .collect();
@@ -618,7 +620,7 @@ mod tests {
         fn id_to_token(&self, id: u32) -> Option<String> {
             self.vocab_r.get(&id).cloned()
         }
-        fn get_vocab(&self) -> HashMap<String, u32> {
+        fn get_vocab(&self) -> FxHashMap<String, u32> {
             self.vocab.clone()
         }
         fn get_vocab_size(&self) -> usize {
@@ -715,7 +717,7 @@ mod tests {
         assert!(vocab.is_special_token("test"));
         assert_eq!(
             *vocab.get_added_tokens_decoder(),
-            HashMap::from([
+            FxHashMap::from_iter([
                 (0, AddedToken::from("test", true)),
                 (2, AddedToken::from("added_token_1", true)),
                 (3, AddedToken::from("added_token_2", true)),
diff --git a/tokenizers/src/tokenizer/encoding.rs b/tokenizers/src/tokenizer/encoding.rs
index 1732686e4..140abde77 100644
--- a/tokenizers/src/tokenizer/encoding.rs
+++ b/tokenizers/src/tokenizer/encoding.rs
@@ -2,8 +2,11 @@ use crate::parallelism::*;
 use crate::tokenizer::{Offsets, Token};
 use crate::utils::padding::PaddingDirection;
 use crate::utils::truncation::TruncationDirection;
+use rustc_hash::FxHashMap;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
+use std::hash::BuildHasher;
+use std::iter::FromIterator;
 use std::ops::Range;
 
 /// Represents the output of a `Tokenizer`.
@@ -27,11 +30,11 @@ pub struct Encoding {
     overflowing: Vec<Encoding>,
     /// Ranges of tokens covered by each sequence. If this is empty we consider
     /// there is only one sequence in this Encoding, and that it covers the entire range.
-    sequence_ranges: HashMap<usize, Range<usize>>,
+    sequence_ranges: FxHashMap<usize, Range<usize>>,
 }
 impl Encoding {
     #[allow(clippy::too_many_arguments)]
-    pub fn new(
+    pub fn new<S: BuildHasher>(
         ids: Vec<u32>,
         type_ids: Vec<u32>,
         tokens: Vec<String>,
@@ -40,7 +43,7 @@ impl Encoding {
         special_tokens_mask: Vec<u32>,
         attention_mask: Vec<u32>,
         overflowing: Vec<Self>,
-        sequence_ranges: HashMap<usize, Range<usize>>,
+        sequence_ranges: HashMap<usize, Range<usize>, S>,
     ) -> Self {
         Self {
             ids,
@@ -51,7 +54,7 @@ impl Encoding {
             special_tokens_mask,
             attention_mask,
             overflowing,
-            sequence_ranges,
+            sequence_ranges: FxHashMap::from_iter(sequence_ranges),
         }
     }
 
@@ -65,7 +68,7 @@ impl Encoding {
             special_tokens_mask: Vec::with_capacity(len),
             attention_mask: Vec::with_capacity(len),
             overflowing: vec![],
-            sequence_ranges: HashMap::new(),
+            sequence_ranges: FxHashMap::default(),
         }
     }
 
@@ -94,7 +97,7 @@ impl Encoding {
             attention_mask: vec![1; length],
             special_tokens_mask: vec![0; length],
             overflowing: vec![],
-            sequence_ranges: HashMap::new(),
+            sequence_ranges: FxHashMap::default(),
         }
     }
 
@@ -363,7 +366,7 @@ impl Encoding {
             special_tokens_mask: self.special_tokens_mask[start..stop].to_vec(),
             attention_mask: self.attention_mask[start..stop].to_vec(),
             overflowing: vec![],
-            sequence_ranges: HashMap::new(),
+            sequence_ranges: FxHashMap::default(),
         };
 
         loop {
@@ -381,7 +384,7 @@ impl Encoding {
                 special_tokens_mask: self.special_tokens_mask[start..stop].to_vec(),
                 attention_mask: self.attention_mask[start..stop].to_vec(),
                 overflowing: vec![],
-                sequence_ranges: HashMap::new(),
+                sequence_ranges: FxHashMap::default(),
             });
         }
         *self = new_encoding;
@@ -563,9 +566,10 @@ impl std::iter::FromIterator<(u32, String, (usize, usize), Option<u32>, u32)> fo
 
 #[cfg(test)]
 mod tests {
-    use super::*;
     use std::iter::FromIterator;
 
+    use super::*;
+
     #[test]
     fn merge_encodings() {
         let mut a = Encoding {
@@ -890,7 +894,7 @@ mod tests {
             offsets: vec![(0, 6)],
             special_tokens_mask: vec![0],
             attention_mask: vec![1],
-            sequence_ranges: HashMap::from([(0, 0..1)]),
+            sequence_ranges: HashMap::from_iter([(0, 0..1)]),
             ..Default::default()
         };
         let target_length = 2;
@@ -904,6 +908,6 @@ mod tests {
             pad_token,
             PaddingDirection::Left,
         );
-        assert_eq!(a.sequence_ranges, HashMap::from([(0, 1..2)]));
+        assert_eq!(a.sequence_ranges, HashMap::from_iter([(0, 1..2)]));
     }
 }
diff --git a/tokenizers/src/tokenizer/mod.rs b/tokenizers/src/tokenizer/mod.rs
index 808d120d5..ba26a75fe 100644
--- a/tokenizers/src/tokenizer/mod.rs
+++ b/tokenizers/src/tokenizer/mod.rs
@@ -9,8 +9,8 @@
 //!   - [`PostProcessor`](trait.PostProcessor.html): Takes care of the processing after tokenization (like truncating, padding,
 //!     ...).
 
+use rustc_hash::FxHashMap;
 use std::{
-    collections::HashMap,
     fs::{read_to_string, File},
     io::{prelude::*, BufReader},
     ops::{Deref, DerefMut},
@@ -77,7 +77,7 @@ pub trait Model {
     /// Find the string token associated to an ID
     fn id_to_token(&self, id: u32) -> Option<String>;
     /// Retrieve the entire vocabulary mapping (token -> ID)
-    fn get_vocab(&self) -> HashMap<String, u32>;
+    fn get_vocab(&self) -> FxHashMap<String, u32>;
     /// Retrieve the size of the vocabulary
     fn get_vocab_size(&self) -> usize;
     /// Save the current `Model` in the given folder, using the given `prefix` for the various
@@ -658,7 +658,7 @@ where
     }
 
     /// Get the vocabulary
-    pub fn get_vocab(&self, with_added_tokens: bool) -> HashMap<String, u32> {
+    pub fn get_vocab(&self, with_added_tokens: bool) -> FxHashMap<String, u32> {
         let mut final_vocab = self.model.get_vocab();
 
         if with_added_tokens {
@@ -675,7 +675,7 @@ where
     }
 
     /// Get the added tokens decoder
-    pub fn get_added_tokens_decoder(&self) -> HashMap<u32, AddedToken> {
+    pub fn get_added_tokens_decoder(&self) -> FxHashMap<u32, AddedToken> {
         self.added_vocabulary.get_added_tokens_decoder().clone()
     }
 
@@ -947,7 +947,7 @@ where
 /// use std::collections::HashMap;
 /// use std::iter::FromIterator;
 ///
-/// let vocab = HashMap::from_iter([
+/// let vocab = HashMap::from([
 ///     ("<0x20>".to_string(), 0),
 ///     ("<0xC3>".to_string(), 1),
 ///     ("<0xA9>".to_string(), 2),
@@ -984,7 +984,7 @@ where
 /// use std::collections::HashMap;
 /// use std::iter::FromIterator;
 ///
-/// let vocab = HashMap::from_iter([
+/// let vocab = HashMap::from([
 ///     ("▁This".to_string(), 0),
 /// ]);
 /// let merges = vec![];
diff --git a/tokenizers/src/tokenizer/pre_tokenizer.rs b/tokenizers/src/tokenizer/pre_tokenizer.rs
index 0d54cd62b..f777e28a1 100644
--- a/tokenizers/src/tokenizer/pre_tokenizer.rs
+++ b/tokenizers/src/tokenizer/pre_tokenizer.rs
@@ -1,7 +1,7 @@
 use crate::{
     normalizer::Range, Encoding, NormalizedString, OffsetReferential, Offsets, Result, Token,
 };
-use std::collections::HashMap;
+use rustc_hash::FxHashMap;
 
 /// Various possible types of offsets
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -265,7 +265,7 @@ impl From<String> for PreTokenizedString {
 }
 
 struct BytesToCharOffsetConverter {
-    map: HashMap<usize, usize>,
+    map: FxHashMap<usize, usize>,
 }
 
 impl BytesToCharOffsetConverter {
diff --git a/tokenizers/src/utils/cache.rs b/tokenizers/src/utils/cache.rs
index 002fb1d61..3f62a2d1a 100644
--- a/tokenizers/src/utils/cache.rs
+++ b/tokenizers/src/utils/cache.rs
@@ -1,5 +1,5 @@
+use rustc_hash::FxHashMap;
 use std::borrow::Borrow;
-use std::collections::HashMap;
 use std::hash::Hash;
 use std::sync::RwLock;
 
@@ -19,7 +19,7 @@ where
     K: Eq + Hash + Clone,
     V: Clone,
 {
-    map: RwLock<HashMap<K, V>>,
+    map: RwLock<FxHashMap<K, V>>,
     pub capacity: usize,
 }
 
@@ -51,7 +51,10 @@ where
 {
     /// Create new `Cache` with the given capacity.
     pub(crate) fn new(capacity: usize) -> Self {
-        let map = RwLock::new(HashMap::with_capacity(capacity));
+        let map = RwLock::new(FxHashMap::with_capacity_and_hasher(
+            capacity,
+            Default::default(),
+        ));
         Cache { map, capacity }
     }
 
diff --git a/tokenizers/src/utils/from_pretrained.rs b/tokenizers/src/utils/from_pretrained.rs
index 223fbbeb9..dd6849e66 100644
--- a/tokenizers/src/utils/from_pretrained.rs
+++ b/tokenizers/src/utils/from_pretrained.rs
@@ -1,13 +1,13 @@
 use crate::Result;
 use hf_hub::{api::sync::ApiBuilder, Repo, RepoType};
-use std::collections::HashMap;
+use rustc_hash::FxHashMap;
 use std::path::PathBuf;
 
 /// Defines the additional parameters available for the `from_pretrained` function
 #[derive(Debug, Clone)]
 pub struct FromPretrainedParameters {
     pub revision: String,
-    pub user_agent: HashMap<String, String>,
+    pub user_agent: FxHashMap<String, String>,
     pub token: Option<String>,
 }
 
@@ -15,7 +15,7 @@ impl Default for FromPretrainedParameters {
     fn default() -> Self {
         Self {
             revision: "main".into(),
-            user_agent: HashMap::new(),
+            user_agent: FxHashMap::default(),
             token: None,
         }
     }
diff --git a/tokenizers/src/utils/mod.rs b/tokenizers/src/utils/mod.rs
index deda862eb..7204917cb 100644
--- a/tokenizers/src/utils/mod.rs
+++ b/tokenizers/src/utils/mod.rs
@@ -18,16 +18,20 @@ pub(crate) mod progress;
 pub mod truncation;
 
 use serde::{Serialize, Serializer};
-use std::collections::{BTreeMap, HashMap};
+use std::{
+    collections::{BTreeMap, HashMap},
+    hash::BuildHasher,
+};
 
-pub(crate) fn ordered_map<S, K, V>(
-    value: &HashMap<K, V>,
+pub(crate) fn ordered_map<S, K, V, H>(
+    value: &HashMap<K, V, H>,
     serializer: S,
 ) -> std::result::Result<S::Ok, S::Error>
 where
     S: Serializer,
     K: Serialize + std::cmp::Ord,
     V: Serialize,
+    H: BuildHasher,
 {
     let ordered: BTreeMap<_, _> = value.iter().collect();
     ordered.serialize(serializer)
diff --git a/tokenizers/src/utils/padding.rs b/tokenizers/src/utils/padding.rs
index 39585a304..eb3debc08 100644
--- a/tokenizers/src/utils/padding.rs
+++ b/tokenizers/src/utils/padding.rs
@@ -82,9 +82,10 @@ pub fn pad_encodings(encodings: &mut [Encoding], params: &PaddingParams) -> Resu
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashMap;
+
     use super::*;
     use crate::tokenizer::Encoding;
-    use std::collections::HashMap;
 
     #[test]
     fn pad_to_multiple() {
diff --git a/tokenizers/src/utils/truncation.rs b/tokenizers/src/utils/truncation.rs
index 9acc297bf..5bdda1ca1 100644
--- a/tokenizers/src/utils/truncation.rs
+++ b/tokenizers/src/utils/truncation.rs
@@ -168,9 +168,10 @@ pub fn truncate_encodings(
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashMap;
+
     use super::*;
     use crate::tokenizer::Encoding;
-    use std::collections::HashMap;
 
     fn get_empty() -> Encoding {
         Encoding::new(
diff --git a/tokenizers/tests/documentation.rs b/tokenizers/tests/documentation.rs
index 304211e77..186def3d7 100644
--- a/tokenizers/tests/documentation.rs
+++ b/tokenizers/tests/documentation.rs
@@ -1,5 +1,4 @@
 use std::collections::HashMap;
-use std::iter::FromIterator;
 
 use tokenizers::decoders::byte_fallback::ByteFallback;
 use tokenizers::models::bpe::{BpeTrainerBuilder, BPE};
@@ -91,7 +90,7 @@ fn streaming_tokenizer() {
     );
 
     // None example
-    let vocab = HashMap::from_iter([
+    let vocab = HashMap::from([
         ("<0x20>".to_string(), 0),
         ("<0xC3>".to_string(), 1),
         ("<0xA9>".to_string(), 2),