diff --git a/Cargo.lock b/Cargo.lock index 6d5c07bb489..15283a5791d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2111,7 +2111,7 @@ dependencies = [ [[package]] name = "kvproto" version = "0.0.2" -source = "git+https://github.com/pingcap/kvproto.git#8def300426bc8733f065588b5c5826da7452096f" +source = "git+https://github.com/pingcap/kvproto.git#dc1709169bb155de3bea6b28c871215387942994" dependencies = [ "futures 0.3.15", "grpcio", diff --git a/cmd/tikv-ctl/src/main.rs b/cmd/tikv-ctl/src/main.rs index 317ec8dbddb..51acd2a0a3f 100644 --- a/cmd/tikv-ctl/src/main.rs +++ b/cmd/tikv-ctl/src/main.rs @@ -35,6 +35,7 @@ use raft_log_engine::RaftLogEngine; use raftstore::store::INIT_EPOCH_CONF_VER; use regex::Regex; use security::{SecurityConfig, SecurityManager}; +use serde_json::json; use server::setup::initial_logger; use std::borrow::ToOwned; use std::cmp::Ordering; @@ -180,7 +181,7 @@ trait DebugExecutor { } fn dump_all_region_size(&self, cfs: Vec<&str>) { - let regions = self.get_all_meta_regions(); + let regions = self.get_all_regions_in_store(); let regions_number = regions.len(); let mut total_size = 0; for region in regions { @@ -190,30 +191,71 @@ trait DebugExecutor { println!("total region size: {}", convert_gbmb(total_size as u64)); } - fn dump_region_info(&self, region: u64, skip_tombstone: bool) { - let r = self.get_region_info(region); - if skip_tombstone { - let region_state = r.region_local_state.as_ref(); - if region_state.map_or(false, |s| s.get_state() == PeerState::Tombstone) { - return; + fn dump_region_info(&self, region_ids: Option>, skip_tombstone: bool) { + let region_ids = region_ids.unwrap_or_else(|| self.get_all_regions_in_store()); + let mut region_objects = serde_json::map::Map::new(); + for region_id in region_ids { + let r = self.get_region_info(region_id); + if skip_tombstone { + let region_state = r.region_local_state.as_ref(); + if region_state.map_or(false, |s| s.get_state() == PeerState::Tombstone) { + return; + } } + let region_object = json!({ + "region_id": region_id, + "region_local_state": r.region_local_state.map(|s| { + let r = s.get_region(); + let region_epoch = r.get_region_epoch(); + let peers = r.get_peers(); + json!({ + "region": json!({ + "id": r.get_id(), + "start_key": hex::encode_upper(r.get_start_key()), + "end_key": hex::encode_upper(r.get_end_key()), + "region_epoch": json!({ + "conf_ver": region_epoch.get_conf_ver(), + "version": region_epoch.get_version() + }), + "peers": peers.iter().map(|p| json!({ + "id": p.get_id(), + "store_id": p.get_store_id(), + "role": format!("{:?}", p.get_role()), + })).collect::>(), + }), + }) + }), + "raft_local_state": r.raft_local_state.map(|s| { + let hard_state = s.get_hard_state(); + json!({ + "hard_state": json!({ + "term": hard_state.get_term(), + "vote": hard_state.get_vote(), + "commit": hard_state.get_commit(), + }), + "last_index": s.get_last_index(), + }) + }), + "raft_apply_state": r.raft_apply_state.map(|s| { + let truncated_state = s.get_truncated_state(); + json!({ + "applied_index": s.get_applied_index(), + "commit_index": s.get_commit_index(), + "commit_term": s.get_commit_term(), + "truncated_state": json!({ + "index": truncated_state.get_index(), + "term": truncated_state.get_term(), + }) + }) + }) + }); + region_objects.insert(region_id.to_string(), region_object); } - let region_state_key = keys::region_state_key(region); - let raft_state_key = keys::raft_state_key(region); - let apply_state_key = keys::apply_state_key(region); - println!("region id: {}", region); - println!("region state key: {}", escape(®ion_state_key)); - println!("region state: {:?}", r.region_local_state); - println!("raft state key: {}", escape(&raft_state_key)); - println!("raft state: {:?}", r.raft_local_state); - println!("apply state key: {}", escape(&apply_state_key)); - println!("apply state: {:?}", r.raft_apply_state); - } - fn dump_all_region_info(&self, skip_tombstone: bool) { - for region in self.get_all_meta_regions() { - self.dump_region_info(region, skip_tombstone); - } + println!( + "{}", + serde_json::to_string_pretty(&json!({ "region_infos": region_objects })).unwrap() + ); } fn dump_raft_log(&self, region: u64, index: u64) { @@ -532,7 +574,14 @@ trait DebugExecutor { } /// Recover the cluster when given `store_ids` are failed. - fn remove_fail_stores(&self, store_ids: Vec, region_ids: Option>); + fn remove_fail_stores( + &self, + store_ids: Vec, + region_ids: Option>, + promote_learner: bool, + ); + + fn drop_unapplied_raftlog(&self, region_ids: Option>); /// Recreate the region with metadata from pd, but alloc new id for it. fn recreate_region(&self, sec_mgr: Arc, pd_cfg: &PdConfig, region_id: u64); @@ -572,7 +621,7 @@ trait DebugExecutor { self.recover_all(threads, read_only); } - fn get_all_meta_regions(&self) -> Vec; + fn get_all_regions_in_store(&self) -> Vec; fn get_value_by_key(&self, cf: &str, key: Vec) -> Vec; @@ -623,8 +672,10 @@ impl DebugExecutor for DebugClient { process::exit(-1); } - fn get_all_meta_regions(&self) -> Vec { - unimplemented!(); + fn get_all_regions_in_store(&self) -> Vec { + DebugClient::get_all_regions_in_store(self, &GetAllRegionsInStoreRequest::default()) + .unwrap_or_else(|e| perror_and_exit("DebugClient::get_all_regions_in_store", e)) + .take_regions() } fn get_value_by_key(&self, cf: &str, key: Vec) -> Vec { @@ -760,12 +811,16 @@ impl DebugExecutor for DebugClient { unimplemented!("only available for local mode"); } - fn remove_fail_stores(&self, _: Vec, _: Option>) { - self.check_local_mode(); + fn remove_fail_stores(&self, _: Vec, _: Option>, _: bool) { + unimplemented!("only available for local mode"); + } + + fn drop_unapplied_raftlog(&self, _: Option>) { + unimplemented!("only available for local mode"); } fn recreate_region(&self, _: Arc, _: &PdConfig, _: u64) { - self.check_local_mode(); + unimplemented!("only available for local mode"); } fn check_region_consistency(&self, region_id: u64) { @@ -820,9 +875,9 @@ impl DebugExecutor for DebugClient { impl DebugExecutor for Debugger { fn check_local_mode(&self) {} - fn get_all_meta_regions(&self) -> Vec { - self.get_all_meta_regions() - .unwrap_or_else(|e| perror_and_exit("Debugger::get_all_meta_regions", e)) + fn get_all_regions_in_store(&self) -> Vec { + self.get_all_regions_in_store() + .unwrap_or_else(|e| perror_and_exit("Debugger::get_all_regions_in_store", e)) } fn get_value_by_key(&self, cf: &str, key: Vec) -> Vec { @@ -938,9 +993,21 @@ impl DebugExecutor for Debugger { println!("all regions are healthy") } - fn remove_fail_stores(&self, store_ids: Vec, region_ids: Option>) { + fn remove_fail_stores( + &self, + store_ids: Vec, + region_ids: Option>, + promote_learner: bool, + ) { println!("removing stores {:?} from configurations...", store_ids); - self.remove_failed_stores(store_ids, region_ids) + self.remove_failed_stores(store_ids, region_ids, promote_learner) + .unwrap_or_else(|e| perror_and_exit("Debugger::remove_fail_stores", e)); + println!("success"); + } + + fn drop_unapplied_raftlog(&self, region_ids: Option>) { + println!("removing unapplied raftlog on region {:?} ...", region_ids); + self.drop_unapplied_raftlog(region_ids) .unwrap_or_else(|e| perror_and_exit("Debugger::remove_fail_stores", e)); println!("success"); } @@ -1202,10 +1269,25 @@ fn main() { SubCommand::with_name("region") .about("print region info") .arg( - Arg::with_name("region") - .short("r") + Arg::with_name("regions") + .aliases(&["region"]) + .required_unless("all-regions") + .conflicts_with("all-regions") .takes_value(true) - .help("Set the region id, if not specified, print all regions"), + .short("r") + .multiple(true) + .use_delimiter(true) + .require_delimiter(true) + .value_delimiter(",") + .help("Print info for these regions"), + ) + .arg( + Arg::with_name("all-regions") + .required_unless("regions") + .conflicts_with("regions") + .long("all-regions") + .takes_value(false) + .help("Print info for all regions"), ) .arg( Arg::with_name("skip-tombstone") @@ -1564,7 +1646,7 @@ fn main() { .about("Unsafely recover when the store can not start normally, this recover may lose data") .subcommand( SubCommand::with_name("remove-fail-stores") - .about("Unsafely recover the cluster when the majority replicas are failed") + .about("Remove the failed machines from the peer list for the regions") .arg( Arg::with_name("stores") .required(true) @@ -1588,6 +1670,37 @@ fn main() { .value_delimiter(",") .help("Only for these regions"), ) + .arg( + Arg::with_name("promote-learner") + .long("promote-learner") + .takes_value(false) + .required(false) + .help("Promote learner to voter"), + ) + .arg( + Arg::with_name("all-regions") + .required_unless("regions") + .conflicts_with("regions") + .long("all-regions") + .takes_value(false) + .help("Do the command for all regions"), + ) + ) + .subcommand( + SubCommand::with_name("drop-unapplied-raftlog") + .about("Remove unapplied raftlogs on the regions") + .arg( + Arg::with_name("regions") + .required_unless("all-regions") + .conflicts_with("all-regions") + .takes_value(true) + .short("r") + .multiple(true) + .use_delimiter(true) + .require_delimiter(true) + .value_delimiter(",") + .help("Only for these regions"), + ) .arg( Arg::with_name("all-regions") .required_unless("regions") @@ -1963,7 +2076,7 @@ fn main() { println!("{}", escape(&from_hex(hex).unwrap())); return; } else if let Some(escaped) = matches.value_of("escaped-to-hex") { - println!("{}", log_wrappers::hex_encode_upper(unescape(escaped))); + println!("{}", hex::encode_upper(unescape(escaped))); return; } else if let Some(encoded) = matches.value_of("decode") { match Key::from_encoded(unescape(encoded)).into_raw() { @@ -2104,11 +2217,13 @@ fn main() { debug_executor.dump_raft_log(id, index); } else if let Some(matches) = matches.subcommand_matches("region") { let skip_tombstone = matches.is_present("skip-tombstone"); - if let Some(id) = matches.value_of("region") { - debug_executor.dump_region_info(id.parse().unwrap(), skip_tombstone); - } else { - debug_executor.dump_all_region_info(skip_tombstone); - } + let regions = matches.values_of("regions").map(|values| { + values + .map(str::parse) + .collect::, _>>() + .expect("parse regions fail") + }); + debug_executor.dump_region_info(regions, skip_tombstone); } else { let _ = app.print_help(); } @@ -2246,7 +2361,18 @@ fn main() { .collect::, _>>() .expect("parse regions fail") }); - debug_executor.remove_fail_stores(store_ids, region_ids); + debug_executor.remove_fail_stores( + store_ids, + region_ids, + matches.is_present("promote-learner"), + ); + } else if let Some(matches) = matches.subcommand_matches("drop-unapplied-raftlog") { + let region_ids = matches.values_of("regions").map(|ids| { + ids.map(str::parse) + .collect::, _>>() + .expect("parse regions fail") + }); + debug_executor.drop_unapplied_raftlog(region_ids); } else { println!("{}", matches.usage()); } diff --git a/components/engine_traits/src/misc.rs b/components/engine_traits/src/misc.rs index 59fd4c5862e..193a1428e19 100644 --- a/components/engine_traits/src/misc.rs +++ b/components/engine_traits/src/misc.rs @@ -11,10 +11,17 @@ use crate::range::Range; #[derive(Clone, Debug)] pub enum DeleteStrategy { + /// Delete the SST files that are fullly fit in range. However, the SST files that are partially + /// overlapped with the range will not be touched. DeleteFiles, + /// Delete the data stored in Titan. DeleteBlobs, + /// Scan for keys and then delete. Useful when we know the keys in range are not too many. DeleteByKey, + /// Delete by range. Note that this is experimental and you should check whether it is enbaled + /// in config before using it. DeleteByRange, + /// Delete by ingesting a SST file with deletions. Useful when the number of ranges is too many. DeleteByWriter { sst_path: String }, } diff --git a/components/raftstore/src/store/worker/region.rs b/components/raftstore/src/store/worker/region.rs index e176db92e4d..6e496dede27 100644 --- a/components/raftstore/src/store/worker/region.rs +++ b/components/raftstore/src/store/worker/region.rs @@ -585,6 +585,7 @@ where fn delete_all_in_range(&self, ranges: &[Range]) -> Result<()> { for cf in self.engine.cf_names() { + // CF_LOCK usually contains fewer keys than other CFs, so we delete them by key. let strategy = if cf == CF_LOCK { DeleteStrategy::DeleteByKey } else if self.use_delete_range { diff --git a/components/server/Cargo.toml b/components/server/Cargo.toml index 14c30768161..3a882e7dfd0 100644 --- a/components/server/Cargo.toml +++ b/components/server/Cargo.toml @@ -15,8 +15,8 @@ sse = ["tikv/sse"] mem-profiling = ["tikv/mem-profiling"] failpoints = ["tikv/failpoints"] bcc-iosnoop = ["tikv/bcc-iosnoop"] -cloud-aws = [ "encryption_export/cloud-aws" ] -cloud-gcp = [ "encryption_export/cloud-gcp" ] +cloud-aws = ["encryption_export/cloud-aws"] +cloud-gcp = ["encryption_export/cloud-gcp"] protobuf-codec = [ "protobuf/bytes", "backup/protobuf-codec", @@ -66,7 +66,6 @@ test-engines-rocksdb = [ test-engines-panic = [ "tikv/test-engines-panic", ] - nortcheck = ["engine_rocks/nortcheck"] [dependencies] diff --git a/scripts/check-redact-log b/scripts/check-redact-log index 32103e89390..ec35f558bf1 100755 --- a/scripts/check-redact-log +++ b/scripts/check-redact-log @@ -7,12 +7,12 @@ function error_msg() { } if [[ "$(uname)" == "Darwin" ]] ; then - if grep -r -n --color=always --include '*.rs' --exclude hex.rs --exclude-dir target 'encode_upper' . | grep -v log_wrappers ; then + if grep -r -n --color=always --include '*.rs' --exclude hex.rs --exclude-dir tikv-ctl --exclude-dir target 'encode_upper' . | grep -v log_wrappers ; then error_msg exit 1 fi else - if grep -r -n -P '(? Debugger { } /// Get all regions holding region meta data from raft CF in KV storage. - pub fn get_all_meta_regions(&self) -> Result> { + pub fn get_all_regions_in_store(&self) -> Result> { let db = &self.engines.kv; let cf = CF_RAFT; let start_key = keys::REGION_META_MIN_KEY; @@ -151,6 +150,7 @@ impl Debugger { regions.push(id); Ok(true) })); + regions.sort_unstable(); Ok(regions) } @@ -555,6 +555,7 @@ impl Debugger { &self, store_ids: Vec, region_ids: Option>, + promote_learner: bool, ) -> Result<()> { let store_id = self.get_store_id()?; if store_ids.iter().any(|&s| s == store_id) { @@ -582,6 +583,43 @@ impl Debugger { let region_id = region_state.get_region().get_id(); let old_peers = region_state.mut_region().take_peers(); + + if promote_learner { + if new_peers + .iter() + .filter(|peer| peer.get_role() != PeerRole::Learner) + .count() + != 0 + { + // no need to promote learner, do nothing + } else if new_peers + .iter() + .filter(|peer| peer.get_role() == PeerRole::Learner) + .count() + > 1 + { + error!( + "failed to promote learner due to multiple learners, skip promote learner"; + "region_id" => region_id, + ) + } else { + for peer in &mut new_peers { + match peer.get_role() { + PeerRole::Voter + | PeerRole::IncomingVoter + | PeerRole::DemotingVoter => {} + PeerRole::Learner => { + info!( + "promote learner"; + "region_id" => region_id, + "peer_id" => peer.get_id(), + ); + peer.set_role(PeerRole::Voter); + } + } + } + } + } info!( "peers changed"; "region_id" => region_id, @@ -622,6 +660,73 @@ impl Debugger { Ok(()) } + pub fn drop_unapplied_raftlog(&self, region_ids: Option>) -> Result<()> { + let kv = &self.engines.kv; + let raft = &self.engines.raft; + + let region_ids = region_ids.unwrap_or(self.get_all_regions_in_store()?); + for region_id in region_ids { + let region_state = self.region_info(region_id)?; + + // It's safe to unwrap region_local_state here, because get_all_regions_in_store() + // guarantees that the region state exists in kvdb. + if region_state.region_local_state.unwrap().state == PeerState::Tombstone { + continue; + } + + let old_raft_local_state = region_state.raft_local_state.ok_or_else(|| { + Error::Other(format!("No RaftLocalState found for region {}", region_id).into()) + })?; + let old_raft_apply_state = region_state.raft_apply_state.ok_or_else(|| { + Error::Other(format!("No RaftApplyState found for region {}", region_id).into()) + })?; + + let applied_index = old_raft_apply_state.applied_index; + let last_index = old_raft_local_state.last_index; + + let new_raft_local_state = RaftLocalState { + last_index: applied_index, + ..old_raft_local_state.clone() + }; + let new_raft_apply_state = RaftApplyState { + commit_index: applied_index, + ..old_raft_apply_state.clone() + }; + + info!( + "dropping unapplied raft log"; + "region_id" => region_id, + "old_raft_local_state" => ?old_raft_local_state, + "new_raft_local_state" => ?new_raft_local_state, + "old_raft_apply_state" => ?old_raft_apply_state, + "new_raft_apply_state" => ?new_raft_apply_state, + ); + + // flush the changes + box_try!(kv.put_msg_cf( + CF_RAFT, + &keys::apply_state_key(region_id), + &new_raft_apply_state + )); + box_try!(raft.put_raft_state(region_id, &new_raft_local_state)); + let deleted_logs = box_try!(raft.gc(region_id, applied_index + 1, last_index + 1)); + raft.sync().unwrap(); + kv.sync().unwrap(); + + info!( + "dropped unapplied raft log"; + "region_id" => region_id, + "old_raft_local_state" => ?old_raft_local_state, + "new_raft_local_state" => ?new_raft_local_state, + "old_raft_apply_state" => ?old_raft_apply_state, + "new_raft_apply_state" => ?new_raft_apply_state, + "deleted logs" => deleted_logs, + ); + } + + Ok(()) + } + pub fn recreate_region(&self, region: Region) -> Result<()> { let region_id = region.get_id(); let kv = &self.engines.kv; @@ -749,16 +854,17 @@ impl Debugger { let mut res = dump_mvcc_properties(self.engines.kv.as_inner(), &start, &end)?; let middle_key = match box_try!(get_region_approximate_middle(&self.engines.kv, region)) { - Some(data_key) => { - let mut key = keys::origin_key(&data_key); - box_try!(bytes::decode_bytes(&mut key, false)) - } + Some(data_key) => keys::origin_key(&data_key).to_vec(), None => Vec::new(), }; - // Middle key of the range. res.push(( - "middle_key_by_approximate_size".to_owned(), + "region.start_key".to_owned(), + hex::encode(®ion.start_key), + )); + res.push(("region.end_key".to_owned(), hex::encode(®ion.end_key))); + res.push(( + "region.middle_key_by_approximate_size".to_owned(), hex::encode(&middle_key), )); @@ -809,7 +915,7 @@ fn dump_mvcc_properties(db: &Arc, start: &[u8], end: &[u8]) -> Result, region_id: u64, stores: &[u64]) -> Region { + fn init_region_state( + engine: &Arc, + region_id: u64, + stores: &[u64], + mut learner: usize, + ) -> Region { let mut region = Region::default(); region.set_id(region_id); for (i, &store_id) in stores.iter().enumerate() { let mut peer = Peer::default(); peer.set_id(i as u64); peer.set_store_id(store_id); + if learner > 0 { + peer.set_role(PeerRole::Learner); + learner -= 1; + } region.mut_peers().push(peer); } let mut region_state = RegionLocalState::default(); @@ -1248,6 +1363,28 @@ mod tests { region } + fn init_raft_state( + kv_engine: &RocksEngine, + raft_engine: &RocksEngine, + region_id: u64, + last_index: u64, + commit_index: u64, + applied_index: u64, + ) { + let apply_state_key = keys::apply_state_key(region_id); + let mut apply_state = RaftApplyState::default(); + apply_state.set_applied_index(applied_index); + apply_state.set_commit_index(commit_index); + kv_engine + .put_msg_cf(CF_RAFT, &apply_state_key, &apply_state) + .unwrap(); + + let raft_state_key = keys::raft_state_key(region_id); + let mut raft_state = RaftLocalState::default(); + raft_state.set_last_index(last_index); + raft_engine.put_msg(&raft_state_key, &raft_state).unwrap(); + } + fn get_region_state(engine: &Arc, region_id: u64) -> RegionLocalState { let key = keys::region_state_key(region_id); engine @@ -1269,33 +1406,33 @@ mod tests { // For normal case. assert!(region_overlap( &new_region(b"a", b"z"), - &new_region(b"b", b"y") + &new_region(b"b", b"y"), )); assert!(region_overlap( &new_region(b"a", b"n"), - &new_region(b"m", b"z") + &new_region(b"m", b"z"), )); assert!(!region_overlap( &new_region(b"a", b"m"), - &new_region(b"n", b"z") + &new_region(b"n", b"z"), )); // For the first or last region. assert!(region_overlap( &new_region(b"m", b""), - &new_region(b"a", b"n") + &new_region(b"a", b"n"), )); assert!(region_overlap( &new_region(b"a", b"n"), - &new_region(b"m", b"") + &new_region(b"m", b""), )); assert!(region_overlap( &new_region(b"", b""), - &new_region(b"m", b"") + &new_region(b"m", b""), )); assert!(!region_overlap( &new_region(b"a", b"m"), - &new_region(b"n", b"") + &new_region(b"n", b""), )); } @@ -1517,21 +1654,21 @@ mod tests { let engine = &debugger.engines.kv; // region 1 with peers at stores 11, 12, 13. - let region_1 = init_region_state(engine.as_inner(), 1, &[11, 12, 13]); + let region_1 = init_region_state(engine.as_inner(), 1, &[11, 12, 13], 0); // Got the target region from pd, which doesn't contains the store. let mut target_region_1 = region_1.clone(); target_region_1.mut_peers().remove(0); target_region_1.mut_region_epoch().set_conf_ver(100); // region 2 with peers at stores 11, 12, 13. - let region_2 = init_region_state(engine.as_inner(), 2, &[11, 12, 13]); + let region_2 = init_region_state(engine.as_inner(), 2, &[11, 12, 13], 0); // Got the target region from pd, which has different peer_id. let mut target_region_2 = region_2.clone(); target_region_2.mut_peers()[0].set_id(100); target_region_2.mut_region_epoch().set_conf_ver(100); // region 3 with peers at stores 21, 22, 23. - let region_3 = init_region_state(engine.as_inner(), 3, &[21, 22, 23]); + let region_3 = init_region_state(engine.as_inner(), 3, &[21, 22, 23], 0); // Got the target region from pd but the peers are not changed. let mut target_region_3 = region_3; target_region_3.mut_region_epoch().set_conf_ver(100); @@ -1575,7 +1712,7 @@ mod tests { assert!(!errors.is_empty()); // region 1 with peers at stores 11, 12, 13. - init_region_state(engine.as_inner(), 1, &[11, 12, 13]); + init_region_state(engine.as_inner(), 1, &[11, 12, 13], 0); let mut expected_state = get_region_state(engine.as_inner(), 1); expected_state.set_state(PeerState::Tombstone); @@ -1605,14 +1742,23 @@ mod tests { .collect::>() }; + let get_region_learner = |engine: &Arc, region_id: u64| { + get_region_state(engine, region_id) + .get_region() + .get_peers() + .iter() + .filter(|p| p.get_role() == PeerRole::Learner) + .count() + }; + // region 1 with peers at stores 11, 12, 13 and 14. - init_region_state(engine.as_inner(), 1, &[11, 12, 13, 14]); + init_region_state(engine.as_inner(), 1, &[11, 12, 13, 14], 0); // region 2 with peers at stores 21, 22 and 23. - init_region_state(engine.as_inner(), 2, &[21, 22, 23]); + init_region_state(engine.as_inner(), 2, &[21, 22, 23], 0); // Only remove specified stores from region 1. debugger - .remove_failed_stores(vec![13, 14, 21, 23], Some(vec![1])) + .remove_failed_stores(vec![13, 14, 21, 23], Some(vec![1]), false) .unwrap(); // 13 and 14 should be removed from region 1. @@ -1621,14 +1767,81 @@ mod tests { assert_eq!(get_region_stores(engine.as_inner(), 2), &[21, 22, 23]); // Remove specified stores from all regions. - debugger.remove_failed_stores(vec![11, 23], None).unwrap(); + debugger + .remove_failed_stores(vec![11, 23], None, false) + .unwrap(); assert_eq!(get_region_stores(engine.as_inner(), 1), &[12]); assert_eq!(get_region_stores(engine.as_inner(), 2), &[21, 22]); // Should fail when the store itself is in the failed list. - init_region_state(engine.as_inner(), 3, &[100, 31, 32, 33]); - debugger.remove_failed_stores(vec![100], None).unwrap_err(); + init_region_state(engine.as_inner(), 3, &[100, 31, 32, 33], 0); + debugger + .remove_failed_stores(vec![100], None, false) + .unwrap_err(); + + // no learner, promote learner does nothing + init_region_state(engine.as_inner(), 4, &[41, 42, 43, 44], 0); + debugger.remove_failed_stores(vec![44], None, true).unwrap(); + assert_eq!(get_region_stores(engine.as_inner(), 4), &[41, 42, 43]); + assert_eq!(get_region_learner(engine.as_inner(), 4), 0); + + // promote learner + init_region_state(engine.as_inner(), 5, &[51, 52, 53, 54], 1); + debugger + .remove_failed_stores(vec![52, 53, 54], None, true) + .unwrap(); + assert_eq!(get_region_stores(engine.as_inner(), 5), &[51]); + assert_eq!(get_region_learner(engine.as_inner(), 5), 0); + + // no need to promote learner + init_region_state(engine.as_inner(), 6, &[61, 62, 63, 64], 1); + debugger.remove_failed_stores(vec![64], None, true).unwrap(); + assert_eq!(get_region_stores(engine.as_inner(), 6), &[61, 62, 63]); + assert_eq!(get_region_learner(engine.as_inner(), 6), 1); + } + + #[test] + fn test_drop_unapplied_raftlog() { + let debugger = new_debugger(); + debugger.set_store_id(100); + let kv_engine = &debugger.engines.kv; + let raft_engine = &debugger.engines.raft; + + init_region_state(kv_engine.as_inner(), 1, &[100, 101], 1); + init_region_state(kv_engine.as_inner(), 2, &[100, 103], 1); + init_raft_state(kv_engine, raft_engine, 1, 100, 90, 80); + init_raft_state(kv_engine, raft_engine, 2, 80, 80, 80); + + let region_info_2_before = debugger.region_info(2).unwrap(); + + // Drop raftlog on all regions + debugger.drop_unapplied_raftlog(None).unwrap(); + + let region_info_1 = debugger.region_info(1).unwrap(); + let region_info_2 = debugger.region_info(2).unwrap(); + + assert_eq!( + region_info_1.raft_local_state.as_ref().unwrap().last_index, + 80 + ); + assert_eq!( + region_info_1 + .raft_apply_state + .as_ref() + .unwrap() + .applied_index, + 80 + ); + assert_eq!( + region_info_1 + .raft_apply_state + .as_ref() + .unwrap() + .commit_index, + 80 + ); + assert_eq!(region_info_2, region_info_2_before); } #[test] diff --git a/src/server/service/debug.rs b/src/server/service/debug.rs index 2396ab752b4..c0de5b67554 100644 --- a/src/server/service/debug.rs +++ b/src/server/service/debug.rs @@ -484,6 +484,30 @@ impl + 'static> debugpb::Debug f self.handle_response(ctx, sink, f, TAG); } + + fn get_all_regions_in_store( + &mut self, + ctx: RpcContext<'_>, + _: GetAllRegionsInStoreRequest, + sink: UnarySink, + ) { + const TAG: &str = "debug_get_all_regions_in_store"; + let debugger = self.debugger.clone(); + + let f = self + .pool + .spawn(async move { + let mut resp = GetAllRegionsInStoreResponse::default(); + match debugger.get_all_regions_in_store() { + Ok(regions) => resp.set_regions(regions), + Err(_) => resp.set_regions(vec![]), + } + Ok(resp) + }) + .map(|res| res.unwrap()); + + self.handle_response(ctx, sink, f, TAG); + } } fn region_detail>(