fix(artifacts): harden reconciliation scanner
This commit is contained in:
@@ -1,10 +1,28 @@
|
||||
//! Test-only crash and fault control for syscall-coupled storage checkpoints.
|
||||
|
||||
use std::{
|
||||
sync::{Condvar, Mutex, OnceLock},
|
||||
collections::BTreeMap,
|
||||
sync::{
|
||||
Condvar, Mutex, OnceLock,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
static TRAVERSAL_CALLS: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
pub fn reset_traversal_calls() {
|
||||
TRAVERSAL_CALLS.store(0, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn traversal_calls() -> usize {
|
||||
TRAVERSAL_CALLS.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn record_traversal_call() {
|
||||
TRAVERSAL_CALLS.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum FaultAction {
|
||||
Exit,
|
||||
@@ -13,7 +31,8 @@ pub enum FaultAction {
|
||||
}
|
||||
|
||||
struct State {
|
||||
checkpoint: Option<(String, FaultAction)>,
|
||||
checkpoint: Option<(String, FaultAction, usize)>,
|
||||
hits: BTreeMap<String, usize>,
|
||||
held: bool,
|
||||
}
|
||||
fn slot() -> &'static (Mutex<State>, Condvar) {
|
||||
@@ -22,6 +41,7 @@ fn slot() -> &'static (Mutex<State>, Condvar) {
|
||||
(
|
||||
Mutex::new(State {
|
||||
checkpoint: None,
|
||||
hits: BTreeMap::new(),
|
||||
held: false,
|
||||
}),
|
||||
Condvar::new(),
|
||||
@@ -30,10 +50,25 @@ fn slot() -> &'static (Mutex<State>, Condvar) {
|
||||
}
|
||||
|
||||
pub fn set_checkpoint(stage: impl Into<String>, action: FaultAction) {
|
||||
set_checkpoint_on_hit(stage, action, 1);
|
||||
}
|
||||
|
||||
pub fn set_checkpoint_on_hit(stage: impl Into<String>, action: FaultAction, hit: usize) {
|
||||
assert!(hit > 0, "checkpoint hit is one-based");
|
||||
let (lock, _) = slot();
|
||||
let mut state = lock.lock().expect("artifact test checkpoint lock poisoned");
|
||||
state.checkpoint = Some((stage.into(), action, hit));
|
||||
state.hits.clear();
|
||||
}
|
||||
|
||||
pub fn checkpoint_hits(stage: &str) -> usize {
|
||||
let (lock, _) = slot();
|
||||
lock.lock()
|
||||
.expect("artifact test checkpoint lock poisoned")
|
||||
.checkpoint = Some((stage.into(), action));
|
||||
.hits
|
||||
.get(stage)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
pub fn clear_checkpoint() {
|
||||
@@ -55,10 +90,17 @@ pub fn wait_until_held(timeout: Duration) -> bool {
|
||||
pub(crate) fn checkpoint(stage: &str) -> Option<FaultAction> {
|
||||
let (lock, wake) = slot();
|
||||
let mut state = lock.lock().expect("artifact test checkpoint lock poisoned");
|
||||
let action = state
|
||||
.checkpoint
|
||||
.as_ref()
|
||||
.and_then(|(expected, action)| (expected == stage).then_some(*action));
|
||||
let configured = state.checkpoint.clone();
|
||||
let action = configured.and_then(|(expected, action, target_hit)| {
|
||||
if expected != stage {
|
||||
return None;
|
||||
}
|
||||
let hit = state.hits.entry(stage.to_owned()).or_insert(0);
|
||||
if *hit == 0 {
|
||||
*hit = 1;
|
||||
}
|
||||
(*hit == target_hit).then_some(action)
|
||||
});
|
||||
if action == Some(FaultAction::Hold) {
|
||||
state.held = true;
|
||||
wake.notify_all();
|
||||
@@ -73,9 +115,13 @@ pub(crate) fn checkpoint(stage: &str) -> Option<FaultAction> {
|
||||
|
||||
pub(crate) fn action(stage: &str) -> Option<FaultAction> {
|
||||
let (lock, _) = slot();
|
||||
let state = lock.lock().expect("artifact test checkpoint lock poisoned");
|
||||
let mut state = lock.lock().expect("artifact test checkpoint lock poisoned");
|
||||
*state.hits.entry(stage.to_owned()).or_insert(0) += 1;
|
||||
let hit = state.hits[stage];
|
||||
state
|
||||
.checkpoint
|
||||
.as_ref()
|
||||
.and_then(|(expected, action)| (expected == stage).then_some(*action))
|
||||
.and_then(|(expected, action, target_hit)| {
|
||||
(expected == stage && hit == *target_hit).then_some(*action)
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user