crank/crates/crank-runtime/tests/product_metrics.rs

use std::{
    collections::BTreeMap,
    sync::{
        Arc,
        atomic::{AtomicUsize, Ordering},
    },
};

use async_trait::async_trait;
use crank_core::{
    AdapterResponse, ConfirmationPolicy, ExecutionConfig, ExecutionMode, HttpMethod,
    IdempotencyMode, IdempotencyPolicy, Operation, OperationId, OperationSafetyClass,
    OperationSafetyPolicy, OperationSecurityLevel, OperationStatus, Protocol, ProtocolAdapter,
    ProtocolAdapterError, ResponseCachePolicy, RestTarget,
    RuntimeRequestContext as ProtocolRequestContext, Target, ToolDescription,
};
use crank_mapping::{MappingRule, MappingSet};
use crank_runtime::{
    InMemoryCoordinationStateStore, InMemoryResponseCacheStore, RuntimeError,
    RuntimeExecutorBuilder, RuntimeRequestContext,
};
use crank_schema::{Schema, SchemaKind};
use metrics_util::debugging::DebuggingRecorder;
use serde_json::json;
use time::OffsetDateTime;

#[tokio::test]
async fn real_runtime_paths_emit_cache_idempotency_and_confirmation_outcomes() {
    let recorder = DebuggingRecorder::new();
    let snapshotter = recorder.snapshotter();
    recorder
        .install()
        .expect("isolated integration test recorder");

    exercise_response_cache().await;
    exercise_idempotency().await;
    exercise_cancelled_idempotency().await;
    exercise_confirmation().await;

    let snapshot = snapshotter.snapshot().into_vec();
    for outcome in ["miss", "stored", "hit"] {
        assert!(has_series(
            &snapshot,
            "crank_runtime_cache_total",
            "outcome",
            outcome
        ));
    }
    for outcome in [
        "execute",
        "completed",
        "replay",
        "conflict",
        "outcome_unknown",
    ] {
        assert!(has_series(
            &snapshot,
            "crank_idempotency_total",
            "outcome",
            outcome
        ));
    }
    for outcome in ["required", "approved", "invalid_token"] {
        assert!(
            has_series(&snapshot, "crank_confirmation_total", "outcome", outcome),
            "missing confirmation outcome {outcome}: {snapshot:?}"
        );
    }
}

async fn exercise_cancelled_idempotency() {
    let calls = Arc::new(AtomicUsize::new(0));
    let executor = RuntimeExecutorBuilder::new()
        .register_adapter(Arc::new(BlockingAdapter {
            calls: Arc::clone(&calls),
        }))
        .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
        .build();
    let operation: crank_runtime::RuntimeOperation = operation(
        "cancelled_idempotent_write",
        HttpMethod::Post,
        None,
        Some(IdempotencyPolicy {
            mode: IdempotencyMode::Required,
            ttl_ms: 60_000,
            input_field: Some("key".to_owned()),
            header_name: Some("Idempotency-Key".to_owned()),
        }),
        None,
    )
    .into();
    let context = RuntimeRequestContext::from_request_id("req_cancelled_idempotency")
        .with_response_cache_scope("workspace_cancelled", "agent_cancelled");
    let input = json!({"key": "cancelled-key"});

    let running_executor = executor.clone();
    let running_operation = operation.clone();
    let running_context = context.clone();
    let running_input = input.clone();
    let task = tokio::spawn(async move {
        running_executor
            .execute_with_context(&running_operation, &running_input, Some(&running_context))
            .await
    });
    wait_for_calls(&calls, 1).await;
    task.abort();
    let _ = task.await;

    let retry = executor
        .execute_with_context(&operation, &input, Some(&context))
        .await
        .expect_err("cancelled external outcome must not be retried automatically");
    assert!(matches!(
        retry,
        RuntimeError::IdempotencyOutcomeUnknown { .. }
    ));
    assert_eq!(calls.load(Ordering::SeqCst), 1);
}

async fn exercise_response_cache() {
    let calls = Arc::new(AtomicUsize::new(0));
    let executor = executor(Arc::clone(&calls));
    let operation: crank_runtime::RuntimeOperation = operation(
        "cached_lookup",
        HttpMethod::Get,
        Some(ResponseCachePolicy { ttl_ms: 60_000 }),
        None,
        None,
    )
    .into();
    let context = RuntimeRequestContext::from_request_id("req_cache")
        .with_response_cache_scope("workspace_cache", "agent_cache");

    executor
        .execute_with_context(&operation, &json!({"key": "cache-key"}), Some(&context))
        .await
        .unwrap();
    executor
        .execute_with_context(&operation, &json!({"key": "cache-key"}), Some(&context))
        .await
        .unwrap();

    assert_eq!(calls.load(Ordering::SeqCst), 1);
}

async fn exercise_idempotency() {
    let calls = Arc::new(AtomicUsize::new(0));
    let executor = executor(Arc::clone(&calls));
    let operation: crank_runtime::RuntimeOperation = operation(
        "idempotent_write",
        HttpMethod::Post,
        None,
        Some(IdempotencyPolicy {
            mode: IdempotencyMode::Required,
            ttl_ms: 60_000,
            input_field: Some("key".to_owned()),
            header_name: Some("Idempotency-Key".to_owned()),
        }),
        None,
    )
    .into();
    let context = RuntimeRequestContext::from_request_id("req_idempotency")
        .with_response_cache_scope("workspace_idempotency", "agent_idempotency");

    let first_input = json!({"key": "stable-key", "variant": "first"});
    executor
        .execute_with_context(&operation, &first_input, Some(&context))
        .await
        .unwrap();
    executor
        .execute_with_context(&operation, &first_input, Some(&context))
        .await
        .unwrap();
    let conflict = executor
        .execute_with_context(
            &operation,
            &json!({"key": "stable-key", "variant": "different"}),
            Some(&context),
        )
        .await
        .expect_err("same idempotency key with a different input must conflict");

    assert!(matches!(conflict, RuntimeError::IdempotencyConflict { .. }));
    assert_eq!(calls.load(Ordering::SeqCst), 1);
}

async fn exercise_confirmation() {
    let calls = Arc::new(AtomicUsize::new(0));
    let executor = executor(Arc::clone(&calls));
    let operation: crank_runtime::RuntimeOperation = operation(
        "destructive_write",
        HttpMethod::Delete,
        None,
        None,
        Some(OperationSafetyPolicy {
            class: OperationSafetyClass::Destructive,
            confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }),
        }),
    )
    .into();
    let context = RuntimeRequestContext::from_request_id("req_confirmation")
        .with_response_cache_scope("workspace_confirmation", "agent_confirmation");
    let input = json!({"key": "delete-key"});

    let required = executor
        .execute_with_context(&operation, &input, Some(&context))
        .await
        .expect_err("destructive operation must require confirmation");
    let RuntimeError::ConfirmationRequired {
        confirmation_token, ..
    } = required
    else {
        panic!("expected confirmation token");
    };
    let confirmed = context
        .clone()
        .with_confirmation_token(confirmation_token.clone());
    executor
        .execute_with_context(&operation, &input, Some(&confirmed))
        .await
        .unwrap();
    let invalid = executor
        .execute_with_context(&operation, &input, Some(&confirmed))
        .await
        .expect_err("confirmation token must be single-use");

    assert!(matches!(
        invalid,
        RuntimeError::InvalidConfirmationToken { .. }
    ));
    assert_eq!(calls.load(Ordering::SeqCst), 1);
}

fn executor(calls: Arc<AtomicUsize>) -> crank_runtime::RuntimeExecutor {
    RuntimeExecutorBuilder::new()
        .register_adapter(Arc::new(CountingAdapter { calls }))
        .with_response_cache(Arc::new(InMemoryResponseCacheStore::default()))
        .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
        .build()
}

struct CountingAdapter {
    calls: Arc<AtomicUsize>,
}

struct BlockingAdapter {
    calls: Arc<AtomicUsize>,
}

#[async_trait]
impl ProtocolAdapter for BlockingAdapter {
    fn protocol(&self) -> Protocol {
        Protocol::Rest
    }

    fn supports_mode(&self, mode: ExecutionMode) -> bool {
        mode == ExecutionMode::Unary
    }

    async fn invoke_unary(
        &self,
        _target: &Target,
        _prepared: &crank_core::PreparedRequest,
        _context: &ProtocolRequestContext,
    ) -> Result<AdapterResponse, ProtocolAdapterError> {
        self.calls.fetch_add(1, Ordering::SeqCst);
        std::future::pending().await
    }
}

#[async_trait]
impl ProtocolAdapter for CountingAdapter {
    fn protocol(&self) -> Protocol {
        Protocol::Rest
    }

    fn supports_mode(&self, mode: ExecutionMode) -> bool {
        mode == ExecutionMode::Unary
    }

    async fn invoke_unary(
        &self,
        _target: &Target,
        _prepared: &crank_core::PreparedRequest,
        _context: &ProtocolRequestContext,
    ) -> Result<AdapterResponse, ProtocolAdapterError> {
        self.calls.fetch_add(1, Ordering::SeqCst);
        Ok(AdapterResponse {
            status_code: 200,
            headers: BTreeMap::new(),
            body: json!({"result": "ok"}),
            data: json!({"result": "ok"}),
        })
    }
}

fn operation(
    name: &str,
    method: HttpMethod,
    response_cache: Option<ResponseCachePolicy>,
    idempotency: Option<IdempotencyPolicy>,
    safety: Option<OperationSafetyPolicy>,
) -> Operation<Schema, MappingSet> {
    Operation {
        id: OperationId::new(format!("op_{name}")),
        name: name.to_owned(),
        display_name: name.to_owned(),
        category: "metrics".to_owned(),
        protocol: Protocol::Rest,
        security_level: OperationSecurityLevel::Standard,
        status: OperationStatus::Published,
        version: 1,
        target: Target::Rest(RestTarget {
            base_url: "https://metrics.example.invalid".to_owned(),
            method,
            path_template: "/resource".to_owned(),
            static_headers: BTreeMap::new(),
        }),
        input_schema: object_schema("key"),
        output_schema: object_schema("result"),
        input_mapping: MappingSet { rules: Vec::new() },
        output_mapping: MappingSet {
            rules: vec![MappingRule {
                source: "$.response.body.result".to_owned(),
                target: "$.output.result".to_owned(),
                required: true,
                default_value: None,
                transform: None,
                condition: None,
                notes: None,
            }],
        },
        execution_config: ExecutionConfig {
            timeout_ms: 1_000,
            retry_policy: None,
            response_cache,
            idempotency,
            safety,
            approval_policy: None,
            auth_profile_ref: None,
            headers: BTreeMap::new(),
        },
        tool_description: ToolDescription {
            title: name.to_owned(),
            description: "Exercises a real runtime metrics path.".to_owned(),
            tags: Vec::new(),
            examples: Vec::new(),
        },
        samples: None,
        generated_draft: None,
        config_export: None,
        wizard_state: None,
        created_at: OffsetDateTime::UNIX_EPOCH,
        updated_at: OffsetDateTime::UNIX_EPOCH,
        published_at: None,
    }
}

fn object_schema(field: &str) -> Schema {
    Schema {
        kind: SchemaKind::Object,
        description: None,
        required: true,
        nullable: false,
        default_value: None,
        fields: BTreeMap::from([(
            field.to_owned(),
            Schema {
                kind: SchemaKind::String,
                description: None,
                required: true,
                nullable: false,
                default_value: None,
                fields: BTreeMap::new(),
                items: None,
                enum_values: Vec::new(),
                variants: Vec::new(),
            },
        )]),
        items: None,
        enum_values: Vec::new(),
        variants: Vec::new(),
    }
}

fn has_series(
    snapshot: &[(
        metrics_util::CompositeKey,
        Option<metrics::Unit>,
        Option<metrics::SharedString>,
        metrics_util::debugging::DebugValue,
    )],
    metric: &str,
    label_name: &str,
    label_value: &str,
) -> bool {
    snapshot.iter().any(|(key, _, _, _)| {
        key.key().name() == metric
            && key
                .key()
                .labels()
                .any(|label| label.key() == label_name && label.value() == label_value)
    })
}

async fn wait_for_calls(calls: &AtomicUsize, expected: usize) {
    for _ in 0..1_000 {
        if calls.load(Ordering::SeqCst) >= expected {
            return;
        }
        tokio::task::yield_now().await;
    }
    panic!("adapter did not receive {expected} call(s)");
}