Files
crank/crates/crank-core/src/observability.rs

342 lines
11 KiB
Rust

use serde::{Deserialize, Serialize};
use serde_json::{Map, Value, json};
use time::OffsetDateTime;
use crate::{
AgentId, ExecutionErrorCode, ExecutionStage, OperationId, OutcomeCertainty, Retryability,
WorkspaceId,
};
pub const INVOCATION_PREVIEW_MAX_BYTES: usize = 16 * 1024;
pub const INVOCATION_PREVIEW_MAX_DEPTH: usize = 12;
pub const INVOCATION_PREVIEW_MAX_OBJECT_FIELDS: usize = 64;
pub const INVOCATION_PREVIEW_MAX_ARRAY_ITEMS: usize = 64;
pub fn sanitize_invocation_preview(value: &Value) -> Value {
let redacted = redact_sensitive_fields(value, 0);
let Ok(encoded) = serde_json::to_vec(&redacted) else {
return Value::Null;
};
if encoded.len() <= INVOCATION_PREVIEW_MAX_BYTES {
return redacted;
}
let preview = String::from_utf8_lossy(&encoded[..INVOCATION_PREVIEW_MAX_BYTES]).into_owned();
json!({
"truncated": true,
"reason": "max_bytes",
"original_bytes": encoded.len(),
"preview": preview,
})
}
fn truncation_marker(reason: &'static str, omitted: usize) -> Value {
json!({
"truncated": true,
"reason": reason,
"omitted": omitted,
})
}
fn redact_sensitive_fields(value: &Value, depth: usize) -> Value {
if depth >= INVOCATION_PREVIEW_MAX_DEPTH {
return truncation_marker("max_depth", 1);
}
match value {
Value::Object(object) => {
let mut redacted = Map::new();
let mut omitted = 0usize;
for (index, (key, value)) in object.iter().enumerate() {
if index >= INVOCATION_PREVIEW_MAX_OBJECT_FIELDS {
omitted += 1;
continue;
}
let value = if is_sensitive_key(key) {
Value::String("[REDACTED]".to_owned())
} else {
redact_sensitive_fields(value, depth + 1)
};
redacted.insert(key.clone(), value);
}
if omitted > 0 {
redacted.insert(
"_crank_truncated".to_owned(),
truncation_marker("max_object_fields", omitted),
);
}
Value::Object(redacted)
}
Value::Array(items) => {
let mut redacted = items
.iter()
.take(INVOCATION_PREVIEW_MAX_ARRAY_ITEMS)
.map(|value| redact_sensitive_fields(value, depth + 1))
.collect::<Vec<_>>();
let omitted = items.len().saturating_sub(redacted.len());
if omitted > 0 {
redacted.push(truncation_marker("max_array_items", omitted));
}
Value::Array(redacted)
}
Value::String(value) if looks_like_sensitive_value(value) => {
Value::String("[REDACTED]".to_owned())
}
_ => value.clone(),
}
}
fn is_sensitive_key(key: &str) -> bool {
let normalized = key.to_ascii_lowercase().replace('-', "_");
let compact = normalized
.chars()
.filter(char::is_ascii_alphanumeric)
.collect::<String>();
[
"authorization",
"cookie",
"password",
"passwd",
"secret",
"token",
]
.iter()
.any(|sensitive| compact.contains(sensitive))
|| ["apikey", "accesskey", "privatekey"]
.iter()
.any(|sensitive| compact.contains(sensitive))
}
fn looks_like_sensitive_value(value: &str) -> bool {
let trimmed = value.trim();
let lower = trimmed.to_ascii_lowercase();
lower.starts_with("bearer ")
|| lower.starts_with("basic ")
|| lower.contains("password=")
|| lower.contains("token=")
|| lower.contains("secret=")
|| lower.contains("api_key=")
|| lower.contains("apikey=")
|| lower.contains("authorization:")
|| trimmed.starts_with("sk_")
|| trimmed.starts_with("crk_")
|| trimmed.contains("SECRET_")
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum InvocationSource {
AdminTestRun,
AgentToolCall,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum InvocationLevel {
Debug,
Info,
Warn,
Error,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum InvocationStatus {
Ok,
Error,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum UsagePeriod {
#[serde(rename = "30m")]
Last30Minutes,
#[serde(rename = "1h")]
LastHour,
#[serde(rename = "6h")]
Last6Hours,
#[serde(rename = "24h")]
Last24Hours,
#[serde(rename = "7d")]
Last7Days,
#[serde(rename = "30d")]
Last30Days,
#[serde(rename = "90d")]
Last90Days,
#[serde(rename = "this_month")]
ThisMonth,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct InvocationLog {
pub id: crate::ids::InvocationLogId,
pub workspace_id: WorkspaceId,
pub agent_id: Option<AgentId>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub platform_api_key_id: Option<crate::ids::PlatformApiKeyId>,
pub operation_id: OperationId,
pub operation_version: Option<u32>,
pub source: InvocationSource,
pub level: InvocationLevel,
pub status: InvocationStatus,
pub tool_name: String,
pub message: String,
pub request_id: Option<String>,
pub trace_id: Option<String>,
pub status_code: Option<u16>,
pub duration_ms: u64,
pub error_kind: Option<String>,
pub execution_stage: Option<ExecutionStage>,
pub execution_error_code: Option<ExecutionErrorCode>,
pub retryability: Option<Retryability>,
pub outcome_certainty: Option<OutcomeCertainty>,
pub request_preview: Value,
pub response_preview: Value,
#[serde(with = "time::serde::rfc3339")]
pub created_at: OffsetDateTime,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct UsageRollup {
pub workspace_id: WorkspaceId,
pub agent_id: Option<AgentId>,
pub operation_id: Option<OperationId>,
pub period: UsagePeriod,
pub calls_total: u64,
pub calls_ok: u64,
pub calls_error: u64,
pub p50_ms: u64,
pub p95_ms: u64,
pub p99_ms: u64,
}
#[cfg(test)]
mod tests {
use serde_json::json;
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use super::{
INVOCATION_PREVIEW_MAX_ARRAY_ITEMS, INVOCATION_PREVIEW_MAX_BYTES,
INVOCATION_PREVIEW_MAX_DEPTH, INVOCATION_PREVIEW_MAX_OBJECT_FIELDS, InvocationLevel,
InvocationLog, InvocationSource, InvocationStatus, UsagePeriod,
sanitize_invocation_preview,
};
use crate::{AgentId, OperationId, OutcomeCertainty, WorkspaceId, ids::InvocationLogId};
fn timestamp(value: &str) -> OffsetDateTime {
OffsetDateTime::parse(value, &Rfc3339).unwrap()
}
#[test]
fn invocation_log_serializes_timestamps_as_rfc3339() {
let log = InvocationLog {
id: InvocationLogId::new("log_01"),
workspace_id: WorkspaceId::new("ws_01"),
agent_id: Some(AgentId::new("agent_01")),
platform_api_key_id: None,
operation_id: OperationId::new("op_01"),
operation_version: Some(1),
source: InvocationSource::AdminTestRun,
level: InvocationLevel::Info,
status: InvocationStatus::Ok,
tool_name: "create_lead".to_owned(),
message: "ok".to_owned(),
request_id: Some("req_01".to_owned()),
trace_id: Some("0af7651916cd43dd8448eb211c80319c".to_owned()),
status_code: Some(200),
duration_ms: 123,
error_kind: None,
execution_stage: None,
execution_error_code: None,
retryability: None,
outcome_certainty: Some(OutcomeCertainty::Certain),
request_preview: json!({"input": "value"}),
response_preview: json!({"ok": true}),
created_at: timestamp("2026-04-19T12:34:56Z"),
};
let value = serde_json::to_value(&log).unwrap();
assert_eq!(value["created_at"], "2026-04-19T12:34:56Z");
assert_eq!(value["source"], "admin_test_run");
assert_eq!(value["level"], "info");
assert_eq!(value["status"], "ok");
}
#[test]
fn usage_period_serializes_compact_labels() {
let value = serde_json::to_value(UsagePeriod::Last7Days).unwrap();
assert_eq!(value, "7d");
}
#[test]
fn redacts_sensitive_fields_recursively() {
let preview = sanitize_invocation_preview(&json!({
"authorization": "Bearer secret",
"nested": {
"api_key": "key",
"refreshToken": "token",
"client_secret_value": "secret",
"body": "Bearer SECRET_APPROVAL_CANARY",
"neutral": "token=SECRET_APPROVAL_CANARY",
"value": 42
}
}));
assert_eq!(preview["authorization"], "[REDACTED]");
assert_eq!(preview["nested"]["api_key"], "[REDACTED]");
assert_eq!(preview["nested"]["refreshToken"], "[REDACTED]");
assert_eq!(preview["nested"]["client_secret_value"], "[REDACTED]");
assert_eq!(preview["nested"]["body"], "[REDACTED]");
assert_eq!(preview["nested"]["neutral"], "[REDACTED]");
assert_eq!(preview["nested"]["value"], 42);
}
#[test]
fn truncates_large_previews() {
let preview = sanitize_invocation_preview(&json!({
"body": "x".repeat(INVOCATION_PREVIEW_MAX_BYTES * 2)
}));
assert_eq!(preview["truncated"], true);
assert_eq!(preview["reason"], "max_bytes");
assert!(preview["original_bytes"].as_u64().unwrap() > INVOCATION_PREVIEW_MAX_BYTES as u64);
}
#[test]
fn bounds_preview_depth_fields_and_array_items() {
let mut nested = json!({"secret_token": "SECRET_VALUE"});
for _ in 0..(INVOCATION_PREVIEW_MAX_DEPTH + 4) {
nested = json!({ "nested": nested });
}
let deep_preview = sanitize_invocation_preview(&nested);
assert!(
serde_json::to_string(&deep_preview)
.unwrap()
.contains("\"reason\":\"max_depth\"")
);
assert!(
!serde_json::to_string(&deep_preview)
.unwrap()
.contains("SECRET_VALUE")
);
let wide = serde_json::Value::Object(
(0..(INVOCATION_PREVIEW_MAX_OBJECT_FIELDS + 5))
.map(|index| (format!("field_{index:03}"), json!(index)))
.collect(),
);
let wide_preview = sanitize_invocation_preview(&wide);
assert_eq!(
wide_preview["_crank_truncated"]["reason"],
"max_object_fields"
);
let array_preview = sanitize_invocation_preview(&json!(
(0..(INVOCATION_PREVIEW_MAX_ARRAY_ITEMS + 3)).collect::<Vec<_>>()
));
assert_eq!(
array_preview.as_array().unwrap().last().unwrap()["reason"],
"max_array_items"
);
}
}