Files
crank/apps/admin-api/src/service/observability.rs
T

691 lines
24 KiB
Rust

use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
use crank_core::{
AgentId, ApprovalRequestId, ApprovalRequestStatus, InvocationLogId, OperationId, UsagePeriod,
WorkspaceId,
};
use crank_registry::{
ApprovalRequestRecord, DecideApprovalRequest, ExpireApprovalRequest, InvocationLogRecord,
InvocationRetentionOutcome, ListApprovalRequestsQuery, ListInvocationLogsQuery, UsageQuery,
UsageRollupRecord,
};
use serde_json::json;
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use tracing::instrument;
use crate::{
error::ApiError,
service::{
AdminService, ApprovalDecisionPayload, ApprovalsQuery, LogsListResponse, LogsQuery,
UsageOverviewResponse, UsageRequestQuery, usage_window,
},
};
const LOG_LIST_DEFAULT_LIMIT: u32 = 100;
const LOG_LIST_MAX_LIMIT: u32 = 200;
const LOG_CSV_MAX_ROWS: u32 = 1_000;
const MAX_EXPLICIT_USAGE_WINDOW_DAYS: i64 = 90;
#[derive(serde::Serialize, serde::Deserialize)]
struct LogsCursor {
created_at: String,
id: String,
}
fn safe_approval_record(mut record: ApprovalRequestRecord) -> ApprovalRequestRecord {
record.approval.request_payload =
crank_core::sanitize_invocation_preview(&record.approval.request_payload);
record.approval.response_payload = record
.approval
.response_payload
.as_ref()
.map(crank_core::sanitize_invocation_preview);
record
}
impl AdminService {
pub async fn cleanup_invocation_logs_before(
&self,
cutoff: OffsetDateTime,
) -> Result<InvocationRetentionOutcome, ApiError> {
self.registry
.delete_invocation_logs_before(cutoff)
.await
.map_err(ApiError::from)
}
#[instrument(skip(self))]
pub async fn list_logs(
&self,
workspace_id: &WorkspaceId,
query: LogsQuery,
) -> Result<LogsListResponse, ApiError> {
self.ensure_workspace_exists(workspace_id).await?;
let operation_id = query.operation_id.as_deref().map(OperationId::new);
let agent_id = query.agent_id.as_deref().map(AgentId::new);
let (_, created_after, created_before, _) =
resolve_usage_window(query.period.unwrap_or(UsagePeriod::Last7Days), &query)?;
let (cursor_created_at, cursor_id) = decode_logs_cursor(query.cursor.as_deref())?;
let limit = query
.limit
.unwrap_or(LOG_LIST_DEFAULT_LIMIT)
.clamp(1, LOG_LIST_MAX_LIMIT);
let mut items = self
.registry
.list_invocation_logs(ListInvocationLogsQuery {
workspace_id,
level: query.level,
status: query.status,
outcome_group: query.outcome_group,
search_text: query.search.as_deref(),
source: query.source,
operation_id: operation_id.as_ref(),
agent_id: agent_id.as_ref(),
created_after: Some(&created_after),
created_before: Some(&created_before),
cursor_created_at: cursor_created_at.as_deref(),
cursor_id: cursor_id.as_ref(),
limit: limit.saturating_add(1),
})
.await?;
let next_cursor = if items.len() > limit as usize {
items.truncate(limit as usize);
items.last().map(encode_logs_cursor).transpose()?
} else {
None
};
Ok(LogsListResponse { items, next_cursor })
}
#[instrument(skip(self))]
pub async fn get_log(
&self,
workspace_id: &WorkspaceId,
log_id: &InvocationLogId,
) -> Result<InvocationLogRecord, ApiError> {
self.registry
.get_invocation_log(workspace_id, log_id)
.await?
.ok_or_else(|| {
ApiError::not_found_with_context(
format!("invocation log {} was not found", log_id.as_str()),
json!({ "log_id": log_id.as_str() }),
)
})
}
#[instrument(skip(self))]
pub async fn export_logs_csv(
&self,
workspace_id: &WorkspaceId,
mut query: LogsQuery,
) -> Result<String, ApiError> {
query.limit = Some(
query
.limit
.unwrap_or(LOG_CSV_MAX_ROWS)
.clamp(1, LOG_CSV_MAX_ROWS),
);
query.cursor = None;
let page = self.list_logs(workspace_id, query).await?;
let mut csv = String::from(
"created_at,level,status,source,agent,operation,operation_version,duration_ms,request_id,trace_id,stage,error_code,message,request_preview,response_preview\n",
);
for record in page.items {
let log = record.log;
let row = [
log.created_at
.format(&Rfc3339)
.map_err(|error| ApiError::internal(error.to_string()))?,
serialize_csv_text(log.level)?,
serialize_csv_text(log.status)?,
serialize_csv_text(log.source)?,
record
.agent_display_name
.or(record.agent_slug)
.unwrap_or_default(),
record.operation_display_name,
log.operation_version
.map(|value| value.to_string())
.unwrap_or_default(),
log.duration_ms.to_string(),
log.request_id.unwrap_or_default(),
log.trace_id.unwrap_or_default(),
log.execution_stage
.map(serialize_csv_text)
.transpose()?
.unwrap_or_default(),
log.execution_error_code
.map(serialize_csv_text)
.transpose()?
.unwrap_or_default(),
log.message,
serde_json::to_string(&log.request_preview)
.map_err(|error| ApiError::internal(error.to_string()))?,
serde_json::to_string(&log.response_preview)
.map_err(|error| ApiError::internal(error.to_string()))?,
];
csv.push_str(&row.into_iter().map(csv_cell).collect::<Vec<_>>().join(","));
csv.push('\n');
if csv.len() > 1_048_576 {
return Err(ApiError::unprocessable_with_context(
"logs CSV export exceeded the bounded response size",
json!({ "error_code": "logs_csv_too_large" }),
));
}
}
Ok(csv)
}
#[instrument(skip(self))]
pub async fn list_approvals(
&self,
workspace_id: &WorkspaceId,
query: ApprovalsQuery,
) -> Result<Vec<ApprovalRequestRecord>, ApiError> {
self.ensure_workspace_exists(workspace_id).await?;
let records = self
.registry
.list_approval_requests(ListApprovalRequestsQuery {
workspace_id,
status: query.status,
limit: query.limit.unwrap_or(50).clamp(1, 200),
})
.await?;
let mut normalized = Vec::with_capacity(records.len());
for record in records {
let record = self.normalize_approval_record(record).await?;
if query.status.is_none() || record.approval.status == query.status.unwrap() {
normalized.push(safe_approval_record(record));
}
}
Ok(normalized)
}
#[instrument(skip(self))]
pub async fn get_approval(
&self,
workspace_id: &WorkspaceId,
approval_id: &ApprovalRequestId,
) -> Result<ApprovalRequestRecord, ApiError> {
self.ensure_workspace_exists(workspace_id).await?;
let record = self
.registry
.get_approval_request(workspace_id, approval_id)
.await?
.ok_or_else(|| {
ApiError::not_found_with_context(
format!("approval request {} was not found", approval_id.as_str()),
json!({ "approval_id": approval_id.as_str() }),
)
})?;
self.normalize_approval_record(record)
.await
.map(safe_approval_record)
}
#[instrument(skip(self, payload))]
pub async fn approve_approval(
&self,
workspace_id: &WorkspaceId,
approval_id: &ApprovalRequestId,
payload: ApprovalDecisionPayload,
) -> Result<ApprovalRequestRecord, ApiError> {
if !payload.approve.eq_ignore_ascii_case("yes") {
return Err(ApiError::unprocessable_with_context(
"approve must be yes for approval confirmation",
json!({ "error_code": "invalid_decision_payload" }),
));
}
self.decide_admin_approval(
workspace_id,
approval_id,
ApprovalRequestStatus::Approved,
Some(json!({"approve": "yes"})),
payload.note.as_deref(),
)
.await
}
#[instrument(skip(self, payload))]
pub async fn deny_approval(
&self,
workspace_id: &WorkspaceId,
approval_id: &ApprovalRequestId,
payload: ApprovalDecisionPayload,
) -> Result<ApprovalRequestRecord, ApiError> {
if !payload.approve.eq_ignore_ascii_case("no") {
return Err(ApiError::unprocessable_with_context(
"approve must be no for approval denial",
json!({ "error_code": "invalid_decision_payload" }),
));
}
self.decide_admin_approval(
workspace_id,
approval_id,
ApprovalRequestStatus::Denied,
Some(json!({"approve": "no"})),
payload.note.as_deref(),
)
.await
}
async fn decide_admin_approval(
&self,
workspace_id: &WorkspaceId,
approval_id: &ApprovalRequestId,
status: ApprovalRequestStatus,
response_payload: Option<serde_json::Value>,
decision_note: Option<&str>,
) -> Result<ApprovalRequestRecord, ApiError> {
self.ensure_workspace_exists(workspace_id).await?;
let current = self
.registry
.get_approval_request(workspace_id, approval_id)
.await?
.ok_or_else(|| {
ApiError::not_found_with_context(
format!("approval request {} was not found", approval_id.as_str()),
json!({ "approval_id": approval_id.as_str() }),
)
})?;
let normalized = self.normalize_approval_record(current).await?;
if normalized.approval.status != ApprovalRequestStatus::Pending {
return Err(ApiError::conflict_with_context(
"approval request is not pending",
json!({
"approval_id": approval_id.as_str(),
"status": normalized.approval.status,
"error_code": "approval_state_conflict"
}),
));
}
let decided = self
.registry
.decide_approval_request(DecideApprovalRequest {
workspace_id,
agent_id: &normalized.approval.agent_id,
approval_id,
operation_id: &normalized.approval.operation_id,
operation_version: normalized.approval.operation_version,
request_payload: &normalized.approval.request_payload,
status,
decided_at: OffsetDateTime::now_utc(),
decided_by_key_id: None,
response_payload,
decision_note,
})
.await?
.ok_or_else(|| {
ApiError::conflict_with_context(
"approval request state changed before decision was recorded",
json!({
"approval_id": approval_id.as_str(),
"error_code": "approval_state_conflict"
}),
)
})?;
Ok(safe_approval_record(decided))
}
async fn normalize_approval_record(
&self,
record: ApprovalRequestRecord,
) -> Result<ApprovalRequestRecord, ApiError> {
if record.approval.status != ApprovalRequestStatus::Pending
|| record.approval.expires_at > OffsetDateTime::now_utc()
{
return Ok(record);
}
Ok(self
.registry
.expire_approval_request(ExpireApprovalRequest {
workspace_id: &record.approval.workspace_id,
agent_id: &record.approval.agent_id,
approval_id: &record.approval.id,
expired_at: OffsetDateTime::now_utc(),
})
.await?
.unwrap_or(record))
}
#[instrument(skip(self))]
pub async fn get_usage_overview(
&self,
workspace_id: &WorkspaceId,
query: UsageRequestQuery,
) -> Result<UsageOverviewResponse, ApiError> {
self.ensure_workspace_exists(workspace_id).await?;
let (period, created_after, created_before, bucket) =
resolve_usage_request_window(query.period.unwrap_or(UsagePeriod::Last7Days), &query)?;
let usage_query = UsageQuery {
workspace_id,
period,
source: query.source,
created_after: &created_after,
created_before: &created_before,
bucket,
};
let summary = self.registry.summarize_usage(usage_query.clone()).await?;
let timeline = self
.registry
.list_usage_timeline(usage_query.clone())
.await?;
let operations = self
.registry
.list_usage_by_operation(usage_query.clone())
.await?;
let agents = self
.registry
.list_usage_by_agent(usage_query.clone())
.await?;
let outcomes = self.registry.list_usage_outcomes(usage_query).await?;
Ok(UsageOverviewResponse {
summary,
timeline,
operations,
agents,
outcomes,
})
}
#[instrument(skip(self))]
pub async fn export_usage_csv(
&self,
workspace_id: &WorkspaceId,
query: UsageRequestQuery,
) -> Result<String, ApiError> {
let usage = self.get_usage_overview(workspace_id, query).await?;
let mut csv = String::from(
"kind,name,group,error_code,calls_total,calls_error,p50_ms,p95_ms,p99_ms\n",
);
for operation in usage.operations {
csv.push_str(
&[
"operation".to_owned(),
operation.operation_display_name,
String::new(),
String::new(),
operation.calls_total.to_string(),
operation.calls_error.to_string(),
operation.p50_ms.to_string(),
operation.p95_ms.to_string(),
operation.p99_ms.to_string(),
]
.into_iter()
.map(csv_cell)
.collect::<Vec<_>>()
.join(","),
);
csv.push('\n');
}
for agent in usage.agents {
csv.push_str(
&[
"agent".to_owned(),
agent.agent_display_name,
String::new(),
String::new(),
agent.calls_total.to_string(),
agent.calls_error.to_string(),
agent.p50_ms.to_string(),
agent.p95_ms.to_string(),
agent.p99_ms.to_string(),
]
.into_iter()
.map(csv_cell)
.collect::<Vec<_>>()
.join(","),
);
csv.push('\n');
}
for outcome in usage.outcomes {
csv.push_str(
&[
"outcome".to_owned(),
String::new(),
serialize_csv_text(outcome.group)?,
outcome
.execution_error_code
.map(serialize_csv_text)
.transpose()?
.unwrap_or_default(),
outcome.calls_total.to_string(),
String::new(),
outcome.p50_ms.to_string(),
outcome.p95_ms.to_string(),
outcome.p99_ms.to_string(),
]
.into_iter()
.map(csv_cell)
.collect::<Vec<_>>()
.join(","),
);
csv.push('\n');
}
if csv.len() > 1_048_576 {
return Err(ApiError::unprocessable_with_context(
"usage CSV export exceeded the bounded response size",
json!({ "error_code": "usage_csv_too_large" }),
));
}
Ok(csv)
}
#[instrument(skip(self))]
pub async fn get_operation_usage(
&self,
workspace_id: &WorkspaceId,
operation_id: &OperationId,
query: UsageRequestQuery,
) -> Result<UsageRollupRecord, ApiError> {
self.get_operation(workspace_id, operation_id).await?;
let (period, created_after, created_before, bucket) =
resolve_usage_request_window(query.period.unwrap_or(UsagePeriod::Last7Days), &query)?;
self.registry
.get_usage_for_operation(
UsageQuery {
workspace_id,
period,
source: query.source,
created_after: &created_after,
created_before: &created_before,
bucket,
},
operation_id,
)
.await?
.ok_or_else(|| {
ApiError::not_found_with_context(
format!(
"usage for operation {} was not found",
operation_id.as_str()
),
json!({ "operation_id": operation_id.as_str() }),
)
})
}
#[instrument(skip(self))]
pub async fn get_agent_usage(
&self,
workspace_id: &WorkspaceId,
agent_id: &AgentId,
query: UsageRequestQuery,
) -> Result<UsageRollupRecord, ApiError> {
self.get_agent(workspace_id, agent_id).await?;
let (period, created_after, created_before, bucket) =
resolve_usage_request_window(query.period.unwrap_or(UsagePeriod::Last7Days), &query)?;
self.registry
.get_usage_for_agent(
UsageQuery {
workspace_id,
period,
source: query.source,
created_after: &created_after,
created_before: &created_before,
bucket,
},
agent_id,
)
.await?
.ok_or_else(|| {
ApiError::not_found_with_context(
format!("usage for agent {} was not found", agent_id.as_str()),
json!({ "agent_id": agent_id.as_str() }),
)
})
}
}
fn resolve_usage_window(
period: UsagePeriod,
query: &LogsQuery,
) -> Result<(UsagePeriod, String, String, crank_registry::UsageBucket), ApiError> {
resolve_explicit_or_period_window(
period,
query.created_after.as_deref(),
query.created_before.as_deref(),
)
}
fn resolve_usage_request_window(
period: UsagePeriod,
query: &UsageRequestQuery,
) -> Result<(UsagePeriod, String, String, crank_registry::UsageBucket), ApiError> {
resolve_explicit_or_period_window(
period,
query.created_after.as_deref(),
query.created_before.as_deref(),
)
}
fn resolve_explicit_or_period_window(
period: UsagePeriod,
created_after: Option<&str>,
created_before: Option<&str>,
) -> Result<(UsagePeriod, String, String, crank_registry::UsageBucket), ApiError> {
match (created_after, created_before) {
(None, None) => usage_window(period),
(Some(_), None) | (None, Some(_)) => Err(invalid_usage_window(
"usage window requires both created_after and created_before",
)),
(Some(after), Some(before)) => {
let after = OffsetDateTime::parse(after, &Rfc3339)
.map_err(|_| invalid_usage_window("created_after must be RFC3339 UTC"))?;
let before = OffsetDateTime::parse(before, &Rfc3339)
.map_err(|_| invalid_usage_window("created_before must be RFC3339 UTC"))?;
if after >= before {
return Err(invalid_usage_window(
"created_after must be before created_before",
));
}
let duration = before - after;
if duration > time::Duration::days(MAX_EXPLICIT_USAGE_WINDOW_DAYS) {
return Err(invalid_usage_window("usage window exceeds maximum range"));
}
let bucket = if duration <= time::Duration::days(1) {
crank_registry::UsageBucket::Hour
} else if duration <= time::Duration::days(14) {
crank_registry::UsageBucket::Day
} else if duration <= time::Duration::days(45) {
crank_registry::UsageBucket::Week
} else {
crank_registry::UsageBucket::Month
};
Ok((
period,
after
.format(&Rfc3339)
.map_err(|error| ApiError::internal(error.to_string()))?,
before
.format(&Rfc3339)
.map_err(|error| ApiError::internal(error.to_string()))?,
bucket,
))
}
}
}
fn invalid_usage_window(message: &'static str) -> ApiError {
ApiError::unprocessable_with_context(message, json!({ "error_code": "usage_window_invalid" }))
}
fn decode_logs_cursor(
cursor: Option<&str>,
) -> Result<(Option<String>, Option<InvocationLogId>), ApiError> {
let Some(cursor) = cursor else {
return Ok((None, None));
};
if cursor.len() > 512 {
return Err(invalid_logs_cursor());
}
let bytes = URL_SAFE_NO_PAD
.decode(cursor)
.map_err(|_| invalid_logs_cursor())?;
let decoded: LogsCursor = serde_json::from_slice(&bytes).map_err(|_| invalid_logs_cursor())?;
OffsetDateTime::parse(&decoded.created_at, &Rfc3339).map_err(|_| invalid_logs_cursor())?;
if decoded.id.is_empty()
|| decoded.id.len() > 128
|| decoded.id.contains(';')
|| decoded.id.contains(',')
{
return Err(invalid_logs_cursor());
}
Ok((
Some(decoded.created_at),
Some(InvocationLogId::new(decoded.id)),
))
}
fn encode_logs_cursor(record: &InvocationLogRecord) -> Result<String, ApiError> {
let cursor = LogsCursor {
created_at: record
.log
.created_at
.format(&Rfc3339)
.map_err(|error| ApiError::internal(error.to_string()))?,
id: record.log.id.as_str().to_owned(),
};
let encoded =
serde_json::to_vec(&cursor).map_err(|error| ApiError::internal(error.to_string()))?;
Ok(URL_SAFE_NO_PAD.encode(encoded))
}
fn invalid_logs_cursor() -> ApiError {
ApiError::unprocessable_with_context(
"logs cursor is invalid",
json!({ "error_code": "logs_cursor_invalid" }),
)
}
fn serialize_csv_text<T: serde::Serialize>(value: T) -> Result<String, ApiError> {
serde_json::to_value(value)
.map_err(|error| ApiError::internal(error.to_string()))?
.as_str()
.map(ToOwned::to_owned)
.ok_or_else(|| ApiError::internal("failed to serialize CSV enum value"))
}
fn csv_cell(mut value: String) -> String {
if value
.chars()
.next()
.is_some_and(|ch| matches!(ch, '=' | '+' | '-' | '@' | '\t' | '\r' | '\n'))
{
value.insert(0, '\'');
}
format!("\"{}\"", value.replace('"', "\"\""))
}