bsodfather/crank
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add observability api foundation

Browse Source
This commit is contained in:
a.tolmachev
2026-03-30 00:00:13 +03:00
parent 0a1680f24e
commit be9ee95cbe
21 changed files with 1535 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/admin-api/src/app.rs
+87 -1
View File
@@ -14,6 +14,7 @@ use crate::{
save_agent_bindings,
},
auth_profiles::{create_auth_profile, get_auth_profile, list_auth_profiles},
observability::{get_agent_usage, get_log, get_operation_usage, get_usage, list_logs},
operations::{
create_operation, create_version, export_operation, generate_draft, get_operation,
get_operation_version, list_grpc_services, list_operations, publish_operation,
@@ -98,7 +99,12 @@ pub fn build_app(state: AppState) -> Router {
.route(
"/platform-api-keys/{key_id}",
axum::routing::delete(delete_platform_api_key),
);
)
.route("/logs", get(list_logs))
.route("/logs/{log_id}", get(get_log))
.route("/usage", get(get_usage))
.route("/usage/operations/{operation_id}", get(get_operation_usage))
.route("/usage/agents/{agent_id}", get(get_agent_usage));
let admin_router = Router::new()
.route("/workspaces", get(list_workspaces).post(create_workspace))
@@ -394,6 +400,86 @@ mod tests {
assert_eq!(delete_key_status, reqwest::StatusCode::NO_CONTENT);
}
#[tokio::test]
async fn exposes_logs_and_usage_from_real_test_runs() {
let registry = test_registry().await;
let storage_root = test_storage_root("observability");
let upstream_base_url = spawn_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = reqwest::Client::new();
let created = client
.post(format!("{base_url}/operations"))
.json(&test_operation_payload(
&upstream_base_url,
"crm_observability",
))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let operation_id = created["operation_id"].as_str().unwrap().to_owned();
client
.post(format!("{base_url}/operations/{operation_id}/test-runs"))
.json(&json!({
"version": 1,
"input": { "email": "user@example.com" }
}))
.send()
.await
.unwrap();
let logs = client
.get(format!("{base_url}/logs?period=7d"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let log_id = logs["items"][0]["log"]["id"].as_str().unwrap().to_owned();
let log_detail = client
.get(format!("{base_url}/logs/{log_id}"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let usage = client
.get(format!("{base_url}/usage?period=7d"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let operation_usage = client
.get(format!(
"{base_url}/usage/operations/{operation_id}?period=7d"
))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
assert_eq!(logs["items"][0]["log"]["source"], "admin_test_run");
assert_eq!(logs["items"][0]["operation_name"], "crm_observability");
assert_eq!(log_detail["log"]["status"], "ok");
assert_eq!(usage["summary"]["rollup"]["calls_total"], 1);
assert_eq!(usage["summary"]["rollup"]["calls_ok"], 1);
assert_eq!(
usage["operations"][0]["operation_name"],
"crm_observability"
);
assert_eq!(operation_usage["rollup"]["calls_total"], 1);
}
#[tokio::test]
async fn creates_publishes_and_tests_graphql_operation() {
let registry = test_registry().await;
apps/admin-api/src/error.rs
+3
View File
@@ -108,6 +108,9 @@ impl From<RegistryError> for ApiError {
RegistryError::PlatformApiKeyNotFound { key_id } => {
Self::not_found(format!("platform api key {key_id} was not found"))
}
RegistryError::InvocationLogNotFound { log_id } => {
Self::not_found(format!("invocation log {log_id} was not found"))
}
RegistryError::PublishedAgentNotFound {
workspace_slug,
agent_slug,
apps/admin-api/src/routes.rs
+1
View File
@@ -1,6 +1,7 @@
pub mod access;
pub mod agents;
pub mod auth_profiles;
pub mod observability;
pub mod operations;
pub mod workspaces;
apps/admin-api/src/routes/observability.rs
+100
View File
@@ -0,0 +1,100 @@
use axum::{
Json,
extract::{Path, Query, State},
};
use serde_json::{Value, json};
use crate::{
error::ApiError,
routes::access::WorkspacePath,
service::{LogsQuery, UsageRequestQuery},
state::AppState,
};
#[derive(serde::Deserialize)]
pub struct WorkspaceLogPath {
pub workspace_id: String,
pub log_id: String,
}
#[derive(serde::Deserialize)]
pub struct WorkspaceOperationUsagePath {
pub workspace_id: String,
pub operation_id: String,
}
#[derive(serde::Deserialize)]
pub struct WorkspaceAgentUsagePath {
pub workspace_id: String,
pub agent_id: String,
}
pub async fn list_logs(
Path(path): Path<WorkspacePath>,
Query(query): Query<LogsQuery>,
State(state): State<AppState>,
) -> Result<Json<Value>, ApiError> {
let items = state
.service
.list_logs(&path.workspace_id.as_str().into(), query)
.await?;
Ok(Json(json!({ "items": items })))
}
pub async fn get_log(
Path(path): Path<WorkspaceLogPath>,
State(state): State<AppState>,
) -> Result<Json<Value>, ApiError> {
let item = state
.service
.get_log(
&path.workspace_id.as_str().into(),
&path.log_id.as_str().into(),
)
.await?;
Ok(Json(json!(item)))
}
pub async fn get_usage(
Path(path): Path<WorkspacePath>,
Query(query): Query<UsageRequestQuery>,
State(state): State<AppState>,
) -> Result<Json<Value>, ApiError> {
let usage = state
.service
.get_usage_overview(&path.workspace_id.as_str().into(), query)
.await?;
Ok(Json(json!(usage)))
}
pub async fn get_operation_usage(
Path(path): Path<WorkspaceOperationUsagePath>,
Query(query): Query<UsageRequestQuery>,
State(state): State<AppState>,
) -> Result<Json<Value>, ApiError> {
let usage = state
.service
.get_operation_usage(
&path.workspace_id.as_str().into(),
&path.operation_id.as_str().into(),
query,
)
.await?;
Ok(Json(json!(usage)))
}
pub async fn get_agent_usage(
Path(path): Path<WorkspaceAgentUsagePath>,
Query(query): Query<UsageRequestQuery>,
State(state): State<AppState>,
) -> Result<Json<Value>, ApiError> {
let usage = state
.service
.get_agent_usage(
&path.workspace_id.as_str().into(),
&path.agent_id.as_str().into(),
query,
)
.await?;
Ok(Json(json!(usage)))
}
apps/admin-api/src/service.rs
+347 -17
View File
@@ -4,19 +4,22 @@ use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
use crank_core::{
Agent, AgentId, AgentOperationBinding, AgentStatus, AgentVersion, AuthConfig, AuthKind,
AuthProfile, AuthProfileId, ConfigExport, ExportMode, GeneratedDraft, GeneratedDraftStatus,
InvitationId, InvitationStatus, InvitationToken, MembershipRole, OperationId, OperationStatus,
PlatformApiKey, PlatformApiKeyId, PlatformApiKeyScope, PlatformApiKeyStatus, Protocol,
SampleId, Samples, Target, Workspace, WorkspaceId, WorkspaceStatus,
InvitationId, InvitationStatus, InvitationToken, InvocationLevel, InvocationLog,
InvocationLogId, InvocationSource, InvocationStatus, MembershipRole, OperationId,
OperationStatus, PlatformApiKey, PlatformApiKeyId, PlatformApiKeyScope, PlatformApiKeyStatus,
Protocol, SampleId, Samples, Target, UsagePeriod, Workspace, WorkspaceId, WorkspaceStatus,
};
use crank_mapping::{JsonPathRoot, MappingSet, infer_mapping_from_samples};
use crank_proto::{ProtoService, services_from_descriptor_set_bytes};
use crank_registry::{
AgentSummary, AgentVersionRecord, CreateAgentRequest, CreateInvitationRequest,
CreatePlatformApiKeyRequest, CreateVersionRequest, CreateWorkspaceRequest, InvitationRecord,
CreateInvocationLogRequest, CreatePlatformApiKeyRequest, CreateVersionRequest,
CreateWorkspaceRequest, InvitationRecord, InvocationLogRecord, ListInvocationLogsQuery,
MembershipRecord, OperationSampleMetadata, OperationSummary, OperationVersionRecord,
PlatformApiKeyRecord, PostgresRegistry, PublishAgentRequest, PublishRequest, RegistryOperation,
SampleKind, SaveAgentBindingsRequest, SaveAuthProfileRequest, SaveDescriptorMetadataRequest,
SaveSampleMetadataRequest, UpdateWorkspaceRequest, WorkspaceRecord,
SaveSampleMetadataRequest, UpdateWorkspaceRequest, UsageAgentBreakdown, UsageBucket,
UsageOperationBreakdown, UsageQuery, UsageSummary, UsageTimelinePoint, WorkspaceRecord,
};
use crank_runtime::{PreparedRequest, RuntimeError, RuntimeExecutor, RuntimeOperation};
use crank_schema::Schema;
@@ -163,6 +166,31 @@ pub struct CreatedPlatformApiKeyResponse {
pub secret: String,
}
#[derive(Clone, Debug, Deserialize)]
pub struct LogsQuery {
pub level: Option<InvocationLevel>,
pub search: Option<String>,
pub source: Option<InvocationSource>,
pub operation_id: Option<String>,
pub agent_id: Option<String>,
pub period: Option<UsagePeriod>,
pub limit: Option<u32>,
}
#[derive(Clone, Debug, Deserialize)]
pub struct UsageRequestQuery {
pub period: Option<UsagePeriod>,
pub source: Option<InvocationSource>,
}
#[derive(Clone, Debug, Serialize)]
pub struct UsageOverviewResponse {
pub summary: UsageSummary,
pub timeline: Vec<UsageTimelinePoint>,
pub operations: Vec<UsageOperationBreakdown>,
pub agents: Vec<UsageAgentBreakdown>,
}
#[derive(Clone, Debug, Deserialize)]
pub struct GenerateDraftPayload {
#[serde(default)]
@@ -461,6 +489,144 @@ impl AdminService {
Ok(())
}
#[instrument(skip(self))]
pub async fn list_logs(
&self,
workspace_id: &WorkspaceId,
query: LogsQuery,
) -> Result<Vec<InvocationLogRecord>, ApiError> {
self.ensure_workspace_exists(workspace_id).await?;
let operation_id = query.operation_id.as_deref().map(OperationId::new);
let agent_id = query.agent_id.as_deref().map(AgentId::new);
let (_, created_after, _) = usage_window(query.period.unwrap_or(UsagePeriod::Last7Days))?;
self.registry
.list_invocation_logs(ListInvocationLogsQuery {
workspace_id,
level: query.level,
search_text: query.search.as_deref(),
source: query.source,
operation_id: operation_id.as_ref(),
agent_id: agent_id.as_ref(),
created_after: Some(&created_after),
limit: query.limit.unwrap_or(100),
})
.await
.map_err(ApiError::from)
}
#[instrument(skip(self))]
pub async fn get_log(
&self,
workspace_id: &WorkspaceId,
log_id: &InvocationLogId,
) -> Result<InvocationLogRecord, ApiError> {
self.registry
.get_invocation_log(workspace_id, log_id)
.await?
.ok_or_else(|| {
ApiError::not_found(format!("invocation log {} was not found", log_id.as_str()))
})
}
#[instrument(skip(self))]
pub async fn get_usage_overview(
&self,
workspace_id: &WorkspaceId,
query: UsageRequestQuery,
) -> Result<UsageOverviewResponse, ApiError> {
self.ensure_workspace_exists(workspace_id).await?;
let (period, created_after, bucket) =
usage_window(query.period.unwrap_or(UsagePeriod::Last7Days))?;
let usage_query = UsageQuery {
workspace_id,
period,
source: query.source,
created_after: &created_after,
bucket,
};
let summary = self.registry.summarize_usage(usage_query.clone()).await?;
let timeline = self
.registry
.list_usage_timeline(usage_query.clone())
.await?;
let operations = self
.registry
.list_usage_by_operation(usage_query.clone())
.await?;
let agents = self.registry.list_usage_by_agent(usage_query).await?;
Ok(UsageOverviewResponse {
summary,
timeline,
operations,
agents,
})
}
#[instrument(skip(self))]
pub async fn get_operation_usage(
&self,
workspace_id: &WorkspaceId,
operation_id: &OperationId,
query: UsageRequestQuery,
) -> Result<crank_registry::UsageRollupRecord, ApiError> {
self.get_operation(workspace_id, operation_id).await?;
let (period, created_after, bucket) =
usage_window(query.period.unwrap_or(UsagePeriod::Last7Days))?;
self.registry
.get_usage_for_operation(
UsageQuery {
workspace_id,
period,
source: query.source,
created_after: &created_after,
bucket,
},
operation_id,
)
.await?
.ok_or_else(|| {
ApiError::not_found(format!(
"usage for operation {} was not found",
operation_id.as_str()
))
})
}
#[instrument(skip(self))]
pub async fn get_agent_usage(
&self,
workspace_id: &WorkspaceId,
agent_id: &AgentId,
query: UsageRequestQuery,
) -> Result<crank_registry::UsageRollupRecord, ApiError> {
self.get_agent(workspace_id, agent_id).await?;
let (period, created_after, bucket) =
usage_window(query.period.unwrap_or(UsagePeriod::Last7Days))?;
self.registry
.get_usage_for_agent(
UsageQuery {
workspace_id,
period,
source: query.source,
created_after: &created_after,
bucket,
},
agent_id,
)
.await?
.ok_or_else(|| {
ApiError::not_found(format!(
"usage for agent {} was not found",
agent_id.as_str()
))
})
}
pub async fn list_operations(
&self,
workspace_id: &WorkspaceId,
@@ -658,6 +824,21 @@ impl AdminService {
match build_request_preview(&record.snapshot.input_mapping, &payload.input) {
Ok(preview) => preview,
Err(error) => {
self.record_invocation(
workspace_id,
None,
&record.snapshot,
InvocationSource::AdminTestRun,
InvocationLevel::Error,
InvocationStatus::Error,
"mapping preview failed".to_owned(),
None,
Some("mapping".to_owned()),
0,
Value::Null,
Value::Null,
)
.await?;
return Ok(TestRunResult {
ok: false,
request_preview: Value::Null,
@@ -669,19 +850,58 @@ impl AdminService {
}
};
let started_at = std::time::Instant::now();
match self.runtime.execute(&runtime, &payload.input).await {
Ok(output) => Ok(TestRunResult {
ok: true,
request_preview,
response_preview: output,
errors: Vec::new(),
}),
Err(error) => Ok(TestRunResult {
ok: false,
request_preview,
response_preview: Value::Null,
errors: vec![crate::error::runtime_test_failure(&error)],
}),
Ok(output) => {
let duration_ms =
u64::try_from(started_at.elapsed().as_millis()).unwrap_or(u64::MAX);
self.record_invocation(
workspace_id,
None,
&record.snapshot,
InvocationSource::AdminTestRun,
InvocationLevel::Info,
InvocationStatus::Ok,
"admin test run completed".to_owned(),
None,
None,
duration_ms,
request_preview.clone(),
output.clone(),
)
.await?;
Ok(TestRunResult {
ok: true,
request_preview,
response_preview: output,
errors: Vec::new(),
})
}
Err(error) => {
let duration_ms =
u64::try_from(started_at.elapsed().as_millis()).unwrap_or(u64::MAX);
self.record_invocation(
workspace_id,
None,
&record.snapshot,
InvocationSource::AdminTestRun,
InvocationLevel::Error,
InvocationStatus::Error,
error.to_string(),
None,
Some(runtime_error_code(&error).to_owned()),
duration_ms,
request_preview.clone(),
Value::Null,
)
.await?;
Ok(TestRunResult {
ok: false,
request_preview,
response_preview: Value::Null,
errors: vec![crate::error::runtime_test_failure(&error)],
})
}
}
}
@@ -1324,6 +1544,47 @@ impl AdminService {
async fn ensure_workspace_exists(&self, workspace_id: &WorkspaceId) -> Result<(), ApiError> {
self.get_workspace(workspace_id).await.map(|_| ())
}
async fn record_invocation(
&self,
workspace_id: &WorkspaceId,
agent_id: Option<&AgentId>,
operation: &RegistryOperation,
source: InvocationSource,
level: InvocationLevel,
status: InvocationStatus,
message: String,
status_code: Option<u16>,
error_kind: Option<String>,
duration_ms: u64,
request_preview: Value,
response_preview: Value,
) -> Result<(), ApiError> {
let log = InvocationLog {
id: InvocationLogId::new(new_prefixed_id("log")),
workspace_id: workspace_id.clone(),
agent_id: agent_id.cloned(),
operation_id: operation.id.clone(),
source,
level,
status,
tool_name: operation.name.clone(),
message,
request_id: None,
status_code,
duration_ms,
error_kind,
request_preview,
response_preview,
created_at: now_string()?,
};
self.registry
.create_invocation_log(CreateInvocationLogRequest { log: &log })
.await?;
Ok(())
}
}
fn build_request_preview(
@@ -1425,6 +1686,75 @@ fn default_invitation_expiry() -> Result<String, ApiError> {
.map_err(|error| ApiError::internal(error.to_string()))
}
fn runtime_error_code(error: &RuntimeError) -> &'static str {
match error {
RuntimeError::Schema(_) => "schema_error",
RuntimeError::Mapping(_) => "mapping_error",
RuntimeError::InvalidPreparedRequest { .. } => "invalid_request",
RuntimeError::GraphqlAdapter(_) => "graphql_error",
RuntimeError::GrpcAdapter(_) => "grpc_error",
RuntimeError::RestAdapter(_) => "rest_error",
RuntimeError::UnsupportedProtocol { .. } => "unsupported_protocol",
}
}
fn usage_window(period: UsagePeriod) -> Result<(UsagePeriod, String, UsageBucket), ApiError> {
let now = OffsetDateTime::now_utc();
let (start, bucket) = match period {
UsagePeriod::Last30Minutes => (
now.checked_sub(time::Duration::minutes(30))
.ok_or_else(|| ApiError::internal("failed to compute usage period"))?,
UsageBucket::Hour,
),
UsagePeriod::LastHour => (
now.checked_sub(time::Duration::hours(1))
.ok_or_else(|| ApiError::internal("failed to compute usage period"))?,
UsageBucket::Hour,
),
UsagePeriod::Last6Hours => (
now.checked_sub(time::Duration::hours(6))
.ok_or_else(|| ApiError::internal("failed to compute usage period"))?,
UsageBucket::Hour,
),
UsagePeriod::Last24Hours => (
now.checked_sub(time::Duration::hours(24))
.ok_or_else(|| ApiError::internal("failed to compute usage period"))?,
UsageBucket::Hour,
),
UsagePeriod::Last7Days => (
now.checked_sub(time::Duration::days(7))
.ok_or_else(|| ApiError::internal("failed to compute usage period"))?,
UsageBucket::Day,
),
UsagePeriod::Last30Days => (
now.checked_sub(time::Duration::days(30))
.ok_or_else(|| ApiError::internal("failed to compute usage period"))?,
UsageBucket::Week,
),
UsagePeriod::Last90Days => (
now.checked_sub(time::Duration::days(90))
.ok_or_else(|| ApiError::internal("failed to compute usage period"))?,
UsageBucket::Month,
),
UsagePeriod::ThisMonth => (
OffsetDateTime::from_unix_timestamp(
now.unix_timestamp() - i64::from(now.day() - 1) * 24 * 60 * 60,
)
.map_err(|error| ApiError::internal(error.to_string()))?
.replace_time(time::Time::MIDNIGHT),
UsageBucket::Week,
),
};
Ok((
period,
start
.format(&Rfc3339)
.map_err(|error| ApiError::internal(error.to_string()))?,
bucket,
))
}
fn proto_service_summary(service: ProtoService) -> Result<GrpcServiceSummary, ApiError> {
let methods = service
.unary_methods()