feat(web): add evaluation api

web/contract/console/evaluation.ts

@@ -0,0 +1,305 @@
+import type {
+  EvaluationConfig,
+  EvaluationConfigData,
+  EvaluationFileInfo,
+  EvaluationLogsResponse,
+  EvaluationMetricsListResponse,
+  EvaluationMetricsMapResponse,
+  EvaluationNodeInfoRequest,
+  EvaluationNodeInfoResponse,
+  EvaluationRun,
+  EvaluationRunDetailResponse,
+  EvaluationRunRequest,
+  EvaluationTargetType,
+  EvaluationVersionDetailResponse,
+} from '@/types/evaluation'
+import { type } from '@orpc/contract'
+import { base } from '../base'
+
+export const datasetEvaluationTemplateDownloadContract = base
+  .route({
+    path: '/datasets/{datasetId}/evaluation/template/download',
+    method: 'POST',
+  })
+  .input(type<{
+    params: {
+      datasetId: string
+    }
+  }>())
+  .output(type<unknown>())
+
+export const datasetEvaluationConfigContract = base
+  .route({
+    path: '/datasets/{datasetId}/evaluation',
+    method: 'GET',
+  })
+  .input(type<{
+    params: {
+      datasetId: string
+    }
+  }>())
+  .output(type<EvaluationConfig>())
+
+export const saveDatasetEvaluationConfigContract = base
+  .route({
+    path: '/datasets/{datasetId}/evaluation',
+    method: 'PUT',
+  })
+  .input(type<{
+    params: {
+      datasetId: string
+    }
+    body: EvaluationConfigData
+  }>())
+  .output(type<EvaluationConfig>())
+
+export const startDatasetEvaluationRunContract = base
+  .route({
+    path: '/datasets/{datasetId}/evaluation/run',
+    method: 'POST',
+  })
+  .input(type<{
+    params: {
+      datasetId: string
+    }
+    body: EvaluationRunRequest
+  }>())
+  .output(type<EvaluationRun>())
+
+export const datasetEvaluationLogsContract = base
+  .route({
+    path: '/datasets/{datasetId}/evaluation/logs',
+    method: 'GET',
+  })
+  .input(type<{
+    params: {
+      datasetId: string
+    }
+    query: {
+      page?: number
+      page_size?: number
+    }
+  }>())
+  .output(type<EvaluationLogsResponse>())
+
+export const datasetEvaluationRunDetailContract = base
+  .route({
+    path: '/datasets/{datasetId}/evaluation/runs/{runId}',
+    method: 'GET',
+  })
+  .input(type<{
+    params: {
+      datasetId: string
+      runId: string
+    }
+    query: {
+      page?: number
+      page_size?: number
+    }
+  }>())
+  .output(type<EvaluationRunDetailResponse>())
+
+export const cancelDatasetEvaluationRunContract = base
+  .route({
+    path: '/datasets/{datasetId}/evaluation/runs/{runId}/cancel',
+    method: 'POST',
+  })
+  .input(type<{
+    params: {
+      datasetId: string
+      runId: string
+    }
+  }>())
+  .output(type<EvaluationRun>())
+
+export const datasetEvaluationMetricsContract = base
+  .route({
+    path: '/datasets/{datasetId}/evaluation/metrics',
+    method: 'GET',
+  })
+  .input(type<{
+    params: {
+      datasetId: string
+    }
+  }>())
+  .output(type<EvaluationMetricsListResponse>())
+
+export const datasetEvaluationFileContract = base
+  .route({
+    path: '/datasets/{datasetId}/evaluation/files/{fileId}',
+    method: 'GET',
+  })
+  .input(type<{
+    params: {
+      datasetId: string
+      fileId: string
+    }
+  }>())
+  .output(type<EvaluationFileInfo>())
+
+export const evaluationTemplateDownloadContract = base
+  .route({
+    path: '/{targetType}/{targetId}/dataset-template/download',
+    method: 'POST',
+  })
+  .input(type<{
+    params: {
+      targetType: EvaluationTargetType
+      targetId: string
+    }
+  }>())
+  .output(type<unknown>())
+
+export const evaluationConfigContract = base
+  .route({
+    path: '/{targetType}/{targetId}/evaluation',
+    method: 'GET',
+  })
+  .input(type<{
+    params: {
+      targetType: EvaluationTargetType
+      targetId: string
+    }
+  }>())
+  .output(type<EvaluationConfig>())
+
+export const saveEvaluationConfigContract = base
+  .route({
+    path: '/{targetType}/{targetId}/evaluation',
+    method: 'PUT',
+  })
+  .input(type<{
+    params: {
+      targetType: EvaluationTargetType
+      targetId: string
+    }
+    body: EvaluationConfigData
+  }>())
+  .output(type<EvaluationConfig>())
+
+export const evaluationLogsContract = base
+  .route({
+    path: '/{targetType}/{targetId}/evaluation/logs',
+    method: 'GET',
+  })
+  .input(type<{
+    params: {
+      targetType: EvaluationTargetType
+      targetId: string
+    }
+    query: {
+      page?: number
+      page_size?: number
+    }
+  }>())
+  .output(type<EvaluationLogsResponse>())
+
+export const startEvaluationRunContract = base
+  .route({
+    path: '/{targetType}/{targetId}/evaluation/run',
+    method: 'POST',
+  })
+  .input(type<{
+    params: {
+      targetType: EvaluationTargetType
+      targetId: string
+    }
+    body: EvaluationRunRequest
+  }>())
+  .output(type<EvaluationRun>())
+
+export const evaluationRunDetailContract = base
+  .route({
+    path: '/{targetType}/{targetId}/evaluation/runs/{runId}',
+    method: 'GET',
+  })
+  .input(type<{
+    params: {
+      targetType: EvaluationTargetType
+      targetId: string
+      runId: string
+    }
+    query: {
+      page?: number
+      page_size?: number
+    }
+  }>())
+  .output(type<EvaluationRunDetailResponse>())
+
+export const cancelEvaluationRunContract = base
+  .route({
+    path: '/{targetType}/{targetId}/evaluation/runs/{runId}/cancel',
+    method: 'POST',
+  })
+  .input(type<{
+    params: {
+      targetType: EvaluationTargetType
+      targetId: string
+      runId: string
+    }
+  }>())
+  .output(type<EvaluationRun>())
+
+export const evaluationMetricsContract = base
+  .route({
+    path: '/{targetType}/{targetId}/evaluation/metrics',
+    method: 'GET',
+  })
+  .input(type<{
+    params: {
+      targetType: EvaluationTargetType
+      targetId: string
+    }
+  }>())
+  .output(type<EvaluationMetricsMapResponse>())
+
+export const evaluationNodeInfoContract = base
+  .route({
+    path: '/{targetType}/{targetId}/evaluation/node-info',
+    method: 'POST',
+  })
+  .input(type<{
+    params: {
+      targetType: EvaluationTargetType
+      targetId: string
+    }
+    body: EvaluationNodeInfoRequest
+  }>())
+  .output(type<EvaluationNodeInfoResponse>())
+
+export const availableEvaluationMetricsContract = base
+  .route({
+    path: '/evaluation/available-metrics',
+    method: 'GET',
+  })
+  .output(type<EvaluationMetricsListResponse>())
+
+export const evaluationFileContract = base
+  .route({
+    path: '/{targetType}/{targetId}/evaluation/files/{fileId}',
+    method: 'GET',
+  })
+  .input(type<{
+    params: {
+      targetType: EvaluationTargetType
+      targetId: string
+      fileId: string
+    }
+  }>())
+  .output(type<EvaluationFileInfo>())
+
+export const evaluationVersionDetailContract = base
+  .route({
+    path: '/{targetType}/{targetId}/evaluation/version',
+    method: 'GET',
+  })
+  .input(type<{
+    params: {
+      targetType: EvaluationTargetType
+      targetId: string
+    }
+    query: {
+      version: string
+    }
+  }>())
+  .output(type<EvaluationVersionDetailResponse>())

web/contract/router.ts

@@ -1,6 +1,29 @@
 import type { InferContractRouterInputs } from '@orpc/contract'
 import { appDeleteContract } from './console/apps'
 import { bindPartnerStackContract, invoicesContract } from './console/billing'
+import {
+  availableEvaluationMetricsContract,
+  cancelDatasetEvaluationRunContract,
+  cancelEvaluationRunContract,
+  datasetEvaluationConfigContract,
+  datasetEvaluationFileContract,
+  datasetEvaluationLogsContract,
+  datasetEvaluationMetricsContract,
+  datasetEvaluationRunDetailContract,
+  datasetEvaluationTemplateDownloadContract,
+  evaluationConfigContract,
+  evaluationFileContract,
+  evaluationLogsContract,
+  evaluationMetricsContract,
+  evaluationNodeInfoContract,
+  evaluationRunDetailContract,
+  evaluationTemplateDownloadContract,
+  evaluationVersionDetailContract,
+  saveDatasetEvaluationConfigContract,
+  saveEvaluationConfigContract,
+  startDatasetEvaluationRunContract,
+  startEvaluationRunContract,
+} from './console/evaluation'
 import {
   exploreAppDetailContract,
   exploreAppsContract,
@@ -97,6 +120,31 @@ export const consoleRouterContract = {
     models: modelProvidersModelsContract,
     changePreferredProviderType: changePreferredProviderTypeContract,
   },
+  evaluation: {
+    templateDownload: evaluationTemplateDownloadContract,
+    config: evaluationConfigContract,
+    saveConfig: saveEvaluationConfigContract,
+    logs: evaluationLogsContract,
+    startRun: startEvaluationRunContract,
+    runDetail: evaluationRunDetailContract,
+    cancelRun: cancelEvaluationRunContract,
+    metrics: evaluationMetricsContract,
+    nodeInfo: evaluationNodeInfoContract,
+    availableMetrics: availableEvaluationMetricsContract,
+    file: evaluationFileContract,
+    versionDetail: evaluationVersionDetailContract,
+  },
+  datasetEvaluation: {
+    templateDownload: datasetEvaluationTemplateDownloadContract,
+    config: datasetEvaluationConfigContract,
+    saveConfig: saveDatasetEvaluationConfigContract,
+    startRun: startDatasetEvaluationRunContract,
+    logs: datasetEvaluationLogsContract,
+    runDetail: datasetEvaluationRunDetailContract,
+    cancelRun: cancelDatasetEvaluationRunContract,
+    metrics: datasetEvaluationMetricsContract,
+    file: datasetEvaluationFileContract,
+  },
   plugins: {
     checkInstalled: pluginCheckInstalledContract,
     latestVersions: pluginLatestVersionsContract,

web/service/use-evaluation.ts

@@ -0,0 +1,222 @@
+import type {
+  EvaluationConfigData,
+  EvaluationNodeInfoRequest,
+  EvaluationTargetType,
+} from '@/types/evaluation'
+import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
+import { consoleQuery } from '@/service/client'
+
+export type EvaluationResourceType = 'workflow' | 'pipeline' | 'snippet'
+
+type EvaluationLogsParams = {
+  page?: number
+  page_size?: number
+}
+
+type EvaluationRunDetailParams = {
+  page?: number
+  page_size?: number
+}
+
+const toEvaluationTargetType = (resourceType: Exclude<EvaluationResourceType, 'pipeline'>): EvaluationTargetType => {
+  return resourceType === 'snippet' ? 'snippets' : 'app'
+}
+
+const invalidateEvaluationQueries = async (
+  queryClient: ReturnType<typeof useQueryClient>,
+  resourceType: EvaluationResourceType,
+) => {
+  const queryKey = resourceType === 'pipeline'
+    ? consoleQuery.datasetEvaluation.key()
+    : consoleQuery.evaluation.key()
+
+  await queryClient.invalidateQueries({ queryKey })
+}
+
+export const useEvaluationConfig = (resourceType: Exclude<EvaluationResourceType, 'pipeline'>, resourceId: string) => {
+  return useQuery(consoleQuery.evaluation.config.queryOptions({
+    input: {
+      params: {
+        targetType: toEvaluationTargetType(resourceType),
+        targetId: resourceId,
+      },
+    },
+    enabled: !!resourceId,
+  }))
+}
+
+export const useDatasetEvaluationConfig = (datasetId: string) => {
+  return useQuery(consoleQuery.datasetEvaluation.config.queryOptions({
+    input: {
+      params: { datasetId },
+    },
+    enabled: !!datasetId,
+  }))
+}
+
+export const useEvaluationLogs = (
+  resourceType: Exclude<EvaluationResourceType, 'pipeline'>,
+  resourceId: string,
+  params: EvaluationLogsParams = {},
+) => {
+  return useQuery(consoleQuery.evaluation.logs.queryOptions({
+    input: {
+      params: {
+        targetType: toEvaluationTargetType(resourceType),
+        targetId: resourceId,
+      },
+      query: params,
+    },
+    enabled: !!resourceId,
+  }))
+}
+
+export const useDatasetEvaluationLogs = (datasetId: string, params: EvaluationLogsParams = {}) => {
+  return useQuery(consoleQuery.datasetEvaluation.logs.queryOptions({
+    input: {
+      params: { datasetId },
+      query: params,
+    },
+    enabled: !!datasetId,
+  }))
+}
+
+export const useEvaluationMetrics = (resourceType: Exclude<EvaluationResourceType, 'pipeline'>, resourceId: string) => {
+  return useQuery(consoleQuery.evaluation.metrics.queryOptions({
+    input: {
+      params: {
+        targetType: toEvaluationTargetType(resourceType),
+        targetId: resourceId,
+      },
+    },
+    enabled: !!resourceId,
+  }))
+}
+
+export const useDatasetEvaluationMetrics = (datasetId: string) => {
+  return useQuery(consoleQuery.datasetEvaluation.metrics.queryOptions({
+    input: {
+      params: { datasetId },
+    },
+    enabled: !!datasetId,
+  }))
+}
+
+export const useAvailableEvaluationMetrics = (enabled = true) => {
+  return useQuery(consoleQuery.evaluation.availableMetrics.queryOptions({
+    enabled,
+  }))
+}
+
+export const useEvaluationNodeInfoMutation = () => {
+  return useMutation(consoleQuery.evaluation.nodeInfo.mutationOptions())
+}
+
+export const useSaveEvaluationConfigMutation = (resourceType: Exclude<EvaluationResourceType, 'pipeline'>) => {
+  const queryClient = useQueryClient()
+
+  return useMutation({
+    ...consoleQuery.evaluation.saveConfig.mutationOptions({
+      onSuccess: async () => {
+        await invalidateEvaluationQueries(queryClient, resourceType)
+      },
+    }),
+  })
+}
+
+export const useSaveDatasetEvaluationConfigMutation = () => {
+  const queryClient = useQueryClient()
+
+  return useMutation({
+    ...consoleQuery.datasetEvaluation.saveConfig.mutationOptions({
+      onSuccess: async () => {
+        await invalidateEvaluationQueries(queryClient, 'pipeline')
+      },
+    }),
+  })
+}
+
+export const useStartEvaluationRunMutation = (resourceType: Exclude<EvaluationResourceType, 'pipeline'>) => {
+  const queryClient = useQueryClient()
+
+  return useMutation({
+    ...consoleQuery.evaluation.startRun.mutationOptions({
+      onSuccess: async () => {
+        await invalidateEvaluationQueries(queryClient, resourceType)
+      },
+    }),
+  })
+}
+
+export const useStartDatasetEvaluationRunMutation = () => {
+  const queryClient = useQueryClient()
+
+  return useMutation({
+    ...consoleQuery.datasetEvaluation.startRun.mutationOptions({
+      onSuccess: async () => {
+        await invalidateEvaluationQueries(queryClient, 'pipeline')
+      },
+    }),
+  })
+}
+
+export const useCancelEvaluationRunMutation = (resourceType: Exclude<EvaluationResourceType, 'pipeline'>) => {
+  const queryClient = useQueryClient()
+
+  return useMutation({
+    ...consoleQuery.evaluation.cancelRun.mutationOptions({
+      onSuccess: async () => {
+        await invalidateEvaluationQueries(queryClient, resourceType)
+      },
+    }),
+  })
+}
+
+export const useCancelDatasetEvaluationRunMutation = () => {
+  const queryClient = useQueryClient()
+
+  return useMutation({
+    ...consoleQuery.datasetEvaluation.cancelRun.mutationOptions({
+      onSuccess: async () => {
+        await invalidateEvaluationQueries(queryClient, 'pipeline')
+      },
+    }),
+  })
+}
+
+export const useEvaluationRunDetail = (
+  resourceType: Exclude<EvaluationResourceType, 'pipeline'>,
+  resourceId: string,
+  runId: string,
+  params: EvaluationRunDetailParams = {},
+) => {
+  return useQuery(consoleQuery.evaluation.runDetail.queryOptions({
+    input: {
+      params: {
+        targetType: toEvaluationTargetType(resourceType),
+        targetId: resourceId,
+        runId,
+      },
+      query: params,
+    },
+    enabled: !!resourceId && !!runId,
+  }))
+}
+
+export const useDatasetEvaluationRunDetail = (datasetId: string, runId: string, params: EvaluationRunDetailParams = {}) => {
+  return useQuery(consoleQuery.datasetEvaluation.runDetail.queryOptions({
+    input: {
+      params: {
+        datasetId,
+        runId,
+      },
+      query: params,
+    },
+    enabled: !!datasetId && !!runId,
+  }))
+}
+
+export type {
+  EvaluationConfigData,
+  EvaluationNodeInfoRequest,
+}

web/types/evaluation.ts

@@ -0,0 +1,126 @@
+export type EvaluationTargetType = 'app' | 'snippets'
+
+export type EvaluationConfig = {
+  evaluation_model: string | null
+  evaluation_model_provider: string | null
+  metrics_config: Record<string, unknown> | null
+  judgement_conditions: Record<string, unknown> | null
+}
+
+export type NodeInfo = {
+  node_id: string
+  type: string
+  title: string
+}
+
+export type EvaluationDefaultMetric = {
+  metric?: string
+  node_info_list?: NodeInfo[]
+}
+
+export type EvaluationCustomizedMetric = {
+  evaluation_workflow_id?: string
+  input_fields?: Record<string, unknown>
+  output_fields?: Record<string, unknown>[]
+}
+
+export type EvaluationConfigData = {
+  evaluation_model?: string
+  evaluation_model_provider?: string
+  default_metrics?: EvaluationDefaultMetric[]
+  customized_metrics?: EvaluationCustomizedMetric | null
+  judgment_config?: Record<string, unknown> | null
+}
+
+export type EvaluationRunRequest = EvaluationConfigData & {
+  file_id: string
+}
+
+export type EvaluationRunStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled'
+
+export type EvaluationRun = {
+  id: string
+  tenant_id: string
+  target_type: string
+  target_id: string
+  evaluation_config_id: string
+  status: EvaluationRunStatus
+  dataset_file_id: string | null
+  result_file_id: string | null
+  total_items: number
+  completed_items: number
+  failed_items: number
+  progress: number
+  metrics_summary: Record<string, unknown>
+  error: string | null
+  created_by: string
+  started_at: number | null
+  completed_at: number | null
+  created_at: number
+}
+
+export type EvaluationRunMetric = {
+  name?: string
+  value?: unknown
+  details?: Record<string, unknown>
+}
+
+export type EvaluationRunItem = {
+  id: string
+  item_index: number
+  inputs: Record<string, unknown>
+  expected_output: string | null
+  actual_output: string | null
+  metrics: EvaluationRunMetric[]
+  judgment: Record<string, unknown>
+  metadata: Record<string, unknown>
+  error: string | null
+  overall_score: number | null
+}
+
+export type EvaluationLogsResponse = {
+  data: EvaluationRun[]
+  total: number
+  page: number
+  page_size: number
+}
+
+export type EvaluationRunItemsPagination = {
+  data: EvaluationRunItem[]
+  total: number
+  page: number
+  page_size: number
+}
+
+export type EvaluationRunDetailResponse = {
+  run: EvaluationRun
+  items: EvaluationRunItemsPagination
+}
+
+export type EvaluationMetricsMapResponse = {
+  metrics: Record<string, string[]>
+}
+
+export type EvaluationMetricsListResponse = {
+  metrics: string[]
+}
+
+export type EvaluationNodeInfoRequest = {
+  metrics?: string[]
+}
+
+export type EvaluationNodeInfoResponse = Record<string, NodeInfo[]>
+
+export type EvaluationFileInfo = {
+  id: string
+  name: string
+  size: number
+  extension: string
+  mime_type: string
+  created_at: number
+  download_url: string
+}
+
+export type EvaluationVersionDetailResponse = {
+  graph: Record<string, unknown>
+}