feat: Add extensible multi-pipeline integration system
This commit implements a plugin-like pipeline architecture with:
Pipeline Core Package (packages/pipeline-core/):
- BasePipeline abstract class all pipelines implement
- PipelineRegistry for database-backed discovery/management
- PipelineRunner for execution with status tracking
- DashboardConfig contracts for dynamic widget definitions
Database Migration (006_pipeline_registry.sql):
- pipeline.registry table for registered pipelines
- pipeline.executions table for execution history
- Views for execution stats and monitoring
ReviewIQ Pipeline Refactor:
- Implements BasePipeline interface
- Adds get_dashboard_config() with widget definitions
- Adds get_widget_data() methods for all dashboard widgets
- Maintains backward compatibility with Pipeline alias
Generic Pipeline API (api/routes/pipelines.py):
- GET /api/pipelines - List all registered pipelines
- GET /api/pipelines/{id} - Pipeline details
- POST /api/pipelines/{id}/execute - Execute pipeline
- GET /api/pipelines/{id}/dashboard - Dashboard config
- GET /api/pipelines/{id}/widgets/{w} - Widget data
- GET /api/pipelines/{id}/executions - Execution history
Frontend Dynamic Dashboard System:
- DynamicDashboard component renders from config
- WidgetRegistry maps types to components
- Widget components: StatCard, LineChart, BarChart,
PieChart, DataTable, Heatmap
- Pipeline API client library
Frontend Pipeline Pages:
- /pipelines - List all registered pipelines
- /pipelines/[id] - Dynamic dashboard for pipeline
- /pipelines/[id]/executions - Execution history
- Pipelines nav item in Sidebar
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Alejandro Gutiérrez
213
web/lib/pipeline-api.ts
Normal file
213
web/lib/pipeline-api.ts
Normal file
@@ -0,0 +1,213 @@
|
||||
/**
|
||||
* Pipeline API client functions.
|
||||
*
|
||||
* Provides methods for interacting with the pipeline API endpoints.
|
||||
*/
|
||||
|
||||
import type {
|
||||
PipelineInfo,
|
||||
PipelineDetail,
|
||||
DashboardConfig,
|
||||
ExecutionStatus,
|
||||
WidgetData,
|
||||
} from './pipeline-types';
|
||||
|
||||
// API base URL - defaults to same origin in production
|
||||
const API_BASE = process.env.NEXT_PUBLIC_API_URL || '';
|
||||
|
||||
/**
|
||||
* Fetch all registered pipelines.
|
||||
*/
|
||||
export async function listPipelines(enabledOnly = true): Promise<PipelineInfo[]> {
|
||||
const url = `${API_BASE}/api/pipelines?enabled_only=${enabledOnly}`;
|
||||
const response = await fetch(url);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch pipelines: ${response.statusText}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch details for a specific pipeline.
|
||||
*/
|
||||
export async function getPipeline(pipelineId: string): Promise<PipelineDetail> {
|
||||
const url = `${API_BASE}/api/pipelines/${pipelineId}`;
|
||||
const response = await fetch(url);
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) {
|
||||
throw new Error(`Pipeline not found: ${pipelineId}`);
|
||||
}
|
||||
throw new Error(`Failed to fetch pipeline: ${response.statusText}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch dashboard configuration for a pipeline.
|
||||
*/
|
||||
export async function getDashboardConfig(pipelineId: string): Promise<DashboardConfig> {
|
||||
const url = `${API_BASE}/api/pipelines/${pipelineId}/dashboard`;
|
||||
const response = await fetch(url);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch dashboard config: ${response.statusText}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch data for a specific widget.
|
||||
*/
|
||||
export async function getWidgetData(
|
||||
pipelineId: string,
|
||||
widgetId: string,
|
||||
params: {
|
||||
business_id?: string;
|
||||
time_range?: string;
|
||||
page?: number;
|
||||
page_size?: number;
|
||||
} = {}
|
||||
): Promise<WidgetData> {
|
||||
const searchParams = new URLSearchParams();
|
||||
|
||||
if (params.business_id) {
|
||||
searchParams.set('business_id', params.business_id);
|
||||
}
|
||||
if (params.time_range) {
|
||||
searchParams.set('time_range', params.time_range);
|
||||
}
|
||||
if (params.page) {
|
||||
searchParams.set('page', params.page.toString());
|
||||
}
|
||||
if (params.page_size) {
|
||||
searchParams.set('page_size', params.page_size.toString());
|
||||
}
|
||||
|
||||
const url = `${API_BASE}/api/pipelines/${pipelineId}/widgets/${widgetId}?${searchParams}`;
|
||||
const response = await fetch(url);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch widget data: ${response.statusText}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a pipeline.
|
||||
*/
|
||||
export async function executePipeline(
|
||||
pipelineId: string,
|
||||
request: {
|
||||
job_id?: string;
|
||||
business_id?: string;
|
||||
input_data?: Record<string, unknown>;
|
||||
stages?: string[];
|
||||
options?: Record<string, unknown>;
|
||||
}
|
||||
): Promise<{
|
||||
execution_id: string;
|
||||
pipeline_id: string;
|
||||
success: boolean;
|
||||
stages_run: string[];
|
||||
error?: string;
|
||||
}> {
|
||||
const url = `${API_BASE}/api/pipelines/${pipelineId}/execute`;
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(request),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to execute pipeline: ${response.statusText}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* List execution history for a pipeline.
|
||||
*/
|
||||
export async function listExecutions(
|
||||
pipelineId: string,
|
||||
params: {
|
||||
status?: string;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
} = {}
|
||||
): Promise<ExecutionStatus[]> {
|
||||
const searchParams = new URLSearchParams();
|
||||
|
||||
if (params.status) {
|
||||
searchParams.set('status', params.status);
|
||||
}
|
||||
if (params.limit) {
|
||||
searchParams.set('limit', params.limit.toString());
|
||||
}
|
||||
if (params.offset) {
|
||||
searchParams.set('offset', params.offset.toString());
|
||||
}
|
||||
|
||||
const url = `${API_BASE}/api/pipelines/${pipelineId}/executions?${searchParams}`;
|
||||
const response = await fetch(url);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch executions: ${response.statusText}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable a pipeline.
|
||||
*/
|
||||
export async function enablePipeline(pipelineId: string): Promise<void> {
|
||||
const url = `${API_BASE}/api/pipelines/${pipelineId}/enable`;
|
||||
const response = await fetch(url, { method: 'POST' });
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to enable pipeline: ${response.statusText}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Disable a pipeline.
|
||||
*/
|
||||
export async function disablePipeline(pipelineId: string): Promise<void> {
|
||||
const url = `${API_BASE}/api/pipelines/${pipelineId}/disable`;
|
||||
const response = await fetch(url, { method: 'POST' });
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to disable pipeline: ${response.statusText}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check pipeline health.
|
||||
*/
|
||||
export async function checkPipelineHealth(
|
||||
pipelineId: string
|
||||
): Promise<{
|
||||
pipeline_id: string;
|
||||
healthy: boolean;
|
||||
checks?: Record<string, unknown>;
|
||||
message?: string;
|
||||
error?: string;
|
||||
}> {
|
||||
const url = `${API_BASE}/api/pipelines/${pipelineId}/health`;
|
||||
const response = await fetch(url);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to check pipeline health: ${response.statusText}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
194
web/lib/pipeline-types.ts
Normal file
194
web/lib/pipeline-types.ts
Normal file
@@ -0,0 +1,194 @@
|
||||
/**
|
||||
* TypeScript types for the pipeline system.
|
||||
*
|
||||
* These types mirror the Python contracts in pipeline_core/contracts.py
|
||||
*/
|
||||
|
||||
// Widget types supported by the dashboard
|
||||
export type WidgetType =
|
||||
| 'stat_card'
|
||||
| 'line_chart'
|
||||
| 'bar_chart'
|
||||
| 'pie_chart'
|
||||
| 'table'
|
||||
| 'heatmap'
|
||||
| 'area_chart'
|
||||
| 'gauge';
|
||||
|
||||
// Grid position for dashboard layout
|
||||
export interface GridPosition {
|
||||
x: number;
|
||||
y: number;
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
// Widget configuration
|
||||
export interface WidgetConfig {
|
||||
id: string;
|
||||
type: WidgetType;
|
||||
title: string;
|
||||
grid: GridPosition;
|
||||
config: Record<string, unknown>;
|
||||
data_endpoint?: string;
|
||||
refresh_interval?: number;
|
||||
}
|
||||
|
||||
// Dashboard section containing widgets
|
||||
export interface DashboardSection {
|
||||
id: string;
|
||||
title: string;
|
||||
description?: string;
|
||||
widgets: WidgetConfig[];
|
||||
collapsed?: boolean;
|
||||
}
|
||||
|
||||
// Full dashboard configuration
|
||||
export interface DashboardConfig {
|
||||
pipeline_id: string;
|
||||
title: string;
|
||||
description?: string;
|
||||
sections: DashboardSection[];
|
||||
default_time_range?: string;
|
||||
refresh_interval?: number;
|
||||
}
|
||||
|
||||
// Pipeline info (summary)
|
||||
export interface PipelineInfo {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
version: string;
|
||||
is_enabled: boolean;
|
||||
stages: string[];
|
||||
input_type: string;
|
||||
}
|
||||
|
||||
// Pipeline detail (full info)
|
||||
export interface PipelineDetail extends PipelineInfo {
|
||||
module_path: string;
|
||||
config?: Record<string, unknown>;
|
||||
created_at?: string;
|
||||
updated_at?: string;
|
||||
}
|
||||
|
||||
// Execution status
|
||||
export interface ExecutionStatus {
|
||||
id: string;
|
||||
pipeline_id: string;
|
||||
job_id?: string;
|
||||
business_id?: string;
|
||||
status: 'pending' | 'running' | 'completed' | 'failed' | 'cancelled';
|
||||
stages_requested: string[];
|
||||
stages_completed: string[];
|
||||
current_stage?: string;
|
||||
progress: number;
|
||||
error_message?: string;
|
||||
started_at?: string;
|
||||
completed_at?: string;
|
||||
created_at?: string;
|
||||
}
|
||||
|
||||
// Widget-specific data types
|
||||
|
||||
export interface StatCardData {
|
||||
[key: string]: number | string;
|
||||
}
|
||||
|
||||
export interface ChartDataPoint {
|
||||
[key: string]: number | string;
|
||||
}
|
||||
|
||||
export interface ChartData {
|
||||
data: ChartDataPoint[];
|
||||
}
|
||||
|
||||
export interface TableData {
|
||||
data: Record<string, unknown>[];
|
||||
total: number;
|
||||
}
|
||||
|
||||
export type WidgetData = StatCardData | ChartData | TableData;
|
||||
|
||||
// Widget props base
|
||||
export interface WidgetProps {
|
||||
config: WidgetConfig;
|
||||
data: WidgetData | null;
|
||||
loading: boolean;
|
||||
error?: string;
|
||||
onRefresh?: () => void;
|
||||
}
|
||||
|
||||
// Stat card specific config
|
||||
export interface StatCardConfig {
|
||||
value_key: string;
|
||||
label?: string;
|
||||
format?: string;
|
||||
trend_key?: string;
|
||||
trend_format?: string;
|
||||
icon?: string;
|
||||
color?: string;
|
||||
}
|
||||
|
||||
// Chart config
|
||||
export interface ChartAxisConfig {
|
||||
key: string;
|
||||
label?: string;
|
||||
type?: 'number' | 'category' | 'time';
|
||||
format?: string;
|
||||
}
|
||||
|
||||
export interface ChartSeriesConfig {
|
||||
key: string;
|
||||
name: string;
|
||||
color?: string;
|
||||
type?: 'line' | 'bar' | 'area';
|
||||
}
|
||||
|
||||
export interface ChartWidgetConfig {
|
||||
x_axis: ChartAxisConfig;
|
||||
y_axis: ChartAxisConfig;
|
||||
series: ChartSeriesConfig[];
|
||||
stacked?: boolean;
|
||||
show_legend?: boolean;
|
||||
show_grid?: boolean;
|
||||
}
|
||||
|
||||
// Pie chart config
|
||||
export interface PieChartConfig {
|
||||
value_key: string;
|
||||
label_key: string;
|
||||
colors?: string[];
|
||||
show_legend?: boolean;
|
||||
show_labels?: boolean;
|
||||
inner_radius?: number;
|
||||
}
|
||||
|
||||
// Table config
|
||||
export interface TableColumnConfig {
|
||||
key: string;
|
||||
header: string;
|
||||
width?: number;
|
||||
align?: 'left' | 'center' | 'right';
|
||||
format?: string;
|
||||
sortable?: boolean;
|
||||
}
|
||||
|
||||
export interface TableWidgetConfig {
|
||||
columns: TableColumnConfig[];
|
||||
row_key: string;
|
||||
page_size?: number;
|
||||
show_pagination?: boolean;
|
||||
sortable?: boolean;
|
||||
filterable?: boolean;
|
||||
}
|
||||
|
||||
// Heatmap config
|
||||
export interface HeatmapConfig {
|
||||
x_key: string;
|
||||
y_key: string;
|
||||
value_key: string;
|
||||
color_scale?: string[];
|
||||
show_values?: boolean;
|
||||
format?: string;
|
||||
}
|
||||
Reference in New Issue
Block a user