feat: Add extensible multi-pipeline integration system

This commit implements a plugin-like pipeline architecture with:

Pipeline Core Package (packages/pipeline-core/):
- BasePipeline abstract class all pipelines implement
- PipelineRegistry for database-backed discovery/management
- PipelineRunner for execution with status tracking
- DashboardConfig contracts for dynamic widget definitions

Database Migration (006_pipeline_registry.sql):
- pipeline.registry table for registered pipelines
- pipeline.executions table for execution history
- Views for execution stats and monitoring

ReviewIQ Pipeline Refactor:
- Implements BasePipeline interface
- Adds get_dashboard_config() with widget definitions
- Adds get_widget_data() methods for all dashboard widgets
- Maintains backward compatibility with Pipeline alias

Generic Pipeline API (api/routes/pipelines.py):
- GET /api/pipelines - List all registered pipelines
- GET /api/pipelines/{id} - Pipeline details
- POST /api/pipelines/{id}/execute - Execute pipeline
- GET /api/pipelines/{id}/dashboard - Dashboard config
- GET /api/pipelines/{id}/widgets/{w} - Widget data
- GET /api/pipelines/{id}/executions - Execution history

Frontend Dynamic Dashboard System:
- DynamicDashboard component renders from config
- WidgetRegistry maps types to components
- Widget components: StatCard, LineChart, BarChart,
  PieChart, DataTable, Heatmap
- Pipeline API client library

Frontend Pipeline Pages:
- /pipelines - List all registered pipelines
- /pipelines/[id] - Dynamic dashboard for pipeline
- /pipelines/[id]/executions - Execution history
- Pipelines nav item in Sidebar

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Alejandro Gutiérrez
2026-01-24 19:05:38 +00:00
parent d64f06ba9e
commit 824634aa76
30 changed files with 5697 additions and 95 deletions

View File

@@ -0,0 +1,106 @@
'use client';
import {
BarChart as RechartsBarChart,
Bar,
XAxis,
YAxis,
CartesianGrid,
Tooltip,
Legend,
ResponsiveContainer,
} from 'recharts';
import type { WidgetConfig, ChartData, ChartWidgetConfig } from '@/lib/pipeline-types';
import { WidgetWrapper } from './WidgetWrapper';
interface BarChartWidgetProps {
config: WidgetConfig;
data: ChartData | null;
loading: boolean;
error?: string;
onRefresh?: () => void;
}
// Default colors for series
const DEFAULT_COLORS = [
'#3b82f6', // blue
'#22c55e', // green
'#ef4444', // red
'#eab308', // yellow
'#8b5cf6', // purple
'#ec4899', // pink
'#06b6d4', // cyan
];
/**
* Bar chart widget using Recharts.
*/
export function BarChartWidget({
config,
data,
loading,
error,
onRefresh,
}: BarChartWidgetProps) {
const chartConfig = config.config as ChartWidgetConfig;
const chartData = data?.data || [];
return (
<WidgetWrapper config={config} loading={loading} error={error} onRefresh={onRefresh}>
{chartData.length === 0 ? (
<div className="flex items-center justify-center h-full text-gray-500">
No data available
</div>
) : (
<ResponsiveContainer width="100%" height="100%">
<RechartsBarChart
data={chartData}
margin={{ top: 5, right: 20, left: 0, bottom: 5 }}
>
{chartConfig.show_grid !== false && (
<CartesianGrid strokeDasharray="3 3" stroke="#e5e7eb" />
)}
<XAxis
dataKey={chartConfig.x_axis?.key || 'x'}
tick={{ fontSize: 12 }}
tickLine={false}
axisLine={{ stroke: '#e5e7eb' }}
/>
<YAxis
tick={{ fontSize: 12 }}
tickLine={false}
axisLine={{ stroke: '#e5e7eb' }}
label={
chartConfig.y_axis?.label
? {
value: chartConfig.y_axis.label,
angle: -90,
position: 'insideLeft',
style: { fontSize: 12 },
}
: undefined
}
/>
<Tooltip
contentStyle={{
backgroundColor: 'white',
border: '1px solid #e5e7eb',
borderRadius: '0.375rem',
}}
/>
{chartConfig.show_legend !== false && <Legend />}
{chartConfig.series?.map((series, index) => (
<Bar
key={series.key}
dataKey={series.key}
name={series.name}
fill={series.color || DEFAULT_COLORS[index % DEFAULT_COLORS.length]}
radius={[4, 4, 0, 0]}
/>
))}
</RechartsBarChart>
</ResponsiveContainer>
)}
</WidgetWrapper>
);
}

View File

@@ -0,0 +1,134 @@
'use client';
import { ChevronLeft, ChevronRight } from 'lucide-react';
import type { WidgetConfig, TableData, TableWidgetConfig } from '@/lib/pipeline-types';
import { WidgetWrapper } from './WidgetWrapper';
interface DataTableWidgetProps {
config: WidgetConfig;
data: TableData | null;
loading: boolean;
error?: string;
onRefresh?: () => void;
onPageChange?: (page: number) => void;
currentPage?: number;
}
/**
* Data table widget with pagination.
*/
export function DataTableWidget({
config,
data,
loading,
error,
onRefresh,
onPageChange,
currentPage = 1,
}: DataTableWidgetProps) {
const tableConfig = config.config as TableWidgetConfig;
const rows = data?.data || [];
const total = data?.total || 0;
const pageSize = tableConfig.page_size || 10;
const totalPages = Math.ceil(total / pageSize);
return (
<WidgetWrapper config={config} loading={loading} error={error} onRefresh={onRefresh}>
{rows.length === 0 ? (
<div className="flex items-center justify-center h-full text-gray-500">
No data available
</div>
) : (
<div className="flex flex-col h-full">
{/* Table */}
<div className="flex-1 overflow-auto">
<table className="min-w-full divide-y divide-gray-200 dark:divide-gray-700">
<thead className="bg-gray-50 dark:bg-gray-800 sticky top-0">
<tr>
{tableConfig.columns.map((col) => (
<th
key={col.key}
className={`px-4 py-3 text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider ${
col.align === 'right'
? 'text-right'
: col.align === 'center'
? 'text-center'
: 'text-left'
}`}
style={{ width: col.width ? `${col.width}px` : undefined }}
>
{col.header}
</th>
))}
</tr>
</thead>
<tbody className="bg-white dark:bg-gray-900 divide-y divide-gray-200 dark:divide-gray-700">
{rows.map((row, rowIndex) => (
<tr
key={row[tableConfig.row_key] as string || rowIndex}
className="hover:bg-gray-50 dark:hover:bg-gray-800"
>
{tableConfig.columns.map((col) => (
<td
key={col.key}
className={`px-4 py-3 text-sm text-gray-900 dark:text-gray-100 whitespace-nowrap ${
col.align === 'right'
? 'text-right'
: col.align === 'center'
? 'text-center'
: 'text-left'
}`}
>
{formatCellValue(row[col.key], col.format)}
</td>
))}
</tr>
))}
</tbody>
</table>
</div>
{/* Pagination */}
{tableConfig.show_pagination !== false && totalPages > 1 && onPageChange && (
<div className="flex items-center justify-between px-4 py-3 border-t border-gray-200 dark:border-gray-700 bg-gray-50 dark:bg-gray-800">
<div className="text-sm text-gray-500">
Showing {(currentPage - 1) * pageSize + 1} to{' '}
{Math.min(currentPage * pageSize, total)} of {total}
</div>
<div className="flex items-center space-x-2">
<button
onClick={() => onPageChange(currentPage - 1)}
disabled={currentPage <= 1}
className="p-1 rounded hover:bg-gray-200 dark:hover:bg-gray-700 disabled:opacity-50 disabled:cursor-not-allowed"
>
<ChevronLeft className="w-5 h-5" />
</button>
<span className="text-sm text-gray-700 dark:text-gray-300">
Page {currentPage} of {totalPages}
</span>
<button
onClick={() => onPageChange(currentPage + 1)}
disabled={currentPage >= totalPages}
className="p-1 rounded hover:bg-gray-200 dark:hover:bg-gray-700 disabled:opacity-50 disabled:cursor-not-allowed"
>
<ChevronRight className="w-5 h-5" />
</button>
</div>
</div>
)}
</div>
)}
</WidgetWrapper>
);
}
function formatCellValue(value: unknown, format?: string): string {
if (value === null || value === undefined) return '-';
if (typeof value === 'number') {
if (format?.includes('%')) {
return `${value.toFixed(1)}%`;
}
return value.toLocaleString();
}
return String(value);
}

View File

@@ -0,0 +1,128 @@
'use client';
import type { WidgetConfig, ChartData, HeatmapConfig } from '@/lib/pipeline-types';
import { WidgetWrapper } from './WidgetWrapper';
interface HeatmapWidgetProps {
config: WidgetConfig;
data: ChartData | null;
loading: boolean;
error?: string;
onRefresh?: () => void;
}
/**
* Heatmap widget for displaying 2D data grids.
*/
export function HeatmapWidget({
config,
data,
loading,
error,
onRefresh,
}: HeatmapWidgetProps) {
const heatmapConfig = config.config as HeatmapConfig;
const rawData = data?.data || [];
// Extract unique x and y values
const xValues = [...new Set(rawData.map((d) => d[heatmapConfig.x_key] as string))];
const yValues = [...new Set(rawData.map((d) => d[heatmapConfig.y_key] as string))];
// Find min/max values for color scaling
const values = rawData.map((d) => d[heatmapConfig.value_key] as number);
const minValue = Math.min(...values, 0);
const maxValue = Math.max(...values, 1);
// Create lookup map
const valueMap = new Map<string, number>();
rawData.forEach((d) => {
const key = `${d[heatmapConfig.y_key]}-${d[heatmapConfig.x_key]}`;
valueMap.set(key, d[heatmapConfig.value_key] as number);
});
// Get color for a value
const getColor = (value: number): string => {
const colors = heatmapConfig.color_scale || ['#f0fdf4', '#22c55e'];
const ratio = maxValue === minValue ? 0.5 : (value - minValue) / (maxValue - minValue);
// Simple interpolation between first and last color
if (colors.length === 2) {
return interpolateColor(colors[0], colors[1], ratio);
}
return colors[Math.min(Math.floor(ratio * colors.length), colors.length - 1)];
};
return (
<WidgetWrapper config={config} loading={loading} error={error} onRefresh={onRefresh}>
{rawData.length === 0 ? (
<div className="flex items-center justify-center h-full text-gray-500">
No data available
</div>
) : (
<div className="overflow-auto h-full">
<table className="min-w-full">
<thead>
<tr>
<th className="px-2 py-2 text-xs font-medium text-gray-500" />
{xValues.map((x) => (
<th
key={x}
className="px-2 py-2 text-xs font-medium text-gray-500 text-center"
>
{x}
</th>
))}
</tr>
</thead>
<tbody>
{yValues.map((y) => (
<tr key={y}>
<td className="px-2 py-2 text-xs font-medium text-gray-700 dark:text-gray-300">
{y}
</td>
{xValues.map((x) => {
const key = `${y}-${x}`;
const value = valueMap.get(key) || 0;
return (
<td
key={key}
className="px-2 py-2 text-center"
style={{ backgroundColor: getColor(value) }}
title={`${y}, ${x}: ${value}`}
>
{heatmapConfig.show_values && (
<span className="text-xs font-medium text-gray-800">
{value.toLocaleString()}
</span>
)}
</td>
);
})}
</tr>
))}
</tbody>
</table>
</div>
)}
</WidgetWrapper>
);
}
/**
* Interpolate between two hex colors.
*/
function interpolateColor(color1: string, color2: string, ratio: number): string {
const hex = (c: string) => parseInt(c, 16);
const r1 = hex(color1.slice(1, 3));
const g1 = hex(color1.slice(3, 5));
const b1 = hex(color1.slice(5, 7));
const r2 = hex(color2.slice(1, 3));
const g2 = hex(color2.slice(3, 5));
const b2 = hex(color2.slice(5, 7));
const r = Math.round(r1 + (r2 - r1) * ratio);
const g = Math.round(g1 + (g2 - g1) * ratio);
const b = Math.round(b1 + (b2 - b1) * ratio);
return `#${r.toString(16).padStart(2, '0')}${g.toString(16).padStart(2, '0')}${b.toString(16).padStart(2, '0')}`;
}

View File

@@ -0,0 +1,109 @@
'use client';
import {
LineChart as RechartsLineChart,
Line,
XAxis,
YAxis,
CartesianGrid,
Tooltip,
Legend,
ResponsiveContainer,
} from 'recharts';
import type { WidgetConfig, ChartData, ChartWidgetConfig } from '@/lib/pipeline-types';
import { WidgetWrapper } from './WidgetWrapper';
interface LineChartWidgetProps {
config: WidgetConfig;
data: ChartData | null;
loading: boolean;
error?: string;
onRefresh?: () => void;
}
// Default colors for series
const DEFAULT_COLORS = [
'#3b82f6', // blue
'#22c55e', // green
'#ef4444', // red
'#eab308', // yellow
'#8b5cf6', // purple
'#ec4899', // pink
'#06b6d4', // cyan
];
/**
* Line chart widget using Recharts.
*/
export function LineChartWidget({
config,
data,
loading,
error,
onRefresh,
}: LineChartWidgetProps) {
const chartConfig = config.config as ChartWidgetConfig;
const chartData = data?.data || [];
return (
<WidgetWrapper config={config} loading={loading} error={error} onRefresh={onRefresh}>
{chartData.length === 0 ? (
<div className="flex items-center justify-center h-full text-gray-500">
No data available
</div>
) : (
<ResponsiveContainer width="100%" height="100%">
<RechartsLineChart
data={chartData}
margin={{ top: 5, right: 20, left: 0, bottom: 5 }}
>
{chartConfig.show_grid !== false && (
<CartesianGrid strokeDasharray="3 3" stroke="#e5e7eb" />
)}
<XAxis
dataKey={chartConfig.x_axis?.key || 'x'}
tick={{ fontSize: 12 }}
tickLine={false}
axisLine={{ stroke: '#e5e7eb' }}
/>
<YAxis
tick={{ fontSize: 12 }}
tickLine={false}
axisLine={{ stroke: '#e5e7eb' }}
label={
chartConfig.y_axis?.label
? {
value: chartConfig.y_axis.label,
angle: -90,
position: 'insideLeft',
style: { fontSize: 12 },
}
: undefined
}
/>
<Tooltip
contentStyle={{
backgroundColor: 'white',
border: '1px solid #e5e7eb',
borderRadius: '0.375rem',
}}
/>
{chartConfig.show_legend !== false && <Legend />}
{chartConfig.series?.map((series, index) => (
<Line
key={series.key}
type="monotone"
dataKey={series.key}
name={series.name}
stroke={series.color || DEFAULT_COLORS[index % DEFAULT_COLORS.length]}
strokeWidth={2}
dot={false}
activeDot={{ r: 4 }}
/>
))}
</RechartsLineChart>
</ResponsiveContainer>
)}
</WidgetWrapper>
);
}

View File

@@ -0,0 +1,106 @@
'use client';
import {
PieChart as RechartsPieChart,
Pie,
Cell,
Tooltip,
Legend,
ResponsiveContainer,
} from 'recharts';
import type { WidgetConfig, ChartData, PieChartConfig } from '@/lib/pipeline-types';
import { WidgetWrapper } from './WidgetWrapper';
interface PieChartWidgetProps {
config: WidgetConfig;
data: ChartData | null;
loading: boolean;
error?: string;
onRefresh?: () => void;
}
// Default colors for pie slices
const DEFAULT_COLORS = [
'#3b82f6', // blue
'#22c55e', // green
'#ef4444', // red
'#eab308', // yellow
'#8b5cf6', // purple
'#ec4899', // pink
'#06b6d4', // cyan
'#f97316', // orange
];
/**
* Pie/Donut chart widget using Recharts.
*/
export function PieChartWidget({
config,
data,
loading,
error,
onRefresh,
}: PieChartWidgetProps) {
const chartConfig = config.config as PieChartConfig;
const chartData = data?.data || [];
const colors = chartConfig.colors || DEFAULT_COLORS;
const innerRadius = chartConfig.inner_radius || 0; // 0 = pie, > 0 = donut
// Transform data to use consistent keys
const transformedData = chartData.map((item) => ({
name: item[chartConfig.label_key] as string,
value: item[chartConfig.value_key] as number,
}));
return (
<WidgetWrapper config={config} loading={loading} error={error} onRefresh={onRefresh}>
{transformedData.length === 0 ? (
<div className="flex items-center justify-center h-full text-gray-500">
No data available
</div>
) : (
<ResponsiveContainer width="100%" height="100%">
<RechartsPieChart>
<Pie
data={transformedData}
cx="50%"
cy="50%"
innerRadius={innerRadius}
outerRadius="80%"
dataKey="value"
nameKey="name"
label={
chartConfig.show_labels !== false
? ({ name, percent }) => `${name} ${(percent * 100).toFixed(0)}%`
: undefined
}
labelLine={chartConfig.show_labels !== false}
>
{transformedData.map((_, index) => (
<Cell
key={`cell-${index}`}
fill={colors[index % colors.length]}
/>
))}
</Pie>
<Tooltip
contentStyle={{
backgroundColor: 'white',
border: '1px solid #e5e7eb',
borderRadius: '0.375rem',
}}
formatter={(value: number) => [value.toLocaleString(), 'Count']}
/>
{chartConfig.show_legend !== false && (
<Legend
layout="horizontal"
verticalAlign="bottom"
align="center"
/>
)}
</RechartsPieChart>
</ResponsiveContainer>
)}
</WidgetWrapper>
);
}

View File

@@ -0,0 +1,113 @@
'use client';
import {
TrendingUp,
TrendingDown,
MessageSquare,
CheckCircle,
AlertTriangle,
Star,
Activity,
} from 'lucide-react';
import type { WidgetConfig, StatCardData, StatCardConfig } from '@/lib/pipeline-types';
import { WidgetWrapper } from './WidgetWrapper';
interface StatCardProps {
config: WidgetConfig;
data: StatCardData | null;
loading: boolean;
error?: string;
onRefresh?: () => void;
}
// Icon mapping
const ICONS: Record<string, React.ComponentType<{ className?: string }>> = {
'message-square': MessageSquare,
'check-circle': CheckCircle,
'alert-triangle': AlertTriangle,
star: Star,
activity: Activity,
};
// Color mapping
const COLORS: Record<string, string> = {
blue: 'text-blue-600 bg-blue-100 dark:text-blue-400 dark:bg-blue-900/30',
green: 'text-green-600 bg-green-100 dark:text-green-400 dark:bg-green-900/30',
red: 'text-red-600 bg-red-100 dark:text-red-400 dark:bg-red-900/30',
yellow: 'text-yellow-600 bg-yellow-100 dark:text-yellow-400 dark:bg-yellow-900/30',
purple: 'text-purple-600 bg-purple-100 dark:text-purple-400 dark:bg-purple-900/30',
gray: 'text-gray-600 bg-gray-100 dark:text-gray-400 dark:bg-gray-700',
};
/**
* Format a value according to a format string.
* Supports: {value:,} for thousands, {value:.1f} for decimals, {value:.1%} for percentages
*/
function formatValue(value: number | string, format?: string): string {
if (!format) return String(value);
const num = typeof value === 'string' ? parseFloat(value) : value;
if (isNaN(num)) return String(value);
// Simple format parsing
if (format.includes(':,}')) {
return num.toLocaleString();
}
if (format.includes(':.1f}')) {
return num.toFixed(1);
}
if (format.includes(':.2f}')) {
return num.toFixed(2);
}
if (format.includes(':.1%}')) {
return (num * 100).toFixed(1) + '%';
}
return String(value);
}
/**
* Stat card widget for displaying KPIs.
*/
export function StatCard({ config, data, loading, error, onRefresh }: StatCardProps) {
const widgetConfig = config.config as StatCardConfig;
const Icon = widgetConfig.icon ? ICONS[widgetConfig.icon] : Activity;
const colorClass = widgetConfig.color ? COLORS[widgetConfig.color] : COLORS.gray;
// Extract value and trend from data
const value = data?.[widgetConfig.value_key] ?? 0;
const trend = widgetConfig.trend_key ? data?.[widgetConfig.trend_key] : undefined;
return (
<WidgetWrapper config={config} loading={loading} error={error} onRefresh={onRefresh}>
<div className="flex items-center justify-between h-full">
<div className="flex-1">
<p className="text-3xl font-bold text-gray-900 dark:text-gray-100">
{formatValue(value, widgetConfig.format)}
</p>
{trend !== undefined && (
<div className="flex items-center mt-1">
{Number(trend) >= 0 ? (
<TrendingUp className="w-4 h-4 text-green-500 mr-1" />
) : (
<TrendingDown className="w-4 h-4 text-red-500 mr-1" />
)}
<span
className={`text-sm ${
Number(trend) >= 0 ? 'text-green-600' : 'text-red-600'
}`}
>
{formatValue(trend, widgetConfig.trend_format || '{value:.1f}')}
</span>
</div>
)}
</div>
{Icon && (
<div className={`p-3 rounded-full ${colorClass}`}>
<Icon className="w-6 h-6" />
</div>
)}
</div>
</WidgetWrapper>
);
}

View File

@@ -0,0 +1,66 @@
'use client';
import { RefreshCw, AlertCircle } from 'lucide-react';
import type { WidgetConfig } from '@/lib/pipeline-types';
interface WidgetWrapperProps {
config: WidgetConfig;
loading: boolean;
error?: string;
onRefresh?: () => void;
children: React.ReactNode;
}
/**
* Common wrapper for dashboard widgets.
* Handles loading, error states, and refresh functionality.
*/
export function WidgetWrapper({
config,
loading,
error,
onRefresh,
children,
}: WidgetWrapperProps) {
return (
<div className="bg-white dark:bg-gray-800 rounded-lg border border-gray-200 dark:border-gray-700 shadow-sm h-full flex flex-col">
{/* Header */}
<div className="flex items-center justify-between px-4 py-3 border-b border-gray-200 dark:border-gray-700">
<h3 className="font-medium text-gray-900 dark:text-gray-100 text-sm">
{config.title}
</h3>
{onRefresh && (
<button
onClick={onRefresh}
disabled={loading}
className="p-1 text-gray-400 hover:text-gray-600 dark:hover:text-gray-300 disabled:opacity-50"
title="Refresh"
>
<RefreshCw className={`w-4 h-4 ${loading ? 'animate-spin' : ''}`} />
</button>
)}
</div>
{/* Content */}
<div className="flex-1 p-4 overflow-auto">
{error ? (
<div className="flex items-center justify-center h-full">
<div className="text-center">
<AlertCircle className="w-8 h-8 text-red-500 mx-auto mb-2" />
<p className="text-sm text-red-600 dark:text-red-400">{error}</p>
</div>
</div>
) : loading ? (
<div className="flex items-center justify-center h-full">
<div className="animate-pulse flex flex-col items-center">
<div className="h-4 w-24 bg-gray-200 dark:bg-gray-700 rounded mb-2" />
<div className="h-3 w-16 bg-gray-200 dark:bg-gray-700 rounded" />
</div>
</div>
) : (
children
)}
</div>
</div>
);
}