Phases 5-7: Dashboard UI, Admin API, and Auth middleware
Phase 5 - Main Dashboard: - Dashboard overview page with system health stats - Jobs by status breakdown, success rates, top clients - Dashboard API (/api/dashboard/overview, by-client, problems, by-version) Phase 6 - Admin/Scraper Management: - Scrapers management page with traffic allocation UI - Admin API for scraper CRUD operations - Traffic percentage updates for A/B testing - Promote/deprecate scraper versions Phase 7 - Authentication: - API key authentication middleware - SHA-256 key hashing (keys never stored in plain text) - Scope-based authorization (jobs:read, jobs:write, admin) - Rate limiting per API key Also: - Updated api_server_production.py to include new routers - Extended core/database.py with dashboard query methods - Added dashboard link to sidebar navigation - Updated CONTEXT-KEEPER.md to mark all phases complete Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Alejandro Gutiérrez
357
core/database.py
357
core/database.py
@@ -1176,3 +1176,360 @@ class DatabaseManager:
|
||||
'by_type': by_type,
|
||||
'by_day': by_day
|
||||
}
|
||||
|
||||
# ==================== API Key Operations ====================
|
||||
|
||||
async def initialize_api_keys_schema(self):
|
||||
"""Create api_keys table if it doesn't exist."""
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS api_keys (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
key_hash VARCHAR(64) NOT NULL UNIQUE,
|
||||
key_prefix VARCHAR(8) NOT NULL,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
client_id VARCHAR(255) NOT NULL,
|
||||
scopes TEXT[] DEFAULT '{}',
|
||||
rate_limit_rpm INTEGER DEFAULT 60,
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
last_used_at TIMESTAMP,
|
||||
expires_at TIMESTAMP,
|
||||
metadata JSONB
|
||||
);
|
||||
""")
|
||||
|
||||
# Create indexes
|
||||
await conn.execute("""
|
||||
CREATE INDEX IF NOT EXISTS idx_api_keys_key_hash ON api_keys (key_hash);
|
||||
""")
|
||||
await conn.execute("""
|
||||
CREATE INDEX IF NOT EXISTS idx_api_keys_client_id ON api_keys (client_id);
|
||||
""")
|
||||
await conn.execute("""
|
||||
CREATE INDEX IF NOT EXISTS idx_api_keys_active ON api_keys (is_active) WHERE is_active = true;
|
||||
""")
|
||||
|
||||
log.info("API keys schema initialized")
|
||||
|
||||
async def get_api_key_by_hash(self, key_hash: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Look up API key by its SHA-256 hash.
|
||||
|
||||
This is the primary authentication lookup method. The hash is computed
|
||||
from the API key provided in the request header.
|
||||
|
||||
Args:
|
||||
key_hash: SHA-256 hash of the API key (64 hex characters)
|
||||
|
||||
Returns:
|
||||
API key record dictionary or None if not found:
|
||||
{
|
||||
"id": UUID,
|
||||
"key_prefix": "riq_a1b2",
|
||||
"name": "Production Key",
|
||||
"client_id": "veritas_123",
|
||||
"scopes": ["jobs:read", "jobs:write"],
|
||||
"rate_limit_rpm": 60,
|
||||
"is_active": True,
|
||||
"created_at": datetime,
|
||||
"last_used_at": datetime or None,
|
||||
"expires_at": datetime or None,
|
||||
"metadata": dict or None
|
||||
}
|
||||
"""
|
||||
async with self.pool.acquire() as conn:
|
||||
row = await conn.fetchrow("""
|
||||
SELECT
|
||||
id,
|
||||
key_prefix,
|
||||
name,
|
||||
client_id,
|
||||
scopes,
|
||||
rate_limit_rpm,
|
||||
is_active,
|
||||
created_at,
|
||||
last_used_at,
|
||||
expires_at,
|
||||
metadata
|
||||
FROM api_keys
|
||||
WHERE key_hash = $1
|
||||
""", key_hash)
|
||||
|
||||
if not row:
|
||||
return None
|
||||
|
||||
result = dict(row)
|
||||
# Convert scopes from PostgreSQL array to Python list
|
||||
result['scopes'] = list(result['scopes']) if result['scopes'] else []
|
||||
return result
|
||||
|
||||
async def create_api_key(
|
||||
self,
|
||||
client_id: str,
|
||||
name: str,
|
||||
scopes: List[str],
|
||||
rate_limit_rpm: int = 60,
|
||||
expires_at: Optional[datetime] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> tuple:
|
||||
"""
|
||||
Create a new API key for a client.
|
||||
|
||||
IMPORTANT: This method returns the plain text API key exactly once.
|
||||
After this, only the hash is stored - the key cannot be recovered.
|
||||
Make sure to display or securely transmit this key to the user.
|
||||
|
||||
Args:
|
||||
client_id: External client identifier (e.g., "veritas_client_123")
|
||||
name: Human-readable name for the key (e.g., "Production API Key")
|
||||
scopes: List of permission scopes (e.g., ["jobs:read", "jobs:write"])
|
||||
rate_limit_rpm: Maximum requests per minute (default: 60)
|
||||
expires_at: Optional expiration datetime (None = never expires)
|
||||
metadata: Optional additional metadata dict
|
||||
|
||||
Returns:
|
||||
Tuple of (plain_api_key, key_id):
|
||||
- plain_api_key: The full API key to give to the user (str)
|
||||
- key_id: UUID of the created key record
|
||||
|
||||
Security Note:
|
||||
The plain_api_key is ONLY returned here. After creation, only
|
||||
the SHA-256 hash is stored. Never log or persist the plain key.
|
||||
"""
|
||||
# Import here to avoid circular dependency
|
||||
from api.middleware.auth import generate_api_key, APIKeyAuth
|
||||
|
||||
# Generate secure random key
|
||||
plain_api_key = generate_api_key()
|
||||
|
||||
# Hash for storage
|
||||
key_hash = APIKeyAuth.hash_api_key(plain_api_key)
|
||||
|
||||
# Extract prefix for identification
|
||||
key_prefix = APIKeyAuth.get_key_prefix(plain_api_key)
|
||||
|
||||
async with self.pool.acquire() as conn:
|
||||
key_id = await conn.fetchval("""
|
||||
INSERT INTO api_keys (
|
||||
key_hash,
|
||||
key_prefix,
|
||||
name,
|
||||
client_id,
|
||||
scopes,
|
||||
rate_limit_rpm,
|
||||
expires_at,
|
||||
metadata
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
RETURNING id
|
||||
""",
|
||||
key_hash,
|
||||
key_prefix,
|
||||
name,
|
||||
client_id,
|
||||
scopes,
|
||||
rate_limit_rpm,
|
||||
expires_at,
|
||||
json.dumps(metadata) if metadata else None
|
||||
)
|
||||
|
||||
# Log creation with only the prefix (never log full key)
|
||||
log.info(
|
||||
f"Created API key {key_prefix}... for client {client_id} "
|
||||
f"with scopes {scopes}"
|
||||
)
|
||||
|
||||
return (plain_api_key, key_id)
|
||||
|
||||
async def update_api_key_last_used(self, key_id: UUID):
|
||||
"""
|
||||
Update the last_used_at timestamp for an API key.
|
||||
|
||||
Called after each successful authentication to track key usage.
|
||||
This is non-blocking and failures are logged but not raised.
|
||||
|
||||
Args:
|
||||
key_id: UUID of the API key record
|
||||
"""
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute("""
|
||||
UPDATE api_keys
|
||||
SET last_used_at = NOW()
|
||||
WHERE id = $1
|
||||
""", key_id)
|
||||
|
||||
async def revoke_api_key(self, key_id: UUID) -> bool:
|
||||
"""
|
||||
Revoke an API key by setting is_active to false.
|
||||
|
||||
This is preferred over deletion as it preserves audit history.
|
||||
The key will immediately become invalid for authentication.
|
||||
|
||||
Args:
|
||||
key_id: UUID of the API key to revoke
|
||||
|
||||
Returns:
|
||||
True if key was found and revoked, False if not found
|
||||
"""
|
||||
async with self.pool.acquire() as conn:
|
||||
result = await conn.execute("""
|
||||
UPDATE api_keys
|
||||
SET is_active = false
|
||||
WHERE id = $1 AND is_active = true
|
||||
""", key_id)
|
||||
|
||||
revoked = result.split()[-1] == "1"
|
||||
if revoked:
|
||||
log.info(f"Revoked API key {key_id}")
|
||||
return revoked
|
||||
|
||||
async def delete_api_key(self, key_id: UUID) -> bool:
|
||||
"""
|
||||
Permanently delete an API key.
|
||||
|
||||
Use revoke_api_key instead if you want to preserve audit history.
|
||||
Deletion is permanent and cannot be undone.
|
||||
|
||||
Args:
|
||||
key_id: UUID of the API key to delete
|
||||
|
||||
Returns:
|
||||
True if deleted, False if not found
|
||||
"""
|
||||
async with self.pool.acquire() as conn:
|
||||
result = await conn.execute("""
|
||||
DELETE FROM api_keys WHERE id = $1
|
||||
""", key_id)
|
||||
|
||||
deleted = result.split()[-1] == "1"
|
||||
if deleted:
|
||||
log.info(f"Deleted API key {key_id}")
|
||||
return deleted
|
||||
|
||||
async def list_api_keys_for_client(
|
||||
self,
|
||||
client_id: str,
|
||||
include_inactive: bool = False
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
List all API keys for a specific client.
|
||||
|
||||
Note: This returns key metadata only, never the actual keys
|
||||
(since we only store hashes).
|
||||
|
||||
Args:
|
||||
client_id: Client identifier to filter by
|
||||
include_inactive: Whether to include revoked keys (default: False)
|
||||
|
||||
Returns:
|
||||
List of API key records (without key_hash for security)
|
||||
"""
|
||||
async with self.pool.acquire() as conn:
|
||||
if include_inactive:
|
||||
rows = await conn.fetch("""
|
||||
SELECT
|
||||
id,
|
||||
key_prefix,
|
||||
name,
|
||||
client_id,
|
||||
scopes,
|
||||
rate_limit_rpm,
|
||||
is_active,
|
||||
created_at,
|
||||
last_used_at,
|
||||
expires_at
|
||||
FROM api_keys
|
||||
WHERE client_id = $1
|
||||
ORDER BY created_at DESC
|
||||
""", client_id)
|
||||
else:
|
||||
rows = await conn.fetch("""
|
||||
SELECT
|
||||
id,
|
||||
key_prefix,
|
||||
name,
|
||||
client_id,
|
||||
scopes,
|
||||
rate_limit_rpm,
|
||||
is_active,
|
||||
created_at,
|
||||
last_used_at,
|
||||
expires_at
|
||||
FROM api_keys
|
||||
WHERE client_id = $1 AND is_active = true
|
||||
ORDER BY created_at DESC
|
||||
""", client_id)
|
||||
|
||||
results = []
|
||||
for row in rows:
|
||||
record = dict(row)
|
||||
record['scopes'] = list(record['scopes']) if record['scopes'] else []
|
||||
results.append(record)
|
||||
|
||||
return results
|
||||
|
||||
async def get_api_key_by_id(self, key_id: UUID) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Get API key metadata by its ID.
|
||||
|
||||
Note: This returns key metadata only, never the actual key
|
||||
(since we only store hashes).
|
||||
|
||||
Args:
|
||||
key_id: UUID of the API key
|
||||
|
||||
Returns:
|
||||
API key record dictionary or None if not found
|
||||
"""
|
||||
async with self.pool.acquire() as conn:
|
||||
row = await conn.fetchrow("""
|
||||
SELECT
|
||||
id,
|
||||
key_prefix,
|
||||
name,
|
||||
client_id,
|
||||
scopes,
|
||||
rate_limit_rpm,
|
||||
is_active,
|
||||
created_at,
|
||||
last_used_at,
|
||||
expires_at,
|
||||
metadata
|
||||
FROM api_keys
|
||||
WHERE id = $1
|
||||
""", key_id)
|
||||
|
||||
if not row:
|
||||
return None
|
||||
|
||||
result = dict(row)
|
||||
result['scopes'] = list(result['scopes']) if result['scopes'] else []
|
||||
return result
|
||||
|
||||
async def update_api_key_scopes(
|
||||
self,
|
||||
key_id: UUID,
|
||||
scopes: List[str]
|
||||
) -> bool:
|
||||
"""
|
||||
Update the scopes for an API key.
|
||||
|
||||
Args:
|
||||
key_id: UUID of the API key
|
||||
scopes: New list of permission scopes
|
||||
|
||||
Returns:
|
||||
True if updated, False if key not found
|
||||
"""
|
||||
async with self.pool.acquire() as conn:
|
||||
result = await conn.execute("""
|
||||
UPDATE api_keys
|
||||
SET scopes = $2
|
||||
WHERE id = $1
|
||||
""", key_id, scopes)
|
||||
|
||||
updated = result.split()[-1] == "1"
|
||||
if updated:
|
||||
log.info(f"Updated scopes for API key {key_id}: {scopes}")
|
||||
return updated
|
||||
|
||||
Reference in New Issue
Block a user