AI Services

RankFlow AI — AI Evaluation: Quality Metrics, LLM-as-Judge & Improvement Loop

Version: 2.0.0

docs/specs/ai/ai-services-03-evaluation.md

On this page

1. Quality Philosophy
1.1 Quality is Not Optional
1.2 Quality Dimensions
1.3 Quality Tiers
2. Quality Metrics Framework
2.1 Accuracy (25%)
2.2 Relevance (20%)
2.3 Voice Consistency (20%)
2.4 Compliance (15%)
2.5 Engagement (10%)
2.6 Formatting (10%)
3. Automated Evaluation Pipeline
3.1 Pipeline Flow
3.2 Evaluation Cost
3.3 Evaluation Caching
4. LLM-as-Judge
4.1 What is LLM-as-Judge?
4.2 Judge Prompt Template
4.3 Judge Configuration
4.4 Judge Calibration
5. Human-in-the-Loop
5.1 When Human Review is Required
5.2 Human Review Queue
5.3 Human Review UI
5.4 Human Feedback Loop
6. Feedback Loop & Improvement
6.1 Continuous Improvement Cycle
6.2 Weekly Quality Report
6.3 Automated Prompt Improvement Suggestions
7. Quality Score Dashboard
7.1 Admin Dashboard: AI Quality Metrics
8. Content Approval Workflow
8.1 Approval States
8.2 Approval Rules by Content Type
8.3 Medical Content Approval (24h Queue)
9. Evaluation Database Schema

Version: 2.0.0
Date: 2026-06-16
Scope: Quality metrics, automated evaluation, LLM-as-judge, human-in-the-loop, feedback loop, and continuous improvement
Target Audience: AI Engineers, Content Strategists, Quality Assurance Team
Service Path: src/server/services/ai/evaluation/

1. Quality Philosophy#

1.1 Quality is Not Optional#

Every piece of AI-generated content in RankFlow AI is evaluated before it reaches the customer. There are no exceptions. Quality evaluation is the gate between AI generation and customer-facing content.

1.2 Quality Dimensions#

Content quality is measured across 5 dimensions, each scored 0-100:

Dimension	Weight	What it Measures
Accuracy	25%	Factual correctness, no hallucinations, no false claims
Relevance	20%	Content matches the task, audience, and context
Voice Consistency	20%	Matches the practice's brand voice (tone, formality, warmth)
Compliance	15%	Passes medical/legal compliance checks
Engagement	10%	Likely to engage the target audience (measured via heuristics)
Formatting	10%	Correct structure, length, keywords, format

Overall Score: Weighted average of all dimensions. Pass threshold: 70/100.

1.3 Quality Tiers#

Score	Tier	Action
90-100	Excellent	Auto-publish, no human review needed
80-89	Good	Auto-publish, flag for periodic review
70-79	Acceptable	Auto-publish for non-medical, queue for medical review
60-69	Marginal	Queue for human review before publish
0-59	Poor	Reject, trigger regeneration (max 3 attempts), then human review

2. Quality Metrics Framework#

2.1 Accuracy (25%)#

// src/server/services/ai/evaluation/metrics/accuracy.ts

interface AccuracyScore {
  score: number; // 0-100
  reasons: string[];
  checks: {
    noHallucinations: boolean; // No made-up facts
    noContradictions: boolean; // No internal contradictions
    factualCorrectness: boolean; // Facts match known data (if verifiable)
    noOffTopic: boolean; // Content stays on topic
    claimsSubstantiated: boolean; // Claims have evidence (if applicable)
  };
}

function evaluateAccuracy(content: string, context: EvaluationContext): AccuracyScore {
  const checks = {
    noHallucinations: !containsHallucinations(content),
    noContradictions: !containsContradictions(content),
    factualCorrectness: verifyFacts(content, context.practiceId),
    noOffTopic: !isOffTopic(content, context.taskType),
    claimsSubstantiated: checkSubstantiatedClaims(content, context.isMedical),
  };
  
  const score = calculateWeightedScore(checks, [0.3, 0.2, 0.2, 0.15, 0.15]);
  
  return {
    score,
    reasons: generateReasons(checks),
    checks,
  };
}

// Hallucination detection:
// - Use LLM-as-judge to identify claims that seem fabricated
// - Cross-reference with practice data (services, locations, etc.)
// - Flag medical claims that are not in the practice's verified services
// - Check for "best in city" claims without evidence

2.2 Relevance (20%)#

// src/server/services/ai/evaluation/metrics/relevance.ts

interface RelevanceScore {
  score: number;
  reasons: string[];
  checks: {
    matchesTask: boolean; // Content matches the requested task type
    matchesAudience: boolean; // Appropriate for target audience
    matchesContext: boolean; // Includes relevant context (city, services, etc.)
    includesKeywords: boolean; // Includes target brand keywords
    appropriateLength: boolean; // Length matches platform requirements
  };
}

function evaluateRelevance(content: string, context: EvaluationContext): RelevanceScore {
  const checks = {
    matchesTask: checkTaskMatch(content, context.taskType),
    matchesAudience: checkAudienceMatch(content, context.targetAudience),
    matchesContext: checkContextInclusion(content, context.practiceId),
    includesKeywords: checkKeywordInclusion(content, context.brandKeywords),
    appropriateLength: checkLength(content, context.taskType),
  };
  
  const score = calculateWeightedScore(checks, [0.3, 0.25, 0.2, 0.15, 0.1]);
  
  return {
    score,
    reasons: generateReasons(checks),
    checks,
  };
}

2.3 Voice Consistency (20%)#

// src/server/services/ai/evaluation/metrics/voice.ts

interface VoiceScore {
  score: number;
  reasons: string[];
  checks: {
    toneMatch: boolean; // Matches practice tone (warm, professional, etc.)
    formalityMatch: boolean; // Matches practice formality level
    warmthMatch: boolean; // Matches practice warmth level
    technicalityMatch: boolean; // Matches practice technicality level
    noGenericAI: boolean; // Doesn't feel like generic AI output
  };
}

function evaluateVoice(content: string, context: EvaluationContext): VoiceScore {
  const brandProfile = context.brandProfile;
  
  const checks = {
    toneMatch: checkToneMatch(content, brandProfile.tone),
    formalityMatch: checkFormalityMatch(content, brandProfile.formalityLevel),
    warmthMatch: checkWarmthMatch(content, brandProfile.warmthLevel),
    technicalityMatch: checkTechnicalityMatch(content, brandProfile.technicalityLevel),
    noGenericAI: !isGenericAI(content), // Heuristic: "We are a..." = generic
  };
  
  const score = calculateWeightedScore(checks, [0.3, 0.2, 0.2, 0.15, 0.15]);
  
  return {
    score,
    reasons: generateReasons(checks),
    checks,
  };
}

// Generic AI detection heuristics:
// - Starts with "We are a [category] located in [city]" = generic
// - Uses "In today's world..." or "In conclusion..." = generic
// - No specific details about the practice = generic
// - Overly formal language without warmth = generic
// - No local context or cultural references = generic

2.4 Compliance (15%)#

// src/server/services/ai/evaluation/metrics/compliance.ts

interface ComplianceScore {
  score: number;
  reasons: string[];
  checks: {
    noBannedPhrases: boolean; // Doesn't contain banned phrases
    medicalDisclaimer: boolean; // Has medical disclaimer if needed
    noFalseClaims: boolean; // No unverified claims
    noSuperlatives: boolean; // No unsubstantiated superlatives
    noSensitiveTopics: boolean; // No sensitive topics without review
  };
}

function evaluateCompliance(content: string, context: EvaluationContext): ComplianceScore {
  const checks = {
    noBannedPhrases: !containsBannedPhrases(content),
    medicalDisclaimer: hasMedicalDisclaimer(content, context.isMedical),
    noFalseClaims: !containsFalseClaims(content),
    noSuperlatives: !containsUnsubstantiatedSuperlatives(content),
    noSensitiveTopics: !containsSensitiveTopics(content, context.isMedical),
  };
  
  const score = calculateWeightedScore(checks, [0.3, 0.25, 0.2, 0.15, 0.1]);
  
  return {
    score,
    reasons: generateReasons(checks),
    checks,
  };
}

// Banned phrase detection (regex + keyword list):
const BANNED_PHRASES = [
  "guaranteed cure",
  "100% success",
  "permanent fix",
  "no side effects",
  "miracle treatment",
  "instant results",
  "never fails",
  "completely safe",
  "risk-free",
  "doctor recommended" // without attribution
];

// Medical disclaimer check:
// For medical content, must contain one of:
// - "Consult a [doctor/dentist] for personalized advice"
// - "This information is for educational purposes"
// - "Always seek professional medical advice"

2.5 Engagement (10%)#

// src/server/services/ai/evaluation/metrics/engagement.ts

interface EngagementScore {
  score: number;
  reasons: string[];
  checks: {
    hasHook: boolean; // Has an engaging opening
    hasCTA: boolean; // Has a clear call to action
    readable: boolean; // Flesch reading ease score > 60
    actionable: boolean; // Content is actionable (not just informational)
    emotionalConnection: boolean; // Creates emotional connection with audience
  };
}

function evaluateEngagement(content: string, context: EvaluationContext): EngagementScore {
  const checks = {
    hasHook: hasEngagingHook(content, context.taskType),
    hasCTA: hasCallToAction(content),
    readable: checkReadability(content) > 60,
    actionable: isActionable(content),
    emotionalConnection: hasEmotionalConnection(content, context.targetAudience),
  };
  
  const score = calculateWeightedScore(checks, [0.25, 0.25, 0.2, 0.15, 0.15]);
  
  return {
    score,
    reasons: generateReasons(checks),
    checks,
  };
}

// Readability score (Flesch-Kincaid):
// > 90: Very easy (5th grade)
// 80-90: Easy (6th grade)
// 70-80: Fairly easy (7th grade)
// 60-70: Standard (8-9th grade)
// 50-60: Fairly difficult (10-12th grade)
// < 50: Difficult (college level)
// Target: 60-80 for most content

2.6 Formatting (10%)#

// src/server/services/ai/evaluation/metrics/formatting.ts

interface FormattingScore {
  score: number;
  reasons: string[];
  checks: {
    correctLength: boolean; // Within platform length limits
    correctStructure: boolean; // Has required sections (hook, body, CTA)
    includesKeywords: boolean; // Includes required brand keywords
    properFormatting: boolean; // No markdown errors, proper line breaks
    platformAppropriate: boolean; // Format matches platform (e.g., Instagram vs. LinkedIn)
  };
}

function evaluateFormatting(content: string, context: EvaluationContext): FormattingScore {
  const checks = {
    correctLength: checkLength(content, context.taskType, context.platform),
    correctStructure: checkStructure(content, context.taskType),
    includesKeywords: checkRequiredKeywords(content, context.brandKeywords),
    properFormatting: checkFormatting(content),
    platformAppropriate: checkPlatformFormat(content, context.platform),
  };
  
  const score = calculateWeightedScore(checks, [0.25, 0.25, 0.2, 0.15, 0.15]);
  
  return {
    score,
    reasons: generateReasons(checks),
    checks,
  };
}

// Length constraints by platform:
// GBP Post: 1500 chars max
// Instagram: 2200 chars max (but optimal: 125-150)
// Facebook: 63,206 chars max (but optimal: 40-80 words)
// LinkedIn: 3000 chars max (but optimal: 100-150 words)
// Twitter: 280 chars max
// Meta Description: 160 chars max
// Alt Text: 125 chars max

3. Automated Evaluation Pipeline#

3.1 Pipeline Flow#

┌─────────────────────────────────────────────────────────────┐
│  AI-Generated Content → Evaluation Pipeline                  │
│                                                              │
│  Step 1: Fast Heuristic Checks (local, no LLM)              │
│  ├─ Banned phrase scan (regex, < 1ms)                        │
│  ├─ Length validation (< 1ms)                                │
│  ├─ Keyword inclusion (< 1ms)                                │
│  ├─ Medical disclaimer check (if medical) (< 1ms)            │
│  └─ JSON validation (if JSON mode) (< 1ms)                  │
│  If any fail → REJECT immediately (no LLM cost)            │
│                                                              │
│  Step 2: LLM-as-Judge Evaluation (expensive but thorough)    │
│  ├─ Accuracy evaluation (Claude Haiku, ~$0.002)              │
│  ├─ Relevance evaluation (Claude Haiku, ~$0.002)              │
│  ├─ Voice consistency evaluation (Claude Haiku, ~$0.002)     │
│  ├─ Engagement evaluation (Claude Haiku, ~$0.002)             │
│  └─ Formatting evaluation (Claude Haiku, ~$0.002)             │
│  → Combined score (0-100)                                    │
│                                                              │
│  Step 3: Compliance Check (local + LLM)                      │
│  ├─ Banned phrases (already checked)                         │
│  ├─ Medical claims verification (Claude Haiku, ~$0.002)    │
│  └─ False claims detection (Claude Haiku, ~$0.002)           │
│                                                              │
│  Step 4: Decision                                             │
│  ├─ Score ≥ 90 → AUTO-PUBLISH                               │
│  ├─ Score 70-89 → AUTO-PUBLISH (non-medical) / QUEUE (medical)│
│  ├─ Score 60-69 → QUEUE for human review                      │
│  └─ Score < 60 → REJECT + REGENERATE (max 3 attempts)       │
│                                                              │
│  Step 5: Audit & Logging                                      │
│  ├─ Log quality score to `ai_usage` table                    │
│  ├─ Log evaluation details to `ai_evaluation` table          │
│  └─ Update quality metrics dashboard                         │
└─────────────────────────────────────────────────────────────┘

3.2 Evaluation Cost#

Step	Cost	Speed	Coverage
Fast Heuristics	$0.00	< 5ms	100% of content
LLM-as-Judge (5 dims)	~$0.01	~500ms	100% of content
Compliance Check	~$0.004	~300ms	100% of content
Total per generation	~$0.014	~800ms	Full evaluation

Total evaluation cost per generation: ~$0.014 (about 10% of generation cost).

3.3 Evaluation Caching#

// If the exact same content was evaluated before, serve cached score
// Cache key: hash(content + task + practiceId)
// Cache TTL: 24 hours (content doesn't change, but standards might)
// Cache hit rate: ~30% (many clients share similar content structures)

4. LLM-as-Judge#

4.1 What is LLM-as-Judge?#

Instead of writing complex rule-based evaluation code, we use a separate LLM (Claude Haiku — fast and cheap) to evaluate content quality. The judge LLM receives:

The generated content
The original prompt/context
The quality criteria
The brand voice profile

And returns a structured evaluation score.

4.2 Judge Prompt Template#

You are a content quality evaluator. Evaluate the following content based on the criteria below.

## Content to Evaluate
"""{{content}}"""

## Context
- Task: {{taskType}}
- Practice: {{practiceName}} ({{category}} in {{city}})
- Audience: {{targetAudience}}
- Brand Voice: {{tone}}, {{formalityLevel}}, {{warmthLevel}}
- Medical Content: {{isMedical}}

## Evaluation Criteria

### 1. Accuracy (0-100)
- Does the content contain factual errors or hallucinations?
- Are there any contradictions?
- Are claims substantiated?

### 2. Relevance (0-100)
- Does the content match the task and audience?
- Does it include appropriate context (city, services)?
- Does it include brand keywords?

### 3. Voice Consistency (0-100)
- Does the tone match the brand voice ({{tone}})?
- Does it feel personalized or generic?
- Does it avoid AI-sounding phrases?

### 4. Compliance (0-100)
- Does it contain banned phrases (guaranteed cure, 100% success, etc.)?
- Does it have medical disclaimers if needed?
- Are superlatives substantiated?

### 5. Engagement (0-100)
- Is the hook engaging?
- Is there a clear call to action?
- Is it readable and actionable?

### 6. Formatting (0-100)
- Is the length appropriate for the platform?
- Is the structure correct?
- Are keywords included?

## Output Format
Return JSON:
{
  "accuracy": { "score": 0-100, "reason": "..." },
  "relevance": { "score": 0-100, "reason": "..." },
  "voice": { "score": 0-100, "reason": "..." },
  "compliance": { "score": 0-100, "reason": "..." },
  "engagement": { "score": 0-100, "reason": "..." },
  "formatting": { "score": 0-100, "reason": "..." },
  "overall": 0-100,
  "pass": true/false,
  "issues": ["issue1", "issue2"]
}

4.3 Judge Configuration#

// The judge LLM is always Claude Haiku (fast, cheap, reliable)
const JUDGE_CONFIG = {
  model: "claude-haiku",
  temperature: 0.0, // Deterministic evaluation
  maxTokens: 1024,
  jsonMode: true,
};

// Evaluation cost: ~$0.002 per dimension (6 dimensions = ~$0.012 total)
// Can be optimized by evaluating all dimensions in a single prompt (~$0.005)

4.4 Judge Calibration#

// To ensure the judge is consistent, we calibrate it against human ratings:

// 1. Collect 100 pieces of content with human ratings
// 2. Run the judge on the same content
// 3. Calculate correlation between human and judge scores
// 4. If correlation < 0.8, adjust the judge prompt or model
// 5. Re-calibrate monthly with new human-rated samples

// Calibration dataset:
// - 100 content pieces (mix of tasks: GBP, social, landing page, blog)
// - 3 human raters per piece (average score)
// - Judge scores per piece
// - Target: Pearson correlation > 0.85 between human and judge

5. Human-in-the-Loop#

5.1 When Human Review is Required#

Scenario	Auto-Action	Human Review?	SLA
Score 90-100	Publish	No	—
Score 80-89	Publish	Optional (flagged)	—
Score 70-79 (non-medical)	Publish	No	—
Score 70-79 (medical)	Queue	Yes (required)	24 hours
Score 60-69	Queue	Yes (required)	24 hours
Score < 60	Reject + Regenerate	Yes (if 3 failures)	48 hours
Compliance FAIL	Reject	Yes (always)	24 hours
Banned phrase detected	Reject	Yes (always)	24 hours

5.2 Human Review Queue#

// Content items queued for human review go to the `content_review_queue` table
// Admin reviews them in the `/admin/ai-evaluation` dashboard

interface ContentReviewItem {
  id: string;
  content: string;
  taskType: TaskType;
  practiceId: string;
  practiceName: string;
  
  // Generated metadata
  generatedAt: Date;
  model: string;
  qualityScore: number;
  
  // Evaluation results
  evaluation: {
    accuracy: number;
    relevance: number;
    voice: number;
    compliance: number;
    engagement: number;
    formatting: number;
    issues: string[];
  };
  
  // Review status
  status: "pending" | "approved" | "rejected" | "edited";
  reviewedBy: string | null; // User ID
  reviewedAt: Date | null;
  
  // Feedback
  humanScore: number | null; // 0-100
  humanFeedback: string | null;
  editsMade: string | null; // If edited, what was changed
  
  // SLA
  slaDeadline: Date; // 24 hours from generation
  slaBreached: boolean;
}

5.3 Human Review UI#

┌─────────────────────────────────────────────────────────────┐
│  Admin Dashboard > AI Evaluation > Content Review Queue    │
│                                                              │
│  ┌─────────────────────────────────────────────────────┐   │
│  │  Pending Reviews: 12 (3 SLA Breached)               │   │
│  │  [Filter: All | Medical | Non-Medical | SLA Breached]│  │
│  │                                                      │   │
│  │  ┌─────────────────────────────────────────────┐   │   │
│  │  │  GBP Post — Dr. Smith's Dental Clinic        │   │   │
│  │  │  Generated: 2026-06-16 09:30 (3 hours ago)  │   │   │
│  │  │  Model: claude-haiku | Quality: 65/100       │   │   │
│  │  │  Issues: Low engagement, missing CTA          │   │   │
│  │  │                                              │   │   │
│  │  │  "🦷 Regular dental checkups are important..."│   │   │
│  │  │  (1500 chars, full text in preview)           │   │   │
│  │  │                                              │   │   │
│  │  │  [Approve] [Edit] [Reject] [Regenerate]     │   │   │
│  │  └─────────────────────────────────────────────┘   │   │
│  │                                                      │   │
│  │  ┌─────────────────────────────────────────────┐   │   │
│  │  │  Social Post — MediCare Hospital              │   │   │
│  │  │  Generated: 2026-06-16 08:00 (5 hours ago)  │   │   │
│  │  │  Model: gpt-4o-mini | Quality: 72/100        │   │   │
│  │  │  Issues: Compliance warning (medical claim)  │   │   │
│  │  │  SLA: 24 hours (expires in 19 hours)          │   │   │
│  │  │  [Approve] [Edit] [Reject] [Regenerate]     │   │   │
│  │  └─────────────────────────────────────────────┘   │   │
│  └─────────────────────────────────────────────────────┘   │
└─────────────────────────────────────────────────────────────┘

5.4 Human Feedback Loop#

// When a human reviews content, their feedback is used to improve the system:

// 1. Human rates the content (0-100) and provides feedback
// 2. System compares human rating to automated score
// 3. If difference > 15 points: flag for judge calibration
// 4. Human feedback is stored in `ai_feedback` table
// 5. Feedback is used to:
//    - Improve the judge prompt (adjust criteria weights)
//    - Update the practice's brand voice profile
//    - Improve the prompt template for that task
//    - Train future prompt improvements

interface AIFeedback {
  id: string;
  contentId: string;
  practiceId: string;
  taskType: string;
  
  // Human feedback
  humanScore: number;
  humanFeedback: string;
  humanIssues: string[];
  
  // Automated scores (for comparison)
  automatedScore: number;
  automatedBreakdown: Record<string, number>;
  
  // Action taken
  action: "approved" | "rejected" | "edited" | "regenerated";
  editedContent: string | null;
  
  // Learning
  usedForTraining: boolean;
  usedForCalibration: boolean;
  
  createdAt: Date;
}

6. Feedback Loop & Improvement#

6.1 Continuous Improvement Cycle#

┌─────────────────────────────────────────────────────────────┐
│  CONTINUOUS IMPROVEMENT CYCLE                               │
│                                                              │
│  1. GENERATE → AI generates content using current prompt    │
│                                                              │
│  2. EVALUATE → Automated evaluation + human review         │
│                                                              │
│  3. FEEDBACK → Humans rate and provide feedback            │
│                                                              │
│  4. ANALYZE → System analyzes patterns in feedback          │
│     - Common issues per task type                           │
│     - Common issues per practice                            │
│     - Judge calibration gaps                                │
│     - Prompt effectiveness                                  │
│                                                              │
│  5. IMPROVE → Generate prompt improvement suggestions        │
│     - "Add CTA to GBP post template"                        │
│     - "Reduce technicality for pediatric clients"         │
│     - "Improve medical disclaimer placement"                │
│                                                              │
│  6. TEST → A/B test the improved prompt                     │
│                                                              │
│  7. DEPLOY → Promote winning prompt to active               │
│                                                              │
│  8. REPEAT → Cycle continues                                │
└─────────────────────────────────────────────────────────────┘

6.2 Weekly Quality Report#

Every week, the system generates an automated quality report:

┌─────────────────────────────────────────────────────────────┐
│  Weekly AI Quality Report (2026-06-09 to 2026-06-16)       │
│                                                              │
│  GENERATIONS                                                │
│  Total: 1,234 | Approved: 1,180 (95.6%) | Rejected: 54    │
│                                                              │
│  QUALITY SCORES                                             │
│  Overall: 78.5/100 (▼ 1.2 from last week)                   │
│  ├─ Accuracy: 82.3 (▼ 0.5)                                  │
│  ├─ Relevance: 80.1 (▼ 2.1) ⚠️                              │
│  ├─ Voice: 76.8 (▼ 1.5) ⚠️                                │
│  ├─ Compliance: 95.2 (▲ 0.8) ✅                           │
│  ├─ Engagement: 71.4 (▼ 3.2) ⚠️⚠️                        │
│  └─ Formatting: 85.6 (▲ 1.0) ✅                           │
│                                                              │
│  TOP ISSUES (This Week)                                     │
│  1. Missing CTA in 23% of social posts → Suggest adding    │
│     CTA to social_caption template                          │
│  2. Voice mismatch in 15% of GBP posts → Calibrate Meta    │
│     Prompt for 3 practices                                   │
│  3. Engagement score dropped → Review hashtag strategy     │
│  4. 2 compliance failures → Update banned phrase list      │
│                                                              │
│  JUDGE CALIBRATION                                          │
│  Human-Judge correlation: 0.84 (▼ 0.03) → Needs recal      │
│                                                              │
│  PROMPT IMPROVEMENTS                                        │
│  ├─ gbp_post v2.1 → v2.2 (A/B test: +4.2% engagement)     │
│  ├─ social_caption v1.3 → v1.4 (A/B test: +2.1% voice)    │
│  └─ blog_post v2.2 → v3.0 (A/B test: +5.8% accuracy)      │
│                                                              │
│  COST                                                       │
│  Total AI cost: $892.50 | Eval cost: $89.25 (10%)           │
│  Cost per generation: $0.72 | Target: $0.70                  │
│                                                              │
│  RECOMMENDATIONS                                            │
│  1. Add CTA requirement to social_caption template          │
│  2. Recalibrate judge for engagement scoring               │
│  3. Review 3 practices with low voice scores                 │
│  4. Consider downgrading blog_post model to reduce cost    │
└─────────────────────────────────────────────────────────────┘

6.3 Automated Prompt Improvement Suggestions#

The system can suggest prompt improvements based on feedback patterns:

// Example: If 30% of social posts are missing CTAs
// System suggests:
const suggestion = {
  taskType: "social_caption",
  issue: "Missing call to action in 30% of posts",
  currentPrompt: "Write a social media post about {{topic}}.",
  suggestedPrompt: "Write a social media post about {{topic}}. IMPORTANT: Always end with a clear call to action (e.g., 'Book now', 'Visit us', 'Call {{phone}}').",
  expectedImprovement: "+15% engagement score",
  confidence: 0.85,
  aBTestRecommended: true,
};

// Admin can approve the suggestion, which triggers an A/B test

7. Quality Score Dashboard#

7.1 Admin Dashboard: AI Quality Metrics#

┌─────────────────────────────────────────────────────────────┐
│  Admin Dashboard > AI Quality Metrics                        │
│                                                              │
│  ┌─────────────────────────────────────────────────────┐   │
│  │  REAL-TIME METRICS (10s refresh)                    │   │
│  │                                                      │   │
│  │  Generations/Min: 45    Avg Latency: 1200ms         │   │
│  │  Avg Quality Score: 79.2/100                        │   │
│  │  Rejection Rate: 4.2%                               │   │
│  │  Compliance Rate: 98.8%                             │   │
│  │                                                      │   │
│  │  [Quality Score] ████████░░ 79.2                    │   │
│  │  [Rejection Rate] ██░░░░░░░░ 4.2%                  │   │
│  │  [Compliance] ██████████░ 98.8%                     │   │
│  └─────────────────────────────────────────────────────┘   │
│                                                              │
│  ┌─────────────────────────────────────────────────────┐   │
│  │  QUALITY BY TASK (Last 7 Days)                       │   │
│  │                                                      │   │
│  │  Task           │ Score │ Reject │ Compliance │ Cost │   │
│  │  ───────────────┼───────┼────────┼────────────┼──────│   │
│  │  gbp_post       │ 82.1  │ 3.1%   │ 99.2%      │ $0.003│  │
│  │  social_caption │ 76.5  │ 5.8%   │ 98.5%      │ $0.001│  │
│  │  landing_page   │ 85.3  │ 2.1%   │ 100%       │ $0.015│  │
│  │  blog_post      │ 80.2  │ 4.5%   │ 99.0%      │ $0.012│  │
│  │  review_reply   │ 88.7  │ 1.2%   │ 99.8%      │ $0.002│  │
│  │  schema_markup  │ 91.2  │ 0.5%   │ 100%       │ $0.008│  │
│  │  translation    │ 78.1  │ 6.2%   │ 97.5%      │ $0.008│  │
│  └─────────────────────────────────────────────────────┘   │
│                                                              │
│  ┌─────────────────────────────────────────────────────┐   │
│  │  QUALITY BY PRACTICE (Top 10 Lowest Scores)        │   │
│  │                                                      │   │
│  │  Practice              │ Score │ Main Issue         │   │
│  │  ──────────────────────┼───────┼────────────────────│   │
│  │  Dr. Smith's Dental   │ 65.2  │ Voice mismatch     │   │
│  │  MediCare Hospital     │ 68.1  │ Missing CTAs       │   │
│  │  Radiant Smiles       │ 70.3  │ Low engagement     │   │
│  │  ...                    │ ...   │ ...               │   │
│  └─────────────────────────────────────────────────────┘   │
│                                                              │
│  ┌─────────────────────────────────────────────────────┐   │
│  │  QUALITY TREND (30 Days)                             │   │
│  │                                                      │   │
│  │  [Line chart: Overall, Accuracy, Relevance, Voice,     │   │
│  │   Compliance, Engagement over 30 days]              │   │
│  │                                                      │   │
│  │  Trend: Overall score stable at 78-80. Engagement     │   │
│  │  dropped 3 points this week. Action recommended.    │   │
│  └─────────────────────────────────────────────────────┘   │
│                                                              │
│  ┌─────────────────────────────────────────────────────┐   │
│  │  CONTENT REVIEW QUEUE                                │   │
│  │  Pending: 12 | SLA Breached: 3 | Medical: 5          │   │
│  │  [View Queue]                                        │   │
│  └─────────────────────────────────────────────────────┘   │
│                                                              │
│  ┌─────────────────────────────────────────────────────┐   │
│  │  JUDGE CALIBRATION STATUS                            │   │
│  │  Human-Judge Correlation: 0.84 (Target: 0.85)       │   │
│  │  Last Calibrated: 2026-06-01 (15 days ago)            │   │
│  │  [Recalibrate Now]                                   │   │
│  └─────────────────────────────────────────────────────┘   │
└─────────────────────────────────────────────────────────────┘

8. Content Approval Workflow#

8.1 Approval States#

┌──────────┐     ┌──────────────┐     ┌───────────┐     ┌──────────┐
│  DRAFT   │────→│ PENDING_REVIEW│────→│ APPROVED  │────→│ PUBLISHED│
│(AI gen)  │     │(24h auto)     │     │(human ok) │     │(live)    │
└──────────┘     └──────────────┘     └───────────┘     └──────────┘
     │                │     │               │
     │                │     │               │
     ▼                ▼     ▼               ▼
  ┌─────────┐   ┌──────────┐          ┌──────────┐
  │ REJECTED│   │ AUTO-    │          │  EDITED   │
  │ (regen) │   │ PUBLISHED│          │ (requeue) │
  └─────────┘   │(non-med) │          └──────────┘
                └──────────┘

8.2 Approval Rules by Content Type#

Content Type	Auto-Publish Threshold	Medical Review?	SLA
GBP Post	Score ≥ 70	Yes (if medical)	24h
Social Post	Score ≥ 70	Yes (if medical)	24h
Landing Page	Score ≥ 80	Always	24h
Blog Post	Score ≥ 75	Yes (if medical)	24h
Review Reply	Score ≥ 70	No	None
Citation Desc	Score ≥ 60	No	None
Schema Markup	Score ≥ 80	Always	24h
Meta Desc	Score ≥ 60	No	None
Alt Text	Score ≥ 60	No	None
Translation	Score ≥ 70	Yes (if medical)	24h
Image Prompt	Score ≥ 60	No	None
Report Summary	Score ≥ 75	No	None

8.3 Medical Content Approval (24h Queue)#

// Medical content has a mandatory 24-hour approval queue:
// 1. AI generates content
// 2. Content is evaluated (must score ≥ 70)
// 3. Content is placed in `content_review_queue` with 24h SLA
// 4. Admin receives notification (email + Slack)
// 5. Admin reviews and approves/rejects within 24h
// 6. If admin doesn't respond within 24h:
//    - If score ≥ 80: Auto-publish (content is good enough)
//    - If score < 80: Auto-reject + alert admin (needs attention)
// 7. If rejected: Content is returned to draft state with feedback
// 8. If approved: Content moves to PUBLISHED state
// 9. If edited: Content is updated and re-evaluated

// Medical content is defined as:
// - isMedical = true (from Practice table)
// - OR content contains medical keywords (treatment, diagnosis, medication, surgery, etc.)
// - OR content is for a practice in categories: CLINIC, HOSPITAL, DOCTOR, DENTAL, etc.

9. Evaluation Database Schema#

-- AI Evaluation Results (per generation)
CREATE TABLE ai_evaluations (
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  usage_id UUID NOT NULL REFERENCES ai_usage(id) ON DELETE CASCADE,
  practice_id UUID NOT NULL REFERENCES practices(id) ON DELETE CASCADE,
  task_type VARCHAR(50) NOT NULL,
  
  -- Scores (0-100)
  overall_score INTEGER NOT NULL,
  accuracy_score INTEGER,
  relevance_score INTEGER,
  voice_score INTEGER,
  compliance_score INTEGER,
  engagement_score INTEGER,
  formatting_score INTEGER,
  
  -- Pass/Fail
  passed BOOLEAN NOT NULL DEFAULT FALSE,
  
  -- Evaluation details (JSON)
  evaluation_details JSONB NOT NULL,
  -- {
  --   "accuracy": { "score": 82, "reason": "No hallucinations detected" },
  --   "relevance": { "score": 75, "reason": "Missing city context" },
  --   ...
  --   "issues": ["Missing CTA", "Too technical for audience"]
  -- }
  
  -- Judge info
  judge_model VARCHAR(50) NOT NULL DEFAULT "claude-haiku",
  judge_latency_ms INTEGER,
  judge_cost_usd DECIMAL(10, 6),
  
  created_at TIMESTAMP DEFAULT NOW()
);

-- Content Review Queue
CREATE TABLE content_review_queue (
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  content_id UUID NOT NULL, -- References the content being reviewed
  content_type VARCHAR(50) NOT NULL, -- gbp_post, social_post, landing_page, etc.
  content_preview TEXT NOT NULL,
  
  practice_id UUID NOT NULL REFERENCES practices(id) ON DELETE CASCADE,
  task_type VARCHAR(50) NOT NULL,
  
  -- Generation metadata
  generated_at TIMESTAMP NOT NULL,
  model VARCHAR(50) NOT NULL,
  quality_score INTEGER NOT NULL,
  evaluation_details JSONB,
  
  -- Review status
  status VARCHAR(20) NOT NULL DEFAULT "pending", -- pending, approved, rejected, edited, auto_published, auto_rejected
  reviewed_by UUID REFERENCES users(id),
  reviewed_at TIMESTAMP,
  
  -- Human feedback
  human_score INTEGER,
  human_feedback TEXT,
  edited_content TEXT,
  
  -- SLA
  sla_deadline TIMESTAMP NOT NULL,
  sla_breached BOOLEAN DEFAULT FALSE,
  
  -- Auto-publish on SLA breach
  auto_publish_on_breach BOOLEAN DEFAULT FALSE, -- Set based on score (score >= 80 = true)
  
  created_at TIMESTAMP DEFAULT NOW()
);

-- AI Feedback (human ratings for improvement)
CREATE TABLE ai_feedback (
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  usage_id UUID NOT NULL REFERENCES ai_usage(id) ON DELETE CASCADE,
  practice_id UUID NOT NULL REFERENCES practices(id) ON DELETE CASCADE,
  task_type VARCHAR(50) NOT NULL,
  
  -- Human feedback
  human_score INTEGER NOT NULL,
  human_feedback TEXT,
  human_issues TEXT[], -- JSON array
  
  -- Automated scores (for comparison)
  automated_score INTEGER NOT NULL,
  automated_breakdown JSONB,
  
  -- Action
  action VARCHAR(20) NOT NULL, -- approved, rejected, edited, regenerated
  edited_content TEXT,
  
  -- Learning flags
  used_for_training BOOLEAN DEFAULT FALSE,
  used_for_calibration BOOLEAN DEFAULT FALSE,
  
  created_at TIMESTAMP DEFAULT NOW()
);

-- Quality Metrics (aggregated, for dashboard)
CREATE TABLE ai_quality_metrics (
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  date DATE NOT NULL,
  practice_id UUID REFERENCES practices(id) ON DELETE CASCADE, -- NULL = system-wide
  task_type VARCHAR(50), -- NULL = all tasks
  
  -- Aggregated metrics
  total_generations INTEGER NOT NULL DEFAULT 0,
  approved_count INTEGER NOT NULL DEFAULT 0,
  rejected_count INTEGER NOT NULL DEFAULT 0,
  pending_count INTEGER NOT NULL DEFAULT 0,
  
  avg_quality_score DECIMAL(5, 2),
  avg_accuracy_score DECIMAL(5, 2),
  avg_relevance_score DECIMAL(5, 2),
  avg_voice_score DECIMAL(5, 2),
  avg_compliance_score DECIMAL(5, 2),
  avg_engagement_score DECIMAL(5, 2),
  avg_formatting_score DECIMAL(5, 2),
  
  avg_cost_usd DECIMAL(10, 6),
  avg_latency_ms INTEGER,
  
  UNIQUE(date, practice_id, task_type)
);

End of AI Evaluation Documentation — RankFlow AI v2.0.0