path: root/bash/talk-to-computer/quality_guard.sh

#!/bin/bash

# Quality Guard - System-Wide Output Quality Protection
# This module provides comprehensive quality monitoring for all AI thinking mechanisms
# to prevent output degradation, nonsense, and repetitive responses.

# --- Configuration ---
MIN_RESPONSE_LENGTH=30
MAX_REPETITION_RATIO=0.4
MAX_NONSENSE_SCORE=0.6
DEGRADATION_THRESHOLD=0.65
MAX_CORRECTION_ATTEMPTS=2
FALLBACK_ENABLED=true

# --- Quality Assessment Functions ---

# Main quality assessment function
assess_quality() {
    local response="$1"
    local context="$2"
    local mechanism="$3"
    
    # Calculate quality metrics
    local length_score=$(assess_length "$response")
    local coherence_score=$(assess_coherence "$response")
    local repetition_score=$(assess_repetition "$response")
    local relevance_score=$(assess_relevance "$response" "$context" "$mechanism")
    local structure_score=$(assess_structure "$response")
    
    # Weighted quality score
    local overall_score=$(echo "scale=2; ($length_score * 0.15 + $coherence_score * 0.25 + $repetition_score * 0.2 + $relevance_score * 0.25 + $structure_score * 0.15)" | bc -l 2>/dev/null || echo "0.5")
    
    echo "$overall_score"
}

# Assess response length
assess_length() {
    local response="$1"
    local word_count=$(echo "$response" | wc -w)
    
    if [ "$word_count" -lt $MIN_RESPONSE_LENGTH ]; then
        echo "0.2"
    elif [ "$word_count" -lt 80 ]; then
        echo "0.6"
    elif [ "$word_count" -lt 200 ]; then
        echo "0.9"
    elif [ "$word_count" -lt 500 ]; then
        echo "0.8"
    else
        echo "0.7"
    fi
}

# Assess coherence
assess_coherence() {
    local response="$1"
    
    # Check for reasonable sentence structure
    local sentences=$(echo "$response" | tr '.' '\n' | grep -v '^[[:space:]]*$' | wc -l)
    local avg_length=$(echo "$response" | tr '.' '\n' | grep -v '^[[:space:]]*$' | awk '{sum += length($0)} END {print sum/NR}' 2>/dev/null || echo "50")
    
    # Penalize extremely long or short sentences
    if (( $(echo "$avg_length > 300" | bc -l 2>/dev/null || echo "0") )); then
        echo "0.3"
    elif (( $(echo "$avg_length < 15" | bc -l 2>/dev/null || echo "0") )); then
        echo "0.4"
    elif [ "$sentences" -lt 2 ]; then
        echo "0.5"
    else
        echo "0.8"
    fi
}

# Assess repetition
assess_repetition() {
    local response="$1"
    local unique_words=$(echo "$response" | tr ' ' '\n' | sort | uniq | wc -l)
    local total_words=$(echo "$response" | wc -w)
    
    if [ "$total_words" -eq 0 ]; then
        echo "0.0"
    else
        local repetition_ratio=$(echo "scale=2; $unique_words / $total_words" | bc -l 2>/dev/null || echo "0.5")
        
        if (( $(echo "$repetition_ratio < $MAX_REPETITION_RATIO" | bc -l 2>/dev/null || echo "0") )); then
            echo "0.1"
        elif (( $(echo "$repetition_ratio < 0.6" | bc -l 2>/dev/null || echo "0") )); then
            echo "0.5"
        else
            echo "0.9"
        fi
    fi
}

# Assess relevance to context and mechanism
assess_relevance() {
    local response="$1"
    local context="$2"
    local mechanism="$3"
    
    # Mechanism-specific relevance checks
    case "$mechanism" in
        "puzzle")
            if echo "$response" | grep -q -i "algorithm\|code\|implement\|function\|solution"; then
                echo "0.9"
            else
                echo "0.6"
            fi
            ;;
        "socratic")
            if echo "$response" | grep -q -i "question\|analyze\|investigate\|examine\|why\|how"; then
                echo "0.9"
            else
                echo "0.6"
            fi
            ;;
        "exploration")
            if echo "$response" | grep -q -i "compare\|alternative\|option\|approach\|strategy"; then
                echo "0.9"
            else
                echo "0.6"
            fi
            ;;
        "consensus")
            if echo "$response" | grep -q -i "perspective\|view\|opinion\|agree\|disagree\|multiple"; then
                echo "0.9"
            else
                echo "0.6"
            fi
            ;;
        "critique")
            if echo "$response" | grep -q -i "improve\|enhance\|fix\|refine\|better\|optimize"; then
                echo "0.9"
            else
                echo "0.6"
            fi
            ;;
        "synthesis")
            if echo "$response" | grep -q -i "combine\|integrate\|merge\|unify\|synthesize"; then
                echo "0.9"
            else
                echo "0.6"
            fi
            ;;
        "peer_review")
            if echo "$response" | grep -q -i "review\|feedback\|suggest\|advice\|collaborate"; then
                echo "0.9"
            else
                echo "0.6"
            fi
            ;;
        *)
            echo "0.7"
            ;;
    esac
}

# Assess structural quality
assess_structure() {
    local response="$1"
    
    # Check for proper formatting and structure
    local has_paragraphs=$(echo "$response" | grep -c '^[[:space:]]*$' 2>/dev/null | tr -d '[:space:]' || echo "0")
    local has_lists=$(echo "$response" | grep -c '^[[:space:]]*[-]' 2>/dev/null | tr -d '[:space:]' || echo "0")
    local has_numbers=$(echo "$response" | grep -c '^[[:space:]]*[0-9]' 2>/dev/null | tr -d '[:space:]' || echo "0")
    
    local structure_score=0.5
    
    if [ "${has_paragraphs:-0}" -gt 0 ]; then structure_score=$(echo "$structure_score + 0.2" | bc -l 2>/dev/null || echo "$structure_score"); fi
    if [ "${has_lists:-0}" -gt 0 ]; then structure_score=$(echo "$structure_score + 0.15" | bc -l 2>/dev/null || echo "$structure_score"); fi
    if [ "${has_numbers:-0}" -gt 0 ]; then structure_score=$(echo "$structure_score + 0.15" | bc -l 2>/dev/null || echo "$structure_score"); fi
    
    echo "$structure_score"
}

# --- Degradation Detection ---

# Detect various degradation patterns
detect_degradation_patterns() {
    local response="$1"
    local degradation_score=0
    
    # Check for nonsense patterns
    if echo "$response" | grep -q -i "lorem ipsum\|asdf\|qwerty\|random text\|test test test"; then
        degradation_score=$(echo "$degradation_score + 0.9" | bc -l 2>/dev/null || echo "$degradation_score")
    fi
    
    # Check for excessive repetition (simplified pattern)
    if echo "$response" | grep -q "the same phrase repeated multiple times"; then
        degradation_score=$(echo "$degradation_score + 0.8" | bc -l 2>/dev/null || echo "$degradation_score")
    fi
    
    # Check for incoherent punctuation (more specific - lines with only punctuation)
    local punct_only_lines=$(echo "$response" | grep "^[[:space:]]*[[:punct:]]*[[:space:]]*$" | wc -l)
    local total_lines=$(echo "$response" | wc -l)
    if [ "$total_lines" -gt 0 ]; then
        local punct_ratio=$(( punct_only_lines * 100 / total_lines ))
        if [ "$punct_ratio" -gt 50 ]; then
            # Only flag if more than half the lines are punctuation-only
            degradation_score=$(echo "$degradation_score + 0.4" | bc -l 2>/dev/null || echo "$degradation_score")
        fi
    fi
    
    # Check for extremely short responses
    local word_count=$(echo "$response" | wc -w)
    if [ "$word_count" -lt 15 ]; then
        degradation_score=$(echo "$degradation_score + 0.5" | bc -l 2>/dev/null || echo "$degradation_score")
    fi
    
    # Check for gibberish (simplified pattern)
    if echo "$response" | grep -q "aaaaa\|bbbbb\|ccccc\|ddddd\|eeeee"; then
        degradation_score=$(echo "$degradation_score + 0.6" | bc -l 2>/dev/null || echo "$degradation_score")
    fi
    
    # Note: Removed problematic markdown check to eliminate syntax warnings
    
    echo "$degradation_score"
}

# --- Correction Mechanisms ---

# Attempt to correct degraded output
correct_degraded_output() {
    local degraded_response="$1"
    local context="$2"
    local mechanism="$3"
    local model="$4"
    local attempt=1
    
    while [ "$attempt" -le "$MAX_CORRECTION_ATTEMPTS" ]; do
        echo "🔄 Correction attempt $attempt/$MAX_CORRECTION_ATTEMPTS..." >&2
        
        # Create correction prompt
        local correction_prompt="The previous response was degraded or nonsensical. Please provide a clear, coherent response to:

ORIGINAL REQUEST: $context

RESPONSE TYPE: $mechanism

Please ensure your response is:
- Relevant and focused on the request
- Well-structured with proper paragraphs and formatting
- Free of repetition, nonsense, or gibberish
- Appropriate length (at least 50 words)
- Clear and understandable

Provide a fresh, high-quality response:"
        
        # Get corrected response
        local corrected_response=$(ollama run "$model" "$correction_prompt")
        
        # Assess correction quality
        local correction_quality=$(assess_quality "$corrected_response" "$context" "$mechanism")
        local degradation_score=$(detect_degradation_patterns "$corrected_response")
        
        echo "Correction quality: $correction_quality, Degradation: $degradation_score" >&2
        
        # Check if correction is successful
        if (( $(echo "$correction_quality > $DEGRADATION_THRESHOLD" | bc -l 2>/dev/null || echo "0") )) && \
           (( $(echo "$degradation_score < $MAX_NONSENSE_SCORE" | bc -l 2>/dev/null || echo "0") )); then
            
            echo "✅ Output corrected successfully (quality: $correction_quality)" >&2
            echo "$corrected_response"
            return 0
        fi
        
        attempt=$((attempt + 1))
    done
    
    echo "❌ All correction attempts failed. Using fallback response." >&2
    echo "$(generate_fallback_response "$mechanism" "$context")"
    return 1
}

# Generate appropriate fallback response
generate_fallback_response() {
    local mechanism="$1"
    local context="$2"
    
    case "$mechanism" in
        "puzzle")
            echo "I apologize, but I'm experiencing difficulties providing a proper response to your puzzle or coding challenge. Please try rephrasing your question or ask for a different type of assistance. You might also want to try breaking down your request into smaller, more specific questions."
            ;;
        "socratic")
            echo "I'm unable to provide the deep analysis you're looking for at this time. Please try asking your question again with more specific details, or consider rephrasing it in a different way."
            ;;
        "exploration")
            echo "I'm having trouble exploring alternatives and strategies for your request. Please try asking your question again or provide more context about what you're looking to explore."
            ;;
        "consensus")
            echo "I cannot provide multiple perspectives or consensus-building guidance currently. Please try rephrasing your request or ask for a different type of assistance."
            ;;
        "critique")
            echo "I'm unable to provide improvement suggestions or critique at this time. Please try asking your question again or request a different approach."
            ;;
        "synthesis")
            echo "I cannot synthesize or combine approaches currently. Please try rephrasing your request or ask for a different form of assistance."
            ;;
        "peer_review")
            echo "I'm having trouble providing collaborative feedback or review. Please try asking your question again or request a different type of help."
            ;;
        *)
            echo "I'm experiencing difficulties providing a proper response. Please try rephrasing your question or ask for a different type of assistance."
            ;;
    esac
}

# --- Main Quality Guard Function ---

# Main function to guard against output degradation
guard_output_quality() {
    local response="$1"
    local context="$2"
    local mechanism="$3"
    local model="$4"
    
    # Assess quality
    local quality_score=$(assess_quality "$response" "$context" "$mechanism")
    local degradation_score=$(detect_degradation_patterns "$response")
    
    echo "Quality Score: $quality_score, Degradation Score: $degradation_score" >&2
    
    # Check if correction is needed
    if (( $(echo "$quality_score < $DEGRADATION_THRESHOLD" | bc -l 2>/dev/null || echo "0") )) || \
       (( $(echo "$degradation_score > $MAX_NONSENSE_SCORE" | bc -l 2>/dev/null || echo "0") )); then
        
        echo "⚠️  Output quality below threshold. Initiating correction..." >&2
        
        if [ "$FALLBACK_ENABLED" = "true" ]; then
            correct_degraded_output "$response" "$context" "$mechanism" "$model"
        else
            echo "❌ Quality below threshold but fallback disabled." >&2
            echo "$response"
        fi
    else
        echo "✅ Response quality acceptable (score: $quality_score)" >&2
        echo "$response"
    fi
}

# --- Utility Functions ---

# Get mechanism name from script path
get_mechanism_name() {
    local script_path="$1"
    local script_name=$(basename "$script_path")
    
    case "$script_name" in
        "puzzle") echo "puzzle" ;;
        "socratic") echo "socratic" ;;
        "exploration") echo "exploration" ;;
        "consensus") echo "consensus" ;;
        "critique") echo "critique" ;;
        "synthesis") echo "synthesis" ;;
        "peer-review") echo "peer_review" ;;
        *) echo "unknown" ;;
    esac
}

# Export functions for use by other scripts
export -f assess_quality
export -f detect_degradation_patterns
export -f guard_output_quality
export -f get_mechanism_name
export -f correct_degraded_output
export -f generate_fallback_response