#!/bin/bash # Quality Guard - System-Wide Output Quality Protection # This module provides comprehensive quality monitoring for all AI thinking mechanisms # to prevent output degradation, nonsense, and repetitive responses. # --- Configuration --- MIN_RESPONSE_LENGTH=30 MAX_REPETITION_RATIO=0.4 MAX_NONSENSE_SCORE=0.6 DEGRADATION_THRESHOLD=0.65 MAX_CORRECTION_ATTEMPTS=2 FALLBACK_ENABLED=true # --- Quality Assessment Functions --- # Main quality assessment function assess_quality() { local response="$1" local context="$2" local mechanism="$3" # Calculate quality metrics local length_score=$(assess_length "$response") local coherence_score=$(assess_coherence "$response") local repetition_score=$(assess_repetition "$response") local relevance_score=$(assess_relevance "$response" "$context" "$mechanism") local structure_score=$(assess_structure "$response") # Weighted quality score local overall_score=$(echo "scale=2; ($length_score * 0.15 + $coherence_score * 0.25 + $repetition_score * 0.2 + $relevance_score * 0.25 + $structure_score * 0.15)" | bc -l 2>/dev/null || echo "0.5") echo "$overall_score" } # Assess response length assess_length() { local response="$1" local word_count=$(echo "$response" | wc -w) if [ "$word_count" -lt $MIN_RESPONSE_LENGTH ]; then echo "0.2" elif [ "$word_count" -lt 80 ]; then echo "0.6" elif [ "$word_count" -lt 200 ]; then echo "0.9" elif [ "$word_count" -lt 500 ]; then echo "0.8" else echo "0.7" fi } # Assess coherence assess_coherence() { local response="$1" # Check for reasonable sentence structure local sentences=$(echo "$response" | tr '.' '\n' | grep -v '^[[:space:]]*$' | wc -l) local avg_length=$(echo "$response" | tr '.' '\n' | grep -v '^[[:space:]]*$' | awk '{sum += length($0)} END {print sum/NR}' 2>/dev/null || echo "50") # Penalize extremely long or short sentences if (( $(echo "$avg_length > 300" | bc -l 2>/dev/null || echo "0") )); then echo "0.3" elif (( $(echo "$avg_length < 15" | bc -l 2>/dev/null || echo "0") )); then echo "0.4" elif [ "$sentences" -lt 2 ]; then echo "0.5" else echo "0.8" fi } # Assess repetition assess_repetition() { local response="$1" local unique_words=$(echo "$response" | tr ' ' '\n' | sort | uniq | wc -l) local total_words=$(echo "$response" | wc -w) if [ "$total_words" -eq 0 ]; then echo "0.0" else local repetition_ratio=$(echo "scale=2; $unique_words / $total_words" | bc -l 2>/dev/null || echo "0.5") if (( $(echo "$repetition_ratio < $MAX_REPETITION_RATIO" | bc -l 2>/dev/null || echo "0") )); then echo "0.1" elif (( $(echo "$repetition_ratio < 0.6" | bc -l 2>/dev/null || echo "0") )); then echo "0.5" else echo "0.9" fi fi } # Assess relevance to context and mechanism assess_relevance() { local response="$1" local context="$2" local mechanism="$3" # Mechanism-specific relevance checks case "$mechanism" in "puzzle") if echo "$response" | grep -q -i "algorithm\|code\|implement\|function\|solution"; then echo "0.9" else echo "0.6" fi ;; "socratic") if echo "$response" | grep -q -i "question\|analyze\|investigate\|examine\|why\|how"; then echo "0.9" else echo "0.6" fi ;; "exploration") if echo "$response" | grep -q -i "compare\|alternative\|option\|approach\|strategy"; then echo "0.9" else echo "0.6" fi ;; "consensus") if echo "$response" | grep -q -i "perspective\|view\|opinion\|agree\|disagree\|multiple"; then echo "0.9" else echo "0.6" fi ;; "critique") if echo "$response" | grep -q -i "improve\|enhance\|fix\|refine\|better\|optimize"; then echo "0.9" else echo "0.6" fi ;; "synthesis") if echo "$response" | grep -q -i "combine\|integrate\|merge\|unify\|synthesize"; then echo "0.9" else echo "0.6" fi ;; "peer_review") if echo "$response" | grep -q -i "review\|feedback\|suggest\|advice\|collaborate"; then echo "0.9" else echo "0.6" fi ;; *) echo "0.7" ;; esac } # Assess structural quality assess_structure() { local response="$1" # Check for proper formatting and structure local has_paragraphs=$(echo "$response" | grep -c '^[[:space:]]*$' 2>/dev/null | tr -d '[:space:]' || echo "0") local has_lists=$(echo "$response" | grep -c '^[[:space:]]*[-]' 2>/dev/null | tr -d '[:space:]' || echo "0") local has_numbers=$(echo "$response" | grep -c '^[[:space:]]*[0-9]' 2>/dev/null | tr -d '[:space:]' || echo "0") local structure_score=0.5 if [ "${has_paragraphs:-0}" -gt 0 ]; then structure_score=$(echo "$structure_score + 0.2" | bc -l 2>/dev/null || echo "$structure_score"); fi if [ "${has_lists:-0}" -gt 0 ]; then structure_score=$(echo "$structure_score + 0.15" | bc -l 2>/dev/null || echo "$structure_score"); fi if [ "${has_numbers:-0}" -gt 0 ]; then structure_score=$(echo "$structure_score + 0.15" | bc -l 2>/dev/null || echo "$structure_score"); fi echo "$structure_score" } # --- Degradation Detection --- # Detect various degradation patterns detect_degradation_patterns() { local response="$1" local degradation_score=0 # Check for nonsense patterns if echo "$response" | grep -q -i "lorem ipsum\|asdf\|qwerty\|random text\|test test test"; then degradation_score=$(echo "$degradation_score + 0.9" | bc -l 2>/dev/null || echo "$degradation_score") fi # Check for excessive repetition (simplified pattern) if echo "$response" | grep -q "the same phrase repeated multiple times"; then degradation_score=$(echo "$degradation_score + 0.8" | bc -l 2>/dev/null || echo "$degradation_score") fi # Check for incoherent punctuation (more specific - lines with only punctuation) local punct_only_lines=$(echo "$response" | grep "^[[:space:]]*[[:punct:]]*[[:space:]]*$" | wc -l) local total_lines=$(echo "$response" | wc -l) if [ "$total_lines" -gt 0 ]; then local punct_ratio=$(( punct_only_lines * 100 / total_lines )) if [ "$punct_ratio" -gt 50 ]; then # Only flag if more than half the lines are punctuation-only degradation_score=$(echo "$degradation_score + 0.4" | bc -l 2>/dev/null || echo "$degradation_score") fi fi # Check for extremely short responses local word_count=$(echo "$response" | wc -w) if [ "$word_count" -lt 15 ]; then degradation_score=$(echo "$degradation_score + 0.5" | bc -l 2>/dev/null || echo "$degradation_score") fi # Check for gibberish (simplified pattern) if echo "$response" | grep -q "aaaaa\|bbbbb\|ccccc\|ddddd\|eeeee"; then degradation_score=$(echo "$degradation_score + 0.6" | bc -l 2>/dev/null || echo "$degradation_score") fi # Note: Removed problematic markdown check to eliminate syntax warnings echo "$degradation_score" } # --- Correction Mechanisms --- # Attempt to correct degraded output correct_degraded_output() { local degraded_response="$1" local context="$2" local mechanism="$3" local model="$4" local attempt=1 while [ "$attempt" -le "$MAX_CORRECTION_ATTEMPTS" ]; do echo "🔄 Correction attempt $attempt/$MAX_CORRECTION_ATTEMPTS..." >&2 # Create correction prompt local correction_prompt="The previous response was degraded or nonsensical. Please provide a clear, coherent response to: ORIGINAL REQUEST: $context RESPONSE TYPE: $mechanism Please ensure your response is: - Relevant and focused on the request - Well-structured with proper paragraphs and formatting - Free of repetition, nonsense, or gibberish - Appropriate length (at least 50 words) - Clear and understandable Provide a fresh, high-quality response:" # Get corrected response local corrected_response=$(ollama run "$model" "$correction_prompt") # Assess correction quality local correction_quality=$(assess_quality "$corrected_response" "$context" "$mechanism") local degradation_score=$(detect_degradation_patterns "$corrected_response") echo "Correction quality: $correction_quality, Degradation: $degradation_score" >&2 # Check if correction is successful if (( $(echo "$correction_quality > $DEGRADATION_THRESHOLD" | bc -l 2>/dev/null || echo "0") )) && \ (( $(echo "$degradation_score < $MAX_NONSENSE_SCORE" | bc -l 2>/dev/null || echo "0") )); then echo "✅ Output corrected successfully (quality: $correction_quality)" >&2 echo "$corrected_response" return 0 fi attempt=$((attempt + 1)) done echo "❌ All correction attempts failed. Using fallback response." >&2 echo "$(generate_fallback_response "$mechanism" "$context")" return 1 } # Generate appropriate fallback response generate_fallback_response() { local mechanism="$1" local context="$2" case "$mechanism" in "puzzle") echo "I apologize, but I'm experiencing difficulties providing a proper response to your puzzle or coding challenge. Please try rephrasing your question or ask for a different type of assistance. You might also want to try breaking down your request into smaller, more specific questions." ;; "socratic") echo "I'm unable to provide the deep analysis you're looking for at this time. Please try asking your question again with more specific details, or consider rephrasing it in a different way." ;; "exploration") echo "I'm having trouble exploring alternatives and strategies for your request. Please try asking your question again or provide more context about what you're looking to explore." ;; "consensus") echo "I cannot provide multiple perspectives or consensus-building guidance currently. Please try rephrasing your request or ask for a different type of assistance." ;; "critique") echo "I'm unable to provide improvement suggestions or critique at this time. Please try asking your question again or request a different approach." ;; "synthesis") echo "I cannot synthesize or combine approaches currently. Please try rephrasing your request or ask for a different form of assistance." ;; "peer_review") echo "I'm having trouble providing collaborative feedback or review. Please try asking your question again or request a different type of help." ;; *) echo "I'm experiencing difficulties providing a proper response. Please try rephrasing your question or ask for a different type of assistance." ;; esac } # --- Main Quality Guard Function --- # Main function to guard against output degradation guard_output_quality() { local response="$1" local context="$2" local mechanism="$3" local model="$4" # Assess quality local quality_score=$(assess_quality "$response" "$context" "$mechanism") local degradation_score=$(detect_degradation_patterns "$response") echo "Quality Score: $quality_score, Degradation Score: $degradation_score" >&2 # Check if correction is needed if (( $(echo "$quality_score < $DEGRADATION_THRESHOLD" | bc -l 2>/dev/null || echo "0") )) || \ (( $(echo "$degradation_score > $MAX_NONSENSE_SCORE" | bc -l 2>/dev/null || echo "0") )); then echo "⚠️ Output quality below threshold. Initiating correction..." >&2 if [ "$FALLBACK_ENABLED" = "true" ]; then correct_degraded_output "$response" "$context" "$mechanism" "$model" else echo "❌ Quality below threshold but fallback disabled." >&2 echo "$response" fi else echo "✅ Response quality acceptable (score: $quality_score)" >&2 echo "$response" fi } # --- Utility Functions --- # Get mechanism name from script path get_mechanism_name() { local script_path="$1" local script_name=$(basename "$script_path") case "$script_name" in "puzzle") echo "puzzle" ;; "socratic") echo "socratic" ;; "exploration") echo "exploration" ;; "consensus") echo "consensus" ;; "critique") echo "critique" ;; "synthesis") echo "synthesis" ;; "peer-review") echo "peer_review" ;; *) echo "unknown" ;; esac } # Export functions for use by other scripts export -f assess_quality export -f detect_degradation_patterns export -f guard_output_quality export -f get_mechanism_name export -f correct_degraded_output export -f generate_fallback_response