diff options
| author | elioat <{ID}+{username}@users.noreply.github.com> | 2025-07-31 10:57:06 -0400 |
|---|---|---|
| committer | elioat <{ID}+{username}@users.noreply.github.com> | 2025-07-31 10:57:06 -0400 |
| commit | 0a45bdc13d1de92c04979c4a6be27db2fdcf8c85 (patch) | |
| tree | 232f24aadca656023fe629ab48588ec88a103afc /awk | |
| parent | 7ec899534ff14bc71bbe39a5f5e252de4a7b6694 (diff) | |
| download | tour-0a45bdc13d1de92c04979c4a6be27db2fdcf8c85.tar.gz | |
*
Diffstat (limited to 'awk')
| -rw-r--r-- | awk/rawk/example.rawk | 13 | ||||
| -rw-r--r-- | awk/rawk/rawk.awk | 468 |
2 files changed, 205 insertions, 276 deletions
diff --git a/awk/rawk/example.rawk b/awk/rawk/example.rawk index e5bce3c..bda56b7 100644 --- a/awk/rawk/example.rawk +++ b/awk/rawk/example.rawk @@ -1,8 +1,8 @@ -# Apache Log Processing Example -# This demonstrates most rawk features with real-world data processing +# This demonstrates most rawk features in a setting familiar to awk # Usage: awk -f rawk.awk example.rawk | awk -f - sample.log -# Predicate functions for log analysis +# User defined predicate functions for log analysis +# This shows off rawk's single line function syntax $is_error = (status) -> status >= 400; $is_success = (status) -> status >= 200 && status < 300; $is_large_request = (bytes) -> bytes > 1000000; # > 1MB @@ -10,6 +10,7 @@ $is_api_request = (url) -> index(url, "/api/") > 0; $is_bot = (user_agent) -> index(user_agent, "bot") > 0 || index(user_agent, "crawler") > 0; # Data parsing and transformation functions +# These show off rawk's multi-line arrow functions $extract_status = (request_line) -> { split(request_line, parts, " ") # The status code is the second part, not the third @@ -46,7 +47,7 @@ BEGIN { # Process each log line { # Parse Apache log format: IP - - [timestamp] "method url status" bytes "referer" "user-agent" - # Use a simpler approach with multiple regex matches + # Note that we use a series of simpler regex matches, rather than trying to do it all at once if (match($0, /^([0-9.]+)/)) { ip = substr($0, RSTART, RLENGTH) @@ -78,7 +79,7 @@ BEGIN { # Store for analysis request_count++ - # Real-time processing with standard library predicates + # Real-time processing using some standard library predicates if (http_is_server_error(status)) { server_error_count++ error_report = format_error_report(ip, status, url, user_agent) @@ -147,7 +148,7 @@ END { print "Bot Requests:", bot_count print "Large Requests (>1MB):", large_count - # Demonstrate functional programming features + # Some functional patterns at play, map, flatMap, and take. if (api_count > 0) { print "" print "API Usage Analysis" diff --git a/awk/rawk/rawk.awk b/awk/rawk/rawk.awk index 034dba6..f1d8dff 100644 --- a/awk/rawk/rawk.awk +++ b/awk/rawk/rawk.awk @@ -22,274 +22,200 @@ # # Process input data # awk -f rawk.awk processor.rawk | awk -f - input.txt # -# # Apache Log Processing Example -# # This demonstrates most rawk features with real-world data processing -# # Usage: awk -f rawk.awk example.rawk | awk -f - sample.log -# -# # Predicate functions for log analysis -# $is_error = (status) -> status >= 400; -# $is_success = (status) -> status >= 200 && status < 300; -# $is_large_request = (bytes) -> bytes > 1000000; # > 1MB -# $is_api_request = (url) -> index(url, "/api/") > 0; -# $is_bot = (user_agent) -> index(user_agent, "bot") > 0 || index(user_agent, "crawler") > 0; -# -# # Data parsing and transformation functions -# $extract_status = (request_line) -> { -# split(request_line, parts, " ") -# # The status code is the second part, not the third -# return parts[2] -# }; -# -# $extract_method = (request_line) -> { -# split(request_line, parts, " ") -# return parts[1] -# }; -# -# $extract_url = (request_line) -> { -# split(request_line, parts, " ") -# return parts[2] -# }; -# -# # Aggregation and reporting functions -# $format_error_report = (ip, status, url, user_agent) -> { -# return "ERROR: " status " - " ip " accessed " url " (" user_agent ")" -# }; -# -# $format_success_report = (ip, method, url, bytes) -> { -# size_label = is_large_request(bytes) ? "LARGE" : "normal" -# return "SUCCESS: " method " " url " (" bytes " bytes, " size_label ")" -# }; -# -# # Main processing pipeline -# BEGIN { -# print "Apache Log Analysis Report" -# print "=============================" -# print "" -# } -# -# # Process each log line -# { -# # Parse Apache log format: IP - - [timestamp] "method url status" bytes "referer" "user-agent" -# # Use a simpler approach with multiple regex matches -# if (match($0, /^([0-9.]+)/)) { -# ip = substr($0, RSTART, RLENGTH) -# -# # Extract request (method url protocol) -# if (match($0, /"([^"]+)"/)) { -# request = substr($0, RSTART + 1, RLENGTH - 2) -# # Extract method and URL from request -# method = extract_method(request) -# url = extract_url(request) -# } -# -# # Extract status code (number after the request) -# if (match($0, /" ([0-9]+) /)) { -# status = substr($0, RSTART + 1, RLENGTH - 2) -# # Remove leading/trailing spaces -# gsub(/^[ \t]+|[ \t]+$/, "", status) -# } -# -# # Extract bytes (number after request) -# if (match($0, /" ([0-9]+) /)) { -# bytes = substr($0, RSTART + 1, RLENGTH - 2) -# } -# -# # Extract user agent (last quoted field) -# if (match($0, /"([^"]*)"$/)) { -# user_agent = substr($0, RSTART + 1, RLENGTH - 2) -# } -# -# # Store for analysis -# request_count++ -# -# # Real-time processing with standard library predicates -# if (http_is_server_error(status)) { -# server_error_count++ -# error_report = format_error_report(ip, status, url, user_agent) -# print "SERVER ERROR: " error_report -# } else if (http_is_client_error(status)) { -# client_error_count++ -# error_report = format_error_report(ip, status, url, user_agent) -# print "CLIENT ERROR: " error_report -# } else if (is_success(status)) { -# success_count++ -# success_report = format_success_report(ip, method, url, bytes) -# print "✓ " success_report -# } -# -# # Track different types of requests -# if (is_api_request(url)) { -# api_count++ -# api_urls[api_count] = url -# } -# -# if (url_is_static_file(url)) { -# static_count++ -# static_urls[static_count] = url -# } -# -# if (http_is_mutating_method(method)) { -# mutation_count++ -# if (ip_is_public(ip)) { -# print "EXTERNAL MUTATION: " ip " " method " " url -# } -# } -# -# # Track user types -# if (is_bot(user_agent)) { -# bot_count++ -# bot_agents[bot_count] = user_agent -# } else if (user_agent_is_mobile(user_agent)) { -# mobile_count++ -# } else if (user_agent_is_desktop(user_agent)) { -# desktop_count++ -# } -# -# # Track large requests -# if (is_large_request(bytes)) { -# large_count++ -# large_urls[large_count] = url -# } -# } -# } -# -# END { -# print "" -# print "Summary Statistics" -# print "====================" -# print "Total Requests:", request_count -# print "Successful:", success_count -# print "Client Errors:", client_error_count -# print "Server Errors:", server_error_count -# print "Total Errors:", client_error_count + server_error_count -# print "Error Rate:", sprintf("%.2f%%", ((client_error_count + server_error_count) / request_count) * 100) -# print "API Requests:", api_count -# print "Static Files:", static_count -# print "Mutating Requests:", mutation_count -# print "Mobile Users:", mobile_count -# print "Desktop Users:", desktop_count -# print "Bot Requests:", bot_count -# print "Large Requests (>1MB):", large_count -# -# # Demonstrate functional programming features -# if (api_count > 0) { -# print "" -# print "API Usage Analysis" -# print "====================" -# -# # Use map to extract API endpoints -# $extract_endpoint = (url) -> url; -# endpoint_count = map("extract_endpoint", api_urls, endpoints) -# print "API Endpoints found:", endpoint_count -# } -# -# if (bot_count > 0) { -# print "" -# print "Bot Activity Analysis" -# print "========================" -# -# # Use flatMap to extract bot user agent components -# $extract_bot_components = (user_agent, result) -> { -# split(user_agent, result, " ") -# return length(result) -# }; -# bot_components_count = flatMap("extract_bot_components", bot_agents, bot_components) -# print "Bot components analyzed:", bot_components_count -# -# # Use take to show top 3 bot components -# top_components_count = take(3, bot_components, top_components) -# print "Top bot components:", top_components_count -# } -# -# print "" -# print "End analysis" -# } -# -# # Real-time processing -# if (is_error(status)) { -# error_report = format_error_report(log_entry["ip"], status, url, log_entry["user_agent"]) -# print "🚨 " error_report -# } else if (is_success(status)) { -# success_report = format_success_report(log_entry["ip"], method, url, bytes) -# print "✅ " success_report -# } -# -# # Track API usage -# if (is_api_request(url)) { -# api_requests[++api_count] = log_entry -# } -# -# # Track bot activity -# if (is_bot(log_entry["user_agent"])) { -# bot_requests[++bot_count] = log_entry -# } -# } -# } -# -# END { -# print "" -# print "📊 Summary Statistics" -# print "====================" -# -# # Calculate overall statistics -# stats_count = calculate_stats(requests, statistics) -# print "Total Requests:", statistics["total"] -# print "Successful:", statistics["success"] -# print "Errors:", statistics["errors"] -# print "Error Rate:", sprintf("%.2f%%", statistics["error_rate"]) -# print "Total Data Transferred:", statistics["bytes"], "bytes" -# -# # API analysis using functional programming -# if (api_count > 0) { -# print "" -# print "🔌 API Usage Analysis" -# print "====================" -# -# # Use map to extract API endpoints -# $extract_endpoint = (entry) -> extract_url(entry["request"]); -# endpoint_count = map("extract_endpoint", api_requests, endpoints) -# -# # Use filter to find high-traffic endpoints -# $is_high_traffic = (endpoint) -> { -# count = 0 -# for (i in endpoints) { -# if (endpoints[i] == endpoint) count++ -# } -# return count > 5 -# }; -# high_traffic_count = filter("is_high_traffic", endpoints, high_traffic_endpoints) -# -# print "API Requests:", api_count -# print "High-Traffic Endpoints:", high_traffic_count -# } -# -# # Bot analysis using enhanced utilities -# if (bot_count > 0) { -# print "" -# print "🤖 Bot Activity Analysis" -# print "========================" -# -# # Use flatMap to extract all bot user agents -# $extract_bot_info = (entry, result) -> { -# split(entry["user_agent"], result, " ") -# return length(result) -# }; -# bot_info_count = flatMap("extract_bot_info", bot_requests, bot_info) -# -# # Use take to show top 3 bot types -# top_bots_count = take(3, bot_info, top_bots) -# -# print "Bot Requests:", bot_count -# print "Bot Types Found:", bot_info_count -# print "Top Bot Types:", top_bots_count -# } -# -# print "" -# print "🎉 Log analysis complete!" -# } -# -# LANGUAGE FEATURES: -# + +# ----------------------------------------------------------------------------- +# BEGIN: Real World Example +# ----------------------------------------------------------------------------- + +# # This demonstrates most rawk features in a setting familiar to awk +# # Usage: awk -f rawk.awk example.rawk | awk -f - sample.log + +# # User defined predicate functions for log analysis +# # This shows off rawk's single line function syntax +# $is_error = (status) -> status >= 400; +# $is_success = (status) -> status >= 200 && status < 300; +# $is_large_request = (bytes) -> bytes > 1000000; # > 1MB +# $is_api_request = (url) -> index(url, "/api/") > 0; +# $is_bot = (user_agent) -> index(user_agent, "bot") > 0 || index(user_agent, "crawler") > 0; + +# # Data parsing and transformation functions +# # These show off rawk's multi-line arrow functions +# $extract_status = (request_line) -> { +# split(request_line, parts, " ") +# # The status code is the second part, not the third +# return parts[2] +# }; + +# $extract_method = (request_line) -> { +# split(request_line, parts, " ") +# return parts[1] +# }; + +# $extract_url = (request_line) -> { +# split(request_line, parts, " ") +# return parts[2] +# }; + +# # Aggregation and reporting functions +# $format_error_report = (ip, status, url, user_agent) -> { +# return "ERROR: " status " - " ip " accessed " url " (" user_agent ")" +# }; + +# $format_success_report = (ip, method, url, bytes) -> { +# size_label = is_large_request(bytes) ? "LARGE" : "normal" +# return "SUCCESS: " method " " url " (" bytes " bytes, " size_label ")" +# }; + +# # Main processing pipeline +# BEGIN { +# print "Apache Log Analysis Report" +# print "=============================" +# print "" +# } + +# # Process each log line +# { +# # Parse Apache log format: IP - - [timestamp] "method url status" bytes "referer" "user-agent" +# # Note that we use a series of simpler regex matches, rather than trying to do it all at once +# if (match($0, /^([0-9.]+)/)) { +# ip = substr($0, RSTART, RLENGTH) + +# # Extract request (method url protocol) +# if (match($0, /"([^"]+)"/)) { +# request = substr($0, RSTART + 1, RLENGTH - 2) +# # Extract method and URL from request +# method = extract_method(request) +# url = extract_url(request) +# } + +# # Extract status code (number after the request) +# if (match($0, /" ([0-9]+) /)) { +# status = substr($0, RSTART + 1, RLENGTH - 2) +# # Remove leading/trailing spaces +# gsub(/^[ \t]+|[ \t]+$/, "", status) +# } + +# # Extract bytes (number after request) +# if (match($0, /" ([0-9]+) /)) { +# bytes = substr($0, RSTART + 1, RLENGTH - 2) +# } + +# # Extract user agent (last quoted field) +# if (match($0, /"([^"]*)"$/)) { +# user_agent = substr($0, RSTART + 1, RLENGTH - 2) +# } + +# # Store for analysis +# request_count++ + +# # Real-time processing using some standard library predicates +# if (http_is_server_error(status)) { +# server_error_count++ +# error_report = format_error_report(ip, status, url, user_agent) +# print "SERVER ERROR: " error_report +# } else if (http_is_client_error(status)) { +# client_error_count++ +# error_report = format_error_report(ip, status, url, user_agent) +# print "CLIENT ERROR: " error_report +# } else if (is_success(status)) { +# success_count++ +# success_report = format_success_report(ip, method, url, bytes) +# print "✓ " success_report +# } + +# # Track different types of requests +# if (is_api_request(url)) { +# api_count++ +# api_urls[api_count] = url +# } + +# if (url_is_static_file(url)) { +# static_count++ +# static_urls[static_count] = url +# } + +# if (http_is_mutating_method(method)) { +# mutation_count++ +# if (ip_is_public(ip)) { +# print "EXTERNAL MUTATION: " ip " " method " " url +# } +# } + +# # Track user types +# if (is_bot(user_agent)) { +# bot_count++ +# bot_agents[bot_count] = user_agent +# } else if (user_agent_is_mobile(user_agent)) { +# mobile_count++ +# } else if (user_agent_is_desktop(user_agent)) { +# desktop_count++ +# } + +# # Track large requests +# if (is_large_request(bytes)) { +# large_count++ +# large_urls[large_count] = url +# } +# } +# } + +# END { +# print "" +# print "Summary Statistics" +# print "====================" +# print "Total Requests:", request_count +# print "Successful:", success_count +# print "Client Errors:", client_error_count +# print "Server Errors:", server_error_count +# print "Total Errors:", client_error_count + server_error_count +# print "Error Rate:", sprintf("%.2f%%", ((client_error_count + server_error_count) / request_count) * 100) +# print "API Requests:", api_count +# print "Static Files:", static_count +# print "Mutating Requests:", mutation_count +# print "Mobile Users:", mobile_count +# print "Desktop Users:", desktop_count +# print "Bot Requests:", bot_count +# print "Large Requests (>1MB):", large_count + +# # Some functional patterns at play, map, flatMap, and take. +# if (api_count > 0) { +# print "" +# print "API Usage Analysis" +# print "====================" + +# # Use map to extract API endpoints +# $extract_endpoint = (url) -> url; +# endpoint_count = map("extract_endpoint", api_urls, endpoints) +# print "API Endpoints found:", endpoint_count +# } + +# if (bot_count > 0) { +# print "" +# print "Bot Activity Analysis" +# print "========================" + +# # Use flatMap to extract bot user agent components +# $extract_bot_components = (user_agent, result) -> { +# split(user_agent, result, " ") +# return length(result) +# }; +# bot_components_count = flatMap("extract_bot_components", bot_agents, bot_components) +# print "Bot components analyzed:", bot_components_count + +# # Use take to show top 3 bot components +# top_components_count = take(3, bot_components, top_components) +# print "Top bot components:", top_components_count +# } + +# print "" +# print "End analysis" +# } + + +# ----------------------------------------------------------------------------- +# LANGUAGE FEATURES +# ----------------------------------------------------------------------------- + # 1. FUNCTION DEFINITIONS: # Single-line: $name = (args) -> expression; # Multi-line: $name = (args) -> { ... }; @@ -308,7 +234,6 @@ # Functions can call other functions within their bodies # # 3. STANDARD LIBRARY: -# The following functions are automatically available: # # ARRAY UTILITIES: # - keys(array): Returns count of keys in array @@ -359,7 +284,10 @@ # { print process($0) } # END { print "Done." } # -# COMPILATION PROCESS: +# ----------------------------------------------------------------------------- +# ARCHITECTURE AND TECHNICAL MISCELLANY +# ----------------------------------------------------------------------------- + # 1. Parse: Extract rawk function definitions using `->` symbol # 2. Generate: Create internal awk functions with unique names (`__lambda_0`, etc.) # 3. Dispatch: Build dispatch table mapping public names to internal names |