aboutsummaryrefslogtreecommitdiffstats
path: root/awk
diff options
context:
space:
mode:
authorelioat <{ID}+{username}@users.noreply.github.com>2025-07-31 10:57:06 -0400
committerelioat <{ID}+{username}@users.noreply.github.com>2025-07-31 10:57:06 -0400
commit0a45bdc13d1de92c04979c4a6be27db2fdcf8c85 (patch)
tree232f24aadca656023fe629ab48588ec88a103afc /awk
parent7ec899534ff14bc71bbe39a5f5e252de4a7b6694 (diff)
downloadtour-0a45bdc13d1de92c04979c4a6be27db2fdcf8c85.tar.gz
*
Diffstat (limited to 'awk')
-rw-r--r--awk/rawk/example.rawk13
-rw-r--r--awk/rawk/rawk.awk468
2 files changed, 205 insertions, 276 deletions
diff --git a/awk/rawk/example.rawk b/awk/rawk/example.rawk
index e5bce3c..bda56b7 100644
--- a/awk/rawk/example.rawk
+++ b/awk/rawk/example.rawk
@@ -1,8 +1,8 @@
-# Apache Log Processing Example
-# This demonstrates most rawk features with real-world data processing
+# This demonstrates most rawk features in a setting familiar to awk
# Usage: awk -f rawk.awk example.rawk | awk -f - sample.log
-# Predicate functions for log analysis
+# User defined predicate functions for log analysis
+# This shows off rawk's single line function syntax
$is_error = (status) -> status >= 400;
$is_success = (status) -> status >= 200 && status < 300;
$is_large_request = (bytes) -> bytes > 1000000; # > 1MB
@@ -10,6 +10,7 @@ $is_api_request = (url) -> index(url, "/api/") > 0;
$is_bot = (user_agent) -> index(user_agent, "bot") > 0 || index(user_agent, "crawler") > 0;
# Data parsing and transformation functions
+# These show off rawk's multi-line arrow functions
$extract_status = (request_line) -> {
split(request_line, parts, " ")
# The status code is the second part, not the third
@@ -46,7 +47,7 @@ BEGIN {
# Process each log line
{
# Parse Apache log format: IP - - [timestamp] "method url status" bytes "referer" "user-agent"
- # Use a simpler approach with multiple regex matches
+ # Note that we use a series of simpler regex matches, rather than trying to do it all at once
if (match($0, /^([0-9.]+)/)) {
ip = substr($0, RSTART, RLENGTH)
@@ -78,7 +79,7 @@ BEGIN {
# Store for analysis
request_count++
- # Real-time processing with standard library predicates
+ # Real-time processing using some standard library predicates
if (http_is_server_error(status)) {
server_error_count++
error_report = format_error_report(ip, status, url, user_agent)
@@ -147,7 +148,7 @@ END {
print "Bot Requests:", bot_count
print "Large Requests (>1MB):", large_count
- # Demonstrate functional programming features
+ # Some functional patterns at play, map, flatMap, and take.
if (api_count > 0) {
print ""
print "API Usage Analysis"
diff --git a/awk/rawk/rawk.awk b/awk/rawk/rawk.awk
index 034dba6..f1d8dff 100644
--- a/awk/rawk/rawk.awk
+++ b/awk/rawk/rawk.awk
@@ -22,274 +22,200 @@
# # Process input data
# awk -f rawk.awk processor.rawk | awk -f - input.txt
#
-# # Apache Log Processing Example
-# # This demonstrates most rawk features with real-world data processing
-# # Usage: awk -f rawk.awk example.rawk | awk -f - sample.log
-#
-# # Predicate functions for log analysis
-# $is_error = (status) -> status >= 400;
-# $is_success = (status) -> status >= 200 && status < 300;
-# $is_large_request = (bytes) -> bytes > 1000000; # > 1MB
-# $is_api_request = (url) -> index(url, "/api/") > 0;
-# $is_bot = (user_agent) -> index(user_agent, "bot") > 0 || index(user_agent, "crawler") > 0;
-#
-# # Data parsing and transformation functions
-# $extract_status = (request_line) -> {
-# split(request_line, parts, " ")
-# # The status code is the second part, not the third
-# return parts[2]
-# };
-#
-# $extract_method = (request_line) -> {
-# split(request_line, parts, " ")
-# return parts[1]
-# };
-#
-# $extract_url = (request_line) -> {
-# split(request_line, parts, " ")
-# return parts[2]
-# };
-#
-# # Aggregation and reporting functions
-# $format_error_report = (ip, status, url, user_agent) -> {
-# return "ERROR: " status " - " ip " accessed " url " (" user_agent ")"
-# };
-#
-# $format_success_report = (ip, method, url, bytes) -> {
-# size_label = is_large_request(bytes) ? "LARGE" : "normal"
-# return "SUCCESS: " method " " url " (" bytes " bytes, " size_label ")"
-# };
-#
-# # Main processing pipeline
-# BEGIN {
-# print "Apache Log Analysis Report"
-# print "============================="
-# print ""
-# }
-#
-# # Process each log line
-# {
-# # Parse Apache log format: IP - - [timestamp] "method url status" bytes "referer" "user-agent"
-# # Use a simpler approach with multiple regex matches
-# if (match($0, /^([0-9.]+)/)) {
-# ip = substr($0, RSTART, RLENGTH)
-#
-# # Extract request (method url protocol)
-# if (match($0, /"([^"]+)"/)) {
-# request = substr($0, RSTART + 1, RLENGTH - 2)
-# # Extract method and URL from request
-# method = extract_method(request)
-# url = extract_url(request)
-# }
-#
-# # Extract status code (number after the request)
-# if (match($0, /" ([0-9]+) /)) {
-# status = substr($0, RSTART + 1, RLENGTH - 2)
-# # Remove leading/trailing spaces
-# gsub(/^[ \t]+|[ \t]+$/, "", status)
-# }
-#
-# # Extract bytes (number after request)
-# if (match($0, /" ([0-9]+) /)) {
-# bytes = substr($0, RSTART + 1, RLENGTH - 2)
-# }
-#
-# # Extract user agent (last quoted field)
-# if (match($0, /"([^"]*)"$/)) {
-# user_agent = substr($0, RSTART + 1, RLENGTH - 2)
-# }
-#
-# # Store for analysis
-# request_count++
-#
-# # Real-time processing with standard library predicates
-# if (http_is_server_error(status)) {
-# server_error_count++
-# error_report = format_error_report(ip, status, url, user_agent)
-# print "SERVER ERROR: " error_report
-# } else if (http_is_client_error(status)) {
-# client_error_count++
-# error_report = format_error_report(ip, status, url, user_agent)
-# print "CLIENT ERROR: " error_report
-# } else if (is_success(status)) {
-# success_count++
-# success_report = format_success_report(ip, method, url, bytes)
-# print "✓ " success_report
-# }
-#
-# # Track different types of requests
-# if (is_api_request(url)) {
-# api_count++
-# api_urls[api_count] = url
-# }
-#
-# if (url_is_static_file(url)) {
-# static_count++
-# static_urls[static_count] = url
-# }
-#
-# if (http_is_mutating_method(method)) {
-# mutation_count++
-# if (ip_is_public(ip)) {
-# print "EXTERNAL MUTATION: " ip " " method " " url
-# }
-# }
-#
-# # Track user types
-# if (is_bot(user_agent)) {
-# bot_count++
-# bot_agents[bot_count] = user_agent
-# } else if (user_agent_is_mobile(user_agent)) {
-# mobile_count++
-# } else if (user_agent_is_desktop(user_agent)) {
-# desktop_count++
-# }
-#
-# # Track large requests
-# if (is_large_request(bytes)) {
-# large_count++
-# large_urls[large_count] = url
-# }
-# }
-# }
-#
-# END {
-# print ""
-# print "Summary Statistics"
-# print "===================="
-# print "Total Requests:", request_count
-# print "Successful:", success_count
-# print "Client Errors:", client_error_count
-# print "Server Errors:", server_error_count
-# print "Total Errors:", client_error_count + server_error_count
-# print "Error Rate:", sprintf("%.2f%%", ((client_error_count + server_error_count) / request_count) * 100)
-# print "API Requests:", api_count
-# print "Static Files:", static_count
-# print "Mutating Requests:", mutation_count
-# print "Mobile Users:", mobile_count
-# print "Desktop Users:", desktop_count
-# print "Bot Requests:", bot_count
-# print "Large Requests (>1MB):", large_count
-#
-# # Demonstrate functional programming features
-# if (api_count > 0) {
-# print ""
-# print "API Usage Analysis"
-# print "===================="
-#
-# # Use map to extract API endpoints
-# $extract_endpoint = (url) -> url;
-# endpoint_count = map("extract_endpoint", api_urls, endpoints)
-# print "API Endpoints found:", endpoint_count
-# }
-#
-# if (bot_count > 0) {
-# print ""
-# print "Bot Activity Analysis"
-# print "========================"
-#
-# # Use flatMap to extract bot user agent components
-# $extract_bot_components = (user_agent, result) -> {
-# split(user_agent, result, " ")
-# return length(result)
-# };
-# bot_components_count = flatMap("extract_bot_components", bot_agents, bot_components)
-# print "Bot components analyzed:", bot_components_count
-#
-# # Use take to show top 3 bot components
-# top_components_count = take(3, bot_components, top_components)
-# print "Top bot components:", top_components_count
-# }
-#
-# print ""
-# print "End analysis"
-# }
-#
-# # Real-time processing
-# if (is_error(status)) {
-# error_report = format_error_report(log_entry["ip"], status, url, log_entry["user_agent"])
-# print "🚨 " error_report
-# } else if (is_success(status)) {
-# success_report = format_success_report(log_entry["ip"], method, url, bytes)
-# print "✅ " success_report
-# }
-#
-# # Track API usage
-# if (is_api_request(url)) {
-# api_requests[++api_count] = log_entry
-# }
-#
-# # Track bot activity
-# if (is_bot(log_entry["user_agent"])) {
-# bot_requests[++bot_count] = log_entry
-# }
-# }
-# }
-#
-# END {
-# print ""
-# print "📊 Summary Statistics"
-# print "===================="
-#
-# # Calculate overall statistics
-# stats_count = calculate_stats(requests, statistics)
-# print "Total Requests:", statistics["total"]
-# print "Successful:", statistics["success"]
-# print "Errors:", statistics["errors"]
-# print "Error Rate:", sprintf("%.2f%%", statistics["error_rate"])
-# print "Total Data Transferred:", statistics["bytes"], "bytes"
-#
-# # API analysis using functional programming
-# if (api_count > 0) {
-# print ""
-# print "🔌 API Usage Analysis"
-# print "===================="
-#
-# # Use map to extract API endpoints
-# $extract_endpoint = (entry) -> extract_url(entry["request"]);
-# endpoint_count = map("extract_endpoint", api_requests, endpoints)
-#
-# # Use filter to find high-traffic endpoints
-# $is_high_traffic = (endpoint) -> {
-# count = 0
-# for (i in endpoints) {
-# if (endpoints[i] == endpoint) count++
-# }
-# return count > 5
-# };
-# high_traffic_count = filter("is_high_traffic", endpoints, high_traffic_endpoints)
-#
-# print "API Requests:", api_count
-# print "High-Traffic Endpoints:", high_traffic_count
-# }
-#
-# # Bot analysis using enhanced utilities
-# if (bot_count > 0) {
-# print ""
-# print "🤖 Bot Activity Analysis"
-# print "========================"
-#
-# # Use flatMap to extract all bot user agents
-# $extract_bot_info = (entry, result) -> {
-# split(entry["user_agent"], result, " ")
-# return length(result)
-# };
-# bot_info_count = flatMap("extract_bot_info", bot_requests, bot_info)
-#
-# # Use take to show top 3 bot types
-# top_bots_count = take(3, bot_info, top_bots)
-#
-# print "Bot Requests:", bot_count
-# print "Bot Types Found:", bot_info_count
-# print "Top Bot Types:", top_bots_count
-# }
-#
-# print ""
-# print "🎉 Log analysis complete!"
-# }
-#
-# LANGUAGE FEATURES:
-#
+
+# -----------------------------------------------------------------------------
+# BEGIN: Real World Example
+# -----------------------------------------------------------------------------
+
+# # This demonstrates most rawk features in a setting familiar to awk
+# # Usage: awk -f rawk.awk example.rawk | awk -f - sample.log
+
+# # User defined predicate functions for log analysis
+# # This shows off rawk's single line function syntax
+# $is_error = (status) -> status >= 400;
+# $is_success = (status) -> status >= 200 && status < 300;
+# $is_large_request = (bytes) -> bytes > 1000000; # > 1MB
+# $is_api_request = (url) -> index(url, "/api/") > 0;
+# $is_bot = (user_agent) -> index(user_agent, "bot") > 0 || index(user_agent, "crawler") > 0;
+
+# # Data parsing and transformation functions
+# # These show off rawk's multi-line arrow functions
+# $extract_status = (request_line) -> {
+# split(request_line, parts, " ")
+# # The status code is the second part, not the third
+# return parts[2]
+# };
+
+# $extract_method = (request_line) -> {
+# split(request_line, parts, " ")
+# return parts[1]
+# };
+
+# $extract_url = (request_line) -> {
+# split(request_line, parts, " ")
+# return parts[2]
+# };
+
+# # Aggregation and reporting functions
+# $format_error_report = (ip, status, url, user_agent) -> {
+# return "ERROR: " status " - " ip " accessed " url " (" user_agent ")"
+# };
+
+# $format_success_report = (ip, method, url, bytes) -> {
+# size_label = is_large_request(bytes) ? "LARGE" : "normal"
+# return "SUCCESS: " method " " url " (" bytes " bytes, " size_label ")"
+# };
+
+# # Main processing pipeline
+# BEGIN {
+# print "Apache Log Analysis Report"
+# print "============================="
+# print ""
+# }
+
+# # Process each log line
+# {
+# # Parse Apache log format: IP - - [timestamp] "method url status" bytes "referer" "user-agent"
+# # Note that we use a series of simpler regex matches, rather than trying to do it all at once
+# if (match($0, /^([0-9.]+)/)) {
+# ip = substr($0, RSTART, RLENGTH)
+
+# # Extract request (method url protocol)
+# if (match($0, /"([^"]+)"/)) {
+# request = substr($0, RSTART + 1, RLENGTH - 2)
+# # Extract method and URL from request
+# method = extract_method(request)
+# url = extract_url(request)
+# }
+
+# # Extract status code (number after the request)
+# if (match($0, /" ([0-9]+) /)) {
+# status = substr($0, RSTART + 1, RLENGTH - 2)
+# # Remove leading/trailing spaces
+# gsub(/^[ \t]+|[ \t]+$/, "", status)
+# }
+
+# # Extract bytes (number after request)
+# if (match($0, /" ([0-9]+) /)) {
+# bytes = substr($0, RSTART + 1, RLENGTH - 2)
+# }
+
+# # Extract user agent (last quoted field)
+# if (match($0, /"([^"]*)"$/)) {
+# user_agent = substr($0, RSTART + 1, RLENGTH - 2)
+# }
+
+# # Store for analysis
+# request_count++
+
+# # Real-time processing using some standard library predicates
+# if (http_is_server_error(status)) {
+# server_error_count++
+# error_report = format_error_report(ip, status, url, user_agent)
+# print "SERVER ERROR: " error_report
+# } else if (http_is_client_error(status)) {
+# client_error_count++
+# error_report = format_error_report(ip, status, url, user_agent)
+# print "CLIENT ERROR: " error_report
+# } else if (is_success(status)) {
+# success_count++
+# success_report = format_success_report(ip, method, url, bytes)
+# print "✓ " success_report
+# }
+
+# # Track different types of requests
+# if (is_api_request(url)) {
+# api_count++
+# api_urls[api_count] = url
+# }
+
+# if (url_is_static_file(url)) {
+# static_count++
+# static_urls[static_count] = url
+# }
+
+# if (http_is_mutating_method(method)) {
+# mutation_count++
+# if (ip_is_public(ip)) {
+# print "EXTERNAL MUTATION: " ip " " method " " url
+# }
+# }
+
+# # Track user types
+# if (is_bot(user_agent)) {
+# bot_count++
+# bot_agents[bot_count] = user_agent
+# } else if (user_agent_is_mobile(user_agent)) {
+# mobile_count++
+# } else if (user_agent_is_desktop(user_agent)) {
+# desktop_count++
+# }
+
+# # Track large requests
+# if (is_large_request(bytes)) {
+# large_count++
+# large_urls[large_count] = url
+# }
+# }
+# }
+
+# END {
+# print ""
+# print "Summary Statistics"
+# print "===================="
+# print "Total Requests:", request_count
+# print "Successful:", success_count
+# print "Client Errors:", client_error_count
+# print "Server Errors:", server_error_count
+# print "Total Errors:", client_error_count + server_error_count
+# print "Error Rate:", sprintf("%.2f%%", ((client_error_count + server_error_count) / request_count) * 100)
+# print "API Requests:", api_count
+# print "Static Files:", static_count
+# print "Mutating Requests:", mutation_count
+# print "Mobile Users:", mobile_count
+# print "Desktop Users:", desktop_count
+# print "Bot Requests:", bot_count
+# print "Large Requests (>1MB):", large_count
+
+# # Some functional patterns at play, map, flatMap, and take.
+# if (api_count > 0) {
+# print ""
+# print "API Usage Analysis"
+# print "===================="
+
+# # Use map to extract API endpoints
+# $extract_endpoint = (url) -> url;
+# endpoint_count = map("extract_endpoint", api_urls, endpoints)
+# print "API Endpoints found:", endpoint_count
+# }
+
+# if (bot_count > 0) {
+# print ""
+# print "Bot Activity Analysis"
+# print "========================"
+
+# # Use flatMap to extract bot user agent components
+# $extract_bot_components = (user_agent, result) -> {
+# split(user_agent, result, " ")
+# return length(result)
+# };
+# bot_components_count = flatMap("extract_bot_components", bot_agents, bot_components)
+# print "Bot components analyzed:", bot_components_count
+
+# # Use take to show top 3 bot components
+# top_components_count = take(3, bot_components, top_components)
+# print "Top bot components:", top_components_count
+# }
+
+# print ""
+# print "End analysis"
+# }
+
+
+# -----------------------------------------------------------------------------
+# LANGUAGE FEATURES
+# -----------------------------------------------------------------------------
+
# 1. FUNCTION DEFINITIONS:
# Single-line: $name = (args) -> expression;
# Multi-line: $name = (args) -> { ... };
@@ -308,7 +234,6 @@
# Functions can call other functions within their bodies
#
# 3. STANDARD LIBRARY:
-# The following functions are automatically available:
#
# ARRAY UTILITIES:
# - keys(array): Returns count of keys in array
@@ -359,7 +284,10 @@
# { print process($0) }
# END { print "Done." }
#
-# COMPILATION PROCESS:
+# -----------------------------------------------------------------------------
+# ARCHITECTURE AND TECHNICAL MISCELLANY
+# -----------------------------------------------------------------------------
+
# 1. Parse: Extract rawk function definitions using `->` symbol
# 2. Generate: Create internal awk functions with unique names (`__lambda_0`, etc.)
# 3. Dispatch: Build dispatch table mapping public names to internal names