aboutsummaryrefslogtreecommitdiffstats
path: root/awk
diff options
context:
space:
mode:
authorelioat <{ID}+{username}@users.noreply.github.com>2025-07-31 19:42:23 -0400
committerelioat <{ID}+{username}@users.noreply.github.com>2025-07-31 19:42:23 -0400
commita65136dad76699744709cd6e4b01229f7a83ceb8 (patch)
tree6a39595cee2b7588919af01f48882f1d2608dad2 /awk
parent4d15c9481f1cd42770013db712d2d350e4cf92a2 (diff)
downloadtour-a65136dad76699744709cd6e4b01229f7a83ceb8.tar.gz
*
Diffstat (limited to 'awk')
-rw-r--r--awk/rawk/example.rawk60
-rw-r--r--awk/rawk/example_output.awk232
-rw-r--r--awk/rawk/rawk_block_based.awk39
3 files changed, 320 insertions, 11 deletions
diff --git a/awk/rawk/example.rawk b/awk/rawk/example.rawk
index ac8dd9d..950f5e9 100644
--- a/awk/rawk/example.rawk
+++ b/awk/rawk/example.rawk
@@ -5,6 +5,49 @@
print ""
}
+ RAWK {
+ # Helper functions for parsing and analysis
+ $extract_method = (request) -> {
+ split(request, parts, " ")
+ return parts[1]
+ };
+
+ $extract_url = (request) -> {
+ split(request, parts, " ")
+ return parts[2]
+ };
+
+ $format_error_report = (ip, status, url, user_agent) -> {
+ return ip " - " status " - " url " (" user_agent ")"
+ };
+
+ $format_success_report = (ip, method, url, bytes) -> {
+ return ip " - " method " " url " (" bytes " bytes)"
+ };
+
+ $is_success = (status) -> {
+ return status >= 200 && status < 300
+ };
+
+ $is_api_request = (url) -> {
+ return index(url, "/api/") > 0
+ };
+
+ $is_large_request = (bytes) -> {
+ return bytes > 1048576 # 1MB
+ };
+
+ # Functional programming examples
+ $extract_endpoint = (url) -> {
+ return url
+ };
+
+ $extract_bot_components = (user_agent, result) -> {
+ split(user_agent, result, " ")
+ return length(result)
+ };
+ }
+
# Process each log line
{
# Parse Apache log format: IP - - [timestamp] "method url status" bytes "referer" "user-agent"
@@ -108,37 +151,32 @@
print "Desktop Users:", desktop_count
print "Bot Requests:", bot_count
print "Large Requests (>1MB):", large_count
-
+
# Some functional patterns at play, map, flatMap, and take.
if (api_count > 0) {
print ""
print "API Usage Analysis"
print "===================="
-
+
# Use map to extract API endpoints
- $extract_endpoint = (url) -> url;
endpoint_count = map("extract_endpoint", api_urls, endpoints)
print "API Endpoints found:", endpoint_count
}
-
+
if (bot_count > 0) {
print ""
print "Bot Activity Analysis"
print "========================"
-
+
# Use flatMap to extract bot user agent components
- $extract_bot_components = (user_agent, result) -> {
- split(user_agent, result, " ")
- return length(result)
- };
bot_components_count = flatMap("extract_bot_components", bot_agents, bot_components)
print "Bot components analyzed:", bot_components_count
-
+
# Use take to show top 3 bot components
top_components_count = take(3, bot_components, top_components)
print "Top bot components:", top_components_count
}
-
+
print ""
print "End analysis"
} \ No newline at end of file
diff --git a/awk/rawk/example_output.awk b/awk/rawk/example_output.awk
new file mode 100644
index 0000000..d0bff1d
--- /dev/null
+++ b/awk/rawk/example_output.awk
@@ -0,0 +1,232 @@
+# Generated by rawk v2.0.0
+# Source: example.rawk
+
+# --- Standard Library ---
+function is_number(value) { return value == value + 0 }
+function is_string(value) { return !(value == value + 0) }
+
+function get_keys(array, result, i, count) { count = 0; for (i in array) { result[++count] = i }; return count }
+
+function ip_is_local(ip) { if (!is_string(ip)) return 0; return index(ip, "127.0.0.1") > 0 || index(ip, "192.168.") > 0 || index(ip, "10.") > 0 || index(ip, "172.") > 0 }
+function is_bot(user_agent) { if (!is_string(user_agent)) return 0; return index(user_agent, "bot") > 0 || index(user_agent, "crawler") > 0 || index(user_agent, "spider") > 0 || index(user_agent, "Googlebot") > 0 || index(user_agent, "Bingbot") > 0 }
+
+function flatMap(func_name, array, result, i, temp_array, temp_count, j) { count = 0; for (i in array) { temp_count = dispatch_call(func_name, array[i], temp_array); for (j = 1; j <= temp_count; j++) { result[++count] = temp_array[j] } }; return count }
+function user_agent_is_desktop(user_agent) { if (!is_string(user_agent)) return 0; return (index(user_agent, "Windows") > 0 || index(user_agent, "Macintosh") > 0 || (index(user_agent, "Linux") > 0 && index(user_agent, "Android") == 0)) }
+function map(func_name, array, result, i, count) { count = 0; for (i in array) { result[++count] = dispatch_call(func_name, array[i]) }; return count }
+function http_is_server_error(status) { return status >= 500 && status < 600 }
+function http_is_client_error(status) { return status >= 400 && status < 500 }
+function http_is_mutating_method(method) { return method == "POST" || method == "PUT" || method == "DELETE" || method == "PATCH" }
+function url_is_static_file(url) { if (!is_string(url)) return 0; return index(url, ".css") > 0 || index(url, ".js") > 0 || index(url, ".png") > 0 || index(url, ".jpg") > 0 || index(url, ".jpeg") > 0 || index(url, ".gif") > 0 || index(url, ".svg") > 0 || index(url, ".ico") > 0 || index(url, ".woff") > 0 || index(url, ".woff2") > 0 }
+function take(count, array, result, i, taken) { taken = 0; for (i in array) { if (taken < count) { result[++taken] = array[i] } }; return taken }
+function ip_is_public(ip) { return !ip_is_local(ip) }
+function user_agent_is_mobile(user_agent) { if (!is_string(user_agent)) return 0; return index(user_agent, "Mobile") > 0 || index(user_agent, "iPhone") > 0 || index(user_agent, "Android") > 0 || index(user_agent, "iPad") > 0 }
+# Dispatch function for functional programming
+function dispatch_call(func_name, arg1, arg2, arg3, arg4, arg5) {
+ # User-defined functions
+ if (func_name == "double") return double(arg1)
+ if (func_name == "add") return add(arg1, arg2)
+ if (func_name == "is_even") return is_even(arg1)
+ if (func_name == "is_positive") return is_positive(arg1)
+ if (func_name == "is_positive_num") return is_positive_num(arg1)
+ if (func_name == "square") return square(arg1)
+ if (func_name == "split_words") return split_words(arg1, arg2)
+ if (func_name == "extract_endpoint") return extract_endpoint(arg1)
+ if (func_name == "extract_bot_components") return extract_bot_components(arg1, arg2)
+ # Standard library functions
+ if (func_name == "is_positive") return is_positive(arg1)
+ if (func_name == "is_even") return is_even(arg1)
+ if (func_name == "is_odd") return is_odd(arg1)
+ if (func_name == "is_number") return is_number(arg1)
+ if (func_name == "is_string") return is_string(arg1)
+ print "Error: Function '" func_name "' not found" > "/dev/stderr"
+ return
+}
+
+
+# --- User Functions ---
+function extract_method(request) { split(request, parts, " ")
+ return parts[1]
+
+}
+
+function extract_url(request) { split(request, parts, " ")
+ return parts[2]
+
+}
+
+function format_error_report(ip,status,url,user_agent) { return ip " - " status " - " url " (" user_agent ")"
+
+}
+
+function format_success_report(ip,method,url,bytes) { return ip " - " method " " url " (" bytes " bytes)"
+
+}
+
+function is_success(status) { return status >= 200 && status < 300
+
+}
+
+function is_api_request(url) { return index(url, "/api/") > 0
+
+}
+
+function is_large_request(bytes) { return bytes > 1048576 # 1MB
+
+}
+
+function extract_endpoint(url) { return url
+
+}
+
+function extract_bot_components(user_agent,result) { split(user_agent, result, " ")
+ return length(result)
+
+}
+
+# --- Main Script ---
+ # Main processing pipeline
+ BEGIN {
+ print "Apache Log Analysis Report"
+ print "============================="
+ print ""
+ }
+
+
+ # Process each log line
+ {
+ # Parse Apache log format: IP - - [timestamp] "method url status" bytes "referer" "user-agent"
+ # Note that we use a series of simpler regex matches, rather than trying to do it all at once
+ if (match($0, /^([0-9.]+)/)) {
+ ip = substr($0, RSTART, RLENGTH)
+
+ # Extract request (method url protocol)
+ if (match($0, /"([^"]+)"/)) {
+ request = substr($0, RSTART + 1, RLENGTH - 2)
+ # Extract method and URL from request
+ method = extract_method(request)
+ url = extract_url(request)
+ }
+
+ # Extract status code (number after the request)
+ if (match($0, /" ([0-9]+) /)) {
+ status = substr($0, RSTART + 1, RLENGTH - 2)
+ # Remove leading/trailing spaces
+ gsub(/^[ \t]+|[ \t]+$/, "", status)
+ }
+
+ # Extract bytes (number after request)
+ if (match($0, /" ([0-9]+) /)) {
+ bytes = substr($0, RSTART + 1, RLENGTH - 2)
+ }
+
+ # Extract user agent (last quoted field)
+ if (match($0, /"([^"]*)"$/)) {
+ user_agent = substr($0, RSTART + 1, RLENGTH - 2)
+ }
+
+ # Store for analysis
+ request_count++
+
+ # Real-time processing using some standard library predicates
+ if (http_is_server_error(status)) {
+ server_error_count++
+ error_report = format_error_report(ip, status, url, user_agent)
+ print "SERVER ERROR: " error_report
+ } else if (http_is_client_error(status)) {
+ client_error_count++
+ error_report = format_error_report(ip, status, url, user_agent)
+ print "CLIENT ERROR: " error_report
+ } else if (is_success(status)) {
+ success_count++
+ success_report = format_success_report(ip, method, url, bytes)
+ print "✓ " success_report
+ }
+
+ # Track different types of requests
+ if (is_api_request(url)) {
+ api_count++
+ api_urls[api_count] = url
+ }
+
+ if (url_is_static_file(url)) {
+ static_count++
+ static_urls[static_count] = url
+ }
+
+ if (http_is_mutating_method(method)) {
+ mutation_count++
+ if (ip_is_public(ip)) {
+ print "EXTERNAL MUTATION: " ip " " method " " url
+ }
+ }
+
+ # Track user types
+ if (is_bot(user_agent)) {
+ bot_count++
+ bot_agents[bot_count] = user_agent
+ } else if (user_agent_is_mobile(user_agent)) {
+ mobile_count++
+ } else if (user_agent_is_desktop(user_agent)) {
+ desktop_count++
+ }
+
+ # Track large requests
+ if (is_large_request(bytes)) {
+ large_count++
+ large_urls[large_count] = url
+ }
+ }
+ }
+
+ END {
+ print ""
+ print "Summary Statistics"
+ print "===================="
+ print "Total Requests:", request_count
+ print "Successful:", success_count
+ print "Client Errors:", client_error_count
+ print "Server Errors:", server_error_count
+ print "Total Errors:", client_error_count + server_error_count
+ print "Error Rate:", sprintf("%.2f%%", ((client_error_count + server_error_count) / request_count) * 100)
+ print "API Requests:", api_count
+ print "Static Files:", static_count
+ print "Mutating Requests:", mutation_count
+ print "Mobile Users:", mobile_count
+ print "Desktop Users:", desktop_count
+ print "Bot Requests:", bot_count
+ print "Large Requests (>1MB):", large_count
+
+ # Some functional patterns at play, map, flatMap, and take.
+ if (api_count > 0) {
+ print ""
+ print "API Usage Analysis"
+ print "===================="
+
+ # Use map to extract API endpoints
+ endpoint_count = map("extract_endpoint", api_urls, endpoints)
+ print "API Endpoints found:", endpoint_count
+ }
+
+ if (bot_count > 0) {
+ print ""
+ print "Bot Activity Analysis"
+ print "========================"
+
+ # Use flatMap to extract bot user agent components
+ bot_components_count = flatMap("extract_bot_components", bot_agents, bot_components)
+ print "Bot components analyzed:", bot_components_count
+
+ # Use take to show top 3 bot components
+ top_components_count = take(3, bot_components, top_components)
+ print "Top bot components:", top_components_count
+ }
+
+ print ""
+ print "End analysis"
+ }
+
+# Rawk compilation summary:
+# - Rawk Version: 2.0.0
+# - Functions defined: 9
+# - Source lines: 182
+# - Standard library functions included: 11
diff --git a/awk/rawk/rawk_block_based.awk b/awk/rawk/rawk_block_based.awk
index b21a4e1..f746723 100644
--- a/awk/rawk/rawk_block_based.awk
+++ b/awk/rawk/rawk_block_based.awk
@@ -75,6 +75,17 @@ BEGIN {
stdlib_functions["is_uppercase"] = 1
stdlib_functions["is_lowercase"] = 1
stdlib_functions["is_length"] = 1
+ stdlib_functions["url_is_static_file"] = 1
+ stdlib_functions["url_has_query_params"] = 1
+ stdlib_functions["url_is_root_path"] = 1
+ stdlib_functions["user_agent_is_mobile"] = 1
+ stdlib_functions["user_agent_is_desktop"] = 1
+ stdlib_functions["user_agent_is_browser"] = 1
+ stdlib_functions["is_bot"] = 1
+ stdlib_functions["ip_is_local"] = 1
+ stdlib_functions["ip_is_public"] = 1
+ stdlib_functions["ip_is_ipv4"] = 1
+ stdlib_functions["ip_is_ipv6"] = 1
}
# Pass 1: Collect all lines
@@ -104,6 +115,7 @@ END {
if (char == "}") brace_count--
if (brace_count == 0) {
rawk_block_end = j
+ in_rawk_block = 0
break
}
}
@@ -224,6 +236,11 @@ END {
print "function get_keys(array, result, i, count) { count = 0; for (i in array) { result[++count] = i }; return count }"
print ""
+ # Always include dependency functions
+ print "function ip_is_local(ip) { if (!is_string(ip)) return 0; return index(ip, \"127.0.0.1\") > 0 || index(ip, \"192.168.\") > 0 || index(ip, \"10.\") > 0 || index(ip, \"172.\") > 0 }"
+ print "function is_bot(user_agent) { if (!is_string(user_agent)) return 0; return index(user_agent, \"bot\") > 0 || index(user_agent, \"crawler\") > 0 || index(user_agent, \"spider\") > 0 || index(user_agent, \"Googlebot\") > 0 || index(user_agent, \"Bingbot\") > 0 }"
+ print ""
+
# Include used standard library functions
for (func_name in USED_STDLIB_FUNCTIONS) {
if (func_name == "assert") {
@@ -332,6 +349,25 @@ END {
print "function is_lowercase(value) { return value ~ /^[a-z]+$/ }"
} else if (func_name == "is_length") {
print "function is_length(value, target_length) { return length(value) == target_length }"
+ } else if (func_name == "url_is_static_file") {
+ print "function url_is_static_file(url) { if (!is_string(url)) return 0; return index(url, \".css\") > 0 || index(url, \".js\") > 0 || index(url, \".png\") > 0 || index(url, \".jpg\") > 0 || index(url, \".jpeg\") > 0 || index(url, \".gif\") > 0 || index(url, \".svg\") > 0 || index(url, \".ico\") > 0 || index(url, \".woff\") > 0 || index(url, \".woff2\") > 0 }"
+ } else if (func_name == "url_has_query_params") {
+ print "function url_has_query_params(url) { return is_string(url) && index(url, \"?\") > 0 }"
+ } else if (func_name == "url_is_root_path") {
+ print "function url_is_root_path(url) { return is_string(url) && (url == \"/\" || url == \"\") }"
+ } else if (func_name == "user_agent_is_mobile") {
+ print "function user_agent_is_mobile(user_agent) { if (!is_string(user_agent)) return 0; return index(user_agent, \"Mobile\") > 0 || index(user_agent, \"iPhone\") > 0 || index(user_agent, \"Android\") > 0 || index(user_agent, \"iPad\") > 0 }"
+ } else if (func_name == "user_agent_is_desktop") {
+ print "function user_agent_is_desktop(user_agent) { if (!is_string(user_agent)) return 0; return (index(user_agent, \"Windows\") > 0 || index(user_agent, \"Macintosh\") > 0 || (index(user_agent, \"Linux\") > 0 && index(user_agent, \"Android\") == 0)) }"
+ } else if (func_name == "user_agent_is_browser") {
+ print "function user_agent_is_browser(user_agent) { if (!is_string(user_agent)) return 0; return index(user_agent, \"Mozilla\") > 0 && !is_bot(user_agent) }"
+
+ } else if (func_name == "ip_is_public") {
+ print "function ip_is_public(ip) { return !ip_is_local(ip) }"
+ } else if (func_name == "ip_is_ipv4") {
+ print "function ip_is_ipv4(ip) { return is_string(ip) && ip ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ }"
+ } else if (func_name == "ip_is_ipv6") {
+ print "function ip_is_ipv6(ip) { return is_string(ip) && ip ~ /^[0-9a-fA-F:]+$/ }"
}
}
@@ -339,6 +375,7 @@ END {
if ("map" in USED_STDLIB_FUNCTIONS || "reduce" in USED_STDLIB_FUNCTIONS || "filter" in USED_STDLIB_FUNCTIONS || "find" in USED_STDLIB_FUNCTIONS || "findIndex" in USED_STDLIB_FUNCTIONS || "flatMap" in USED_STDLIB_FUNCTIONS || "pipe" in USED_STDLIB_FUNCTIONS || "pipe_multi" in USED_STDLIB_FUNCTIONS) {
print "# Dispatch function for functional programming"
print "function dispatch_call(func_name, arg1, arg2, arg3, arg4, arg5) {"
+ print " # User-defined functions"
print " if (func_name == \"double\") return double(arg1)"
print " if (func_name == \"add\") return add(arg1, arg2)"
print " if (func_name == \"is_even\") return is_even(arg1)"
@@ -346,6 +383,8 @@ END {
print " if (func_name == \"is_positive_num\") return is_positive_num(arg1)"
print " if (func_name == \"square\") return square(arg1)"
print " if (func_name == \"split_words\") return split_words(arg1, arg2)"
+ print " if (func_name == \"extract_endpoint\") return extract_endpoint(arg1)"
+ print " if (func_name == \"extract_bot_components\") return extract_bot_components(arg1, arg2)"
print " # Standard library functions"
print " if (func_name == \"is_positive\") return is_positive(arg1)"
print " if (func_name == \"is_even\") return is_even(arg1)"