diff options
Diffstat (limited to 'js/scripting-lang/docs/baba-yaga/0.0.1/lexer.js.html')
| -rw-r--r-- | js/scripting-lang/docs/baba-yaga/0.0.1/lexer.js.html | 591 |
1 files changed, 591 insertions, 0 deletions
diff --git a/js/scripting-lang/docs/baba-yaga/0.0.1/lexer.js.html b/js/scripting-lang/docs/baba-yaga/0.0.1/lexer.js.html new file mode 100644 index 0000000..1ebd7a1 --- /dev/null +++ b/js/scripting-lang/docs/baba-yaga/0.0.1/lexer.js.html @@ -0,0 +1,591 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width,initial-scale=1"> + <title>lexer.js - Documentation</title> + + <script src="scripts/prettify/prettify.js"></script> + <script src="scripts/prettify/lang-css.js"></script> + <!--[if lt IE 9]> + <script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script> + <![endif]--> + <link type="text/css" rel="stylesheet" href="https://code.ionicframework.com/ionicons/2.0.1/css/ionicons.min.css"> + <link type="text/css" rel="stylesheet" href="styles/prettify-tomorrow.css"> + <link type="text/css" rel="stylesheet" href="styles/jsdoc-default.css"> +</head> +<body> + +<input type="checkbox" id="nav-trigger" class="nav-trigger" /> +<label for="nav-trigger" class="navicon-button x"> + <div class="navicon"></div> +</label> + +<label for="nav-trigger" class="overlay"></label> + +<nav> + <li class="nav-link nav-home-link"><a href="index.html">Home</a></li><li class="nav-heading">Tutorials</li><li class="nav-item"><a href="tutorial-00_Introduction.html">00_Introduction</a></li><li class="nav-item"><a href="tutorial-01_Function_Calls.html">01_Function_Calls</a></li><li class="nav-item"><a href="tutorial-02_Function_Composition.html">02_Function_Composition</a></li><li class="nav-item"><a href="tutorial-03_Table_Operations.html">03_Table_Operations</a></li><li class="nav-item"><a href="tutorial-04_Currying.html">04_Currying</a></li><li class="nav-item"><a href="tutorial-05_Pattern_Matching.html">05_Pattern_Matching</a></li><li class="nav-item"><a href="tutorial-06_Immutable_Tables.html">06_Immutable_Tables</a></li><li class="nav-item"><a href="tutorial-07_Function_References.html">07_Function_References</a></li><li class="nav-item"><a href="tutorial-08_Combinators.html">08_Combinators</a></li><li class="nav-item"><a href="tutorial-09_Expression_Based.html">09_Expression_Based</a></li><li class="nav-item"><a href="tutorial-10_Tables_Deep_Dive.html">10_Tables_Deep_Dive</a></li><li class="nav-item"><a href="tutorial-11_Standard_Library.html">11_Standard_Library</a></li><li class="nav-item"><a href="tutorial-12_IO_Operations.html">12_IO_Operations</a></li><li class="nav-item"><a href="tutorial-13_Error_Handling.html">13_Error_Handling</a></li><li class="nav-item"><a href="tutorial-14_Advanced_Combinators.html">14_Advanced_Combinators</a></li><li class="nav-item"><a href="tutorial-15_Integration_Patterns.html">15_Integration_Patterns</a></li><li class="nav-item"><a href="tutorial-16_Best_Practices.html">16_Best_Practices</a></li><li class="nav-item"><a href="tutorial-README.html">README</a></li><li class="nav-heading"><a href="global.html">Globals</a></li><li class="nav-item"><span class="nav-item-type type-member">M</span><span class="nav-item-name"><a href="global.html#callStackTracker">callStackTracker</a></span></li><li class="nav-item"><span class="nav-item-type type-function">F</span><span class="nav-item-name"><a href="global.html#debugError">debugError</a></span></li><li class="nav-item"><span class="nav-item-type type-function">F</span><span class="nav-item-name"><a href="global.html#debugLog">debugLog</a></span></li><li class="nav-item"><span class="nav-item-type type-function">F</span><span class="nav-item-name"><a href="global.html#executeFile">executeFile</a></span></li><li class="nav-item"><span class="nav-item-type type-function">F</span><span class="nav-item-name"><a href="global.html#initializeStandardLibrary">initializeStandardLibrary</a></span></li><li class="nav-item"><span class="nav-item-type type-function">F</span><span class="nav-item-name"><a href="global.html#interpreter">interpreter</a></span></li><li class="nav-item"><span class="nav-item-type type-function">F</span><span class="nav-item-name"><a href="global.html#lexer">lexer</a></span></li><li class="nav-item"><span class="nav-item-type type-function">F</span><span class="nav-item-name"><a href="global.html#main">main</a></span></li><li class="nav-item"><span class="nav-item-type type-function">F</span><span class="nav-item-name"><a href="global.html#parser">parser</a></span></li><li class="nav-item"><span class="nav-item-type type-function">F</span><span class="nav-item-name"><a href="global.html#readFile">readFile</a></span></li><li class="nav-item"><span class="nav-item-type type-function">F</span><span class="nav-item-name"><a href="global.html#run">run</a></span></li> +</nav> + +<div id="main"> + + <h1 class="page-title">lexer.js</h1> + + + + + + + + <section> + <article> + <pre class="prettyprint source linenums"><code>// Lexer for the scripting language +// Supports both Node.js and browser environments + +/** + * Token types for the language + * + * @description Defines all token types used by the lexer and parser. + * Each token type represents a distinct syntactic element in the language. + * + * The token types are organized into categories: + * - Literals: NUMBER, STRING, TRUE, FALSE + * - Operators: PLUS, MINUS, MULTIPLY, DIVIDE, MODULO, POWER, etc. + * - Keywords: WHEN, IS, THEN, FUNCTION, etc. + * - Punctuation: LEFT_PAREN, RIGHT_PAREN, SEMICOLON, COMMA, etc. + * - Special: IO_IN, IO_OUT, IO_ASSERT, IO_LISTEN, IO_EMIT, FUNCTION_REF, FUNCTION_ARG + * + * This enumeration provides a centralized definition of all possible + * token types, ensuring consistency between lexer and parser. The token + * types are designed to support the combinator-based architecture where + * all operations are translated to function calls. + * + * @typedef {Object} TokenType + * @property {string} NUMBER - Numeric literals (integers and floats) + * @property {string} PLUS - Addition operator (+) + * @property {string} MINUS - Subtraction operator (-) + * @property {string} MULTIPLY - Multiplication operator (*) + * @property {string} DIVIDE - Division operator (/) + * @property {string} IDENTIFIER - Variable names and function names + * @property {string} ASSIGNMENT - Assignment operator (:) + * @property {string} ARROW - Function arrow (->) + * @property {string} CASE - Case keyword + * @property {string} OF - Of keyword + * @property {string} WHEN - When keyword for pattern matching + * @property {string} IS - Is keyword for pattern matching + * @property {string} THEN - Then keyword for pattern matching + * @property {string} WILDCARD - Wildcard pattern (_) + * @property {string} FUNCTION - Function keyword + * @property {string} LEFT_PAREN - Left parenthesis (() + * @property {string} RIGHT_PAREN - Right parenthesis ()) + * @property {string} LEFT_BRACE - Left brace ({) + * @property {string} RIGHT_BRACE - Right brace (}) + * @property {string} LEFT_BRACKET - Left bracket ([) + * @property {string} RIGHT_BRACKET - Right bracket (]) + * @property {string} SEMICOLON - Semicolon (;) + * @property {string} COMMA - Comma (,) + * @property {string} DOT - Dot (.) + * @property {string} STRING - String literals + * @property {string} TRUE - Boolean true literal + * @property {string} FALSE - Boolean false literal + * @property {string} AND - Logical AND operator + * @property {string} OR - Logical OR operator + * @property {string} XOR - Logical XOR operator + * @property {string} NOT - Logical NOT operator + * @property {string} EQUALS - Equality operator (==) + * @property {string} LESS_THAN - Less than operator (<) + * @property {string} GREATER_THAN - Greater than operator (>) + * @property {string} LESS_EQUAL - Less than or equal operator (<=) + * @property {string} GREATER_EQUAL - Greater than or equal operator (>=) + * @property {string} NOT_EQUAL - Not equal operator (!=) + * @property {string} MODULO - Modulo operator (%) + * @property {string} POWER - Power operator (^) + * @property {string} IO_IN - Input operation (..in) + * @property {string} IO_OUT - Output operation (..out) + * @property {string} IO_ASSERT - Assertion operation (..assert) + * @property {string} IO_LISTEN - Listen operation (..listen) + * @property {string} IO_EMIT - Emit operation (..emit) + * @property {string} FUNCTION_REF - Function reference (@function) + * @property {string} FUNCTION_ARG - Function argument (@(expression)) + * @property {string} COMPOSE - Function composition (via) + */ +export const TokenType = { + NUMBER: 'NUMBER', + PLUS: 'PLUS', + MINUS: 'MINUS', + UNARY_MINUS: 'UNARY_MINUS', + BINARY_MINUS: 'BINARY_MINUS', + MULTIPLY: 'MULTIPLY', + DIVIDE: 'DIVIDE', + IDENTIFIER: 'IDENTIFIER', + ASSIGNMENT: 'ASSIGNMENT', + ARROW: 'ARROW', + CASE: 'CASE', + OF: 'OF', + WHEN: 'WHEN', + IS: 'IS', + THEN: 'THEN', + WILDCARD: 'WILDCARD', + FUNCTION: 'FUNCTION', + LEFT_PAREN: 'LEFT_PAREN', + RIGHT_PAREN: 'RIGHT_PAREN', + LEFT_BRACE: 'LEFT_BRACE', + RIGHT_BRACE: 'RIGHT_BRACE', + LEFT_BRACKET: 'LEFT_BRACKET', + RIGHT_BRACKET: 'RIGHT_BRACKET', + SEMICOLON: 'SEMICOLON', + COMMA: 'COMMA', + DOT: 'DOT', + STRING: 'STRING', + TRUE: 'TRUE', + FALSE: 'FALSE', + AND: 'AND', + OR: 'OR', + XOR: 'XOR', + NOT: 'NOT', + EQUALS: 'EQUALS', + LESS_THAN: 'LESS_THAN', + GREATER_THAN: 'GREATER_THAN', + LESS_EQUAL: 'LESS_EQUAL', + GREATER_EQUAL: 'GREATER_EQUAL', + NOT_EQUAL: 'NOT_EQUAL', + MODULO: 'MODULO', + POWER: 'POWER', + IO_IN: 'IO_IN', + IO_OUT: 'IO_OUT', + IO_ASSERT: 'IO_ASSERT', + IO_LISTEN: 'IO_LISTEN', + IO_EMIT: 'IO_EMIT', + FUNCTION_REF: 'FUNCTION_REF', + FUNCTION_ARG: 'FUNCTION_ARG', + COMPOSE: 'COMPOSE' +}; + +/** + * Token object structure + * + * @typedef {Object} Token + * @property {string} type - The token type from TokenType enum + * @property {*} [value] - The token's value (for literals and identifiers) + * @property {string} [name] - Function name (for FUNCTION_REF tokens) + * @property {number} line - Line number where token appears (1-indexed) + * @property {number} column - Column number where token appears (1-indexed) + */ + +/** + * Converts source code into tokens for the combinator-based language + * + * @param {string} input - The source code to tokenize + * @returns {Array.<Token>} Array of token objects with type, value, line, and column + * @throws {Error} For unexpected characters or malformed tokens + * + * @description The lexer performs lexical analysis by converting source code + * into a stream of tokens. Each token represents a meaningful unit of the + * language syntax, such as identifiers, literals, operators, and keywords. + * + * The lexer implements a character-by-character scanning approach with + * lookahead for multi-character tokens. It maintains line and column + * information for accurate error reporting and debugging. + * + * Key features: + * - Handles whitespace and comments (single-line and multi-line) + * - Recognizes all language constructs including operators, keywords, and literals + * - Supports string literals with escape sequences + * - Provides detailed position information for error reporting + * - Cross-platform compatibility (Node.js, Bun, browser) + * - Supports function composition with 'via' keyword + * - Handles function references with '@' operator + * + * The lexer is designed to be robust and provide clear error messages + * for malformed input, making it easier to debug syntax errors in user code. + * It supports the combinator-based architecture by recognizing all operators + * and special tokens needed for function composition and application. + * + * The lexer is the first step in the language processing pipeline and must + * correctly identify all tokens that the parser will translate into function + * calls. This includes operators that will become combinator function calls, + * function references that enable higher-order programming, and special + * keywords that support the functional programming paradigm. + * + * The lexer uses a state machine approach where each character type triggers + * different parsing strategies. This design enables efficient tokenization + * while maintaining clear separation of concerns for different token types. + * The character-by-character approach allows for precise error reporting and + * supports multi-character tokens like operators and string literals + * with escape sequences. + * + * Error handling is designed to provide meaningful feedback by including + * line and column information in error messages. This enables users to + * quickly locate and fix syntax errors in their code. + */ +export function lexer(input) { + const tokens = []; + let current = 0; + let line = 1; + let column = 1; + + // Helper functions for spacing detection + function hasLeadingWhitespace() { + let pos = current - 1; + while (pos >= 0 && /\s/.test(input[pos])) pos--; + return pos >= 0 && input[pos] !== '\n' && input[pos] !== ';'; + } + + function hasLeadingAndTrailingSpaces() { + const hasLeading = current > 0 && /\s/.test(input[current - 1]); + const hasTrailing = current + 1 < input.length && /\s/.test(input[current + 1]); + return hasLeading && hasTrailing; + } + + while (current < input.length) { + let char = input[current]; + + // Skip whitespace + if (/\s/.test(char)) { + if (char === '\n') { + line++; + column = 1; + } else { + column++; + } + current++; + continue; + } + + // Skip comments (single line and multi-line) + if (char === '/' && input[current + 1] === '/') { + while (current < input.length && input[current] !== '\n') { + current++; + column++; + } + continue; + } + + // Skip multi-line comments /* ... */ + if (char === '/' && input[current + 1] === '*') { + current += 2; // Skip /* + column += 2; + while (current < input.length - 1 && !(input[current] === '*' && input[current + 1] === '/')) { + if (input[current] === '\n') { + line++; + column = 1; + } else { + column++; + } + current++; + } + if (current < input.length - 1) { + current += 2; // Skip */ + column += 2; + } + continue; + } + + // IO operations (..in, ..out, ..assert) + if (char === '.' && input[current + 1] === '.') { + current += 2; // Skip both dots + column += 2; + + // Read the IO operation name + let operation = ''; + while (current < input.length && /[a-zA-Z]/.test(input[current])) { + operation += input[current]; + current++; + column++; + } + + // Determine the IO operation type + switch (operation) { + case 'in': + tokens.push({ type: TokenType.IO_IN, line, column: column - operation.length - 2 }); + break; + case 'out': + tokens.push({ type: TokenType.IO_OUT, line, column: column - operation.length - 2 }); + break; + case 'assert': + tokens.push({ type: TokenType.IO_ASSERT, line, column: column - operation.length - 2 }); + break; + case 'listen': + tokens.push({ type: TokenType.IO_LISTEN, line, column: column - operation.length - 2 }); + break; + case 'emit': + tokens.push({ type: TokenType.IO_EMIT, line, column: column - operation.length - 2 }); + break; + default: + throw new Error(`Unknown IO operation: ..${operation} at line ${line}, column ${column - operation.length - 2}`); + } + continue; + } + + // Function references (@function) and function arguments (@(expression)) + if (char === '@') { + current++; // Skip '@' + column++; + + // Check if this is @(expression) for function arguments + if (current < input.length && input[current] === '(') { + // This is @(expression) - mark as function argument + tokens.push({ type: TokenType.FUNCTION_ARG, line, column: column - 1 }); + continue; + } + + // Read the function name + let functionName = ''; + while (current < input.length && /[a-zA-Z0-9_]/.test(input[current])) { + functionName += input[current]; + current++; + column++; + } + + if (functionName === '') { + throw new Error(`Invalid function reference at line ${line}, column ${column - 1}`); + } + + tokens.push({ type: TokenType.FUNCTION_REF, name: functionName, line, column: column - functionName.length - 1 }); + continue; + } + + // Numbers + if (/[0-9]/.test(char)) { + let value = ''; + while (current < input.length && /[0-9.]/.test(input[current])) { + value += input[current]; + current++; + column++; + } + tokens.push({ type: TokenType.NUMBER, value: parseFloat(value), line, column: column - value.length }); + continue; + } + + // Identifiers and keywords + if (/[a-zA-Z_]/.test(char)) { + let value = ''; + const startColumn = column; + while (current < input.length && /[a-zA-Z0-9_]/.test(input[current])) { + value += input[current]; + current++; + column++; + } + + // Check for keywords + switch (value) { + case 'true': + tokens.push({ type: TokenType.TRUE, value: true, line, column: startColumn }); + break; + case 'false': + tokens.push({ type: TokenType.FALSE, value: false, line, column: startColumn }); + break; + case 'and': + tokens.push({ type: TokenType.AND, line, column: startColumn }); + break; + case 'or': + tokens.push({ type: TokenType.OR, line, column: startColumn }); + break; + case 'xor': + tokens.push({ type: TokenType.XOR, line, column: startColumn }); + break; + case 'not': + tokens.push({ type: TokenType.NOT, line, column: startColumn }); + break; + case 'case': + tokens.push({ type: TokenType.CASE, line, column: startColumn }); + break; + case 'of': + tokens.push({ type: TokenType.OF, line, column: startColumn }); + break; + case 'when': + tokens.push({ type: TokenType.WHEN, line, column: startColumn }); + break; + case 'is': + tokens.push({ type: TokenType.IS, line, column: startColumn }); + break; + case 'then': + tokens.push({ type: TokenType.THEN, line, column: startColumn }); + break; + case 'function': + tokens.push({ type: TokenType.FUNCTION, line, column: startColumn }); + break; + case 'via': // Function composition operator: f via g = compose(f, g) + tokens.push({ type: TokenType.COMPOSE, line, column: startColumn }); + break; + case '_': + tokens.push({ type: TokenType.WILDCARD, line, column: startColumn }); + break; + default: + tokens.push({ type: TokenType.IDENTIFIER, value, line, column: startColumn }); + } + continue; + } + + // Strings + if (char === '"') { + let value = ''; + current++; + column++; + while (current < input.length && input[current] !== '"') { + if (input[current] === '\\') { + current++; + column++; + if (current < input.length) { + switch (input[current]) { + case 'n': value += '\n'; break; + case 't': value += '\t'; break; + case 'r': value += '\r'; break; + case '\\': value += '\\'; break; + case '"': value += '"'; break; + default: value += input[current]; + } + } + } else { + value += input[current]; + } + current++; + column++; + } + if (current < input.length) { + current++; + column++; + } + tokens.push({ type: TokenType.STRING, value, line, column: column - value.length - 2 }); + continue; + } + + // Operators and punctuation + switch (char) { + case '+': + tokens.push({ type: TokenType.PLUS, line, column }); + break; + case '-': + if (input[current + 1] === '>') { + tokens.push({ type: TokenType.ARROW, line, column }); + current++; + column++; + } else { + // Check spacing to determine token type + const isUnary = !hasLeadingWhitespace(); + const isBinary = hasLeadingAndTrailingSpaces(); + const isFollowedByNumber = current + 1 < input.length && /[0-9]/.test(input[current + 1]); + + if (isUnary && isFollowedByNumber) { + // Unary minus at start of expression: -5 + tokens.push({ type: TokenType.UNARY_MINUS, line, column }); + } else if (isBinary) { + // Binary minus with spaces: 5 - 3 + tokens.push({ type: TokenType.BINARY_MINUS, line, column }); + } else if (isFollowedByNumber) { + // Minus followed by number but not at start: 5-3 (legacy) + tokens.push({ type: TokenType.MINUS, line, column }); + } else { + // Fallback to legacy MINUS token for edge cases + tokens.push({ type: TokenType.MINUS, line, column }); + } + } + break; + case '*': + tokens.push({ type: TokenType.MULTIPLY, line, column }); + break; + case '/': + tokens.push({ type: TokenType.DIVIDE, line, column }); + break; + case '%': + tokens.push({ type: TokenType.MODULO, line, column }); + break; + case '^': + tokens.push({ type: TokenType.POWER, line, column }); + break; + case '(': + tokens.push({ type: TokenType.LEFT_PAREN, line, column }); + break; + case ')': + tokens.push({ type: TokenType.RIGHT_PAREN, line, column }); + break; + case '{': + tokens.push({ type: TokenType.LEFT_BRACE, line, column }); + break; + case '}': + tokens.push({ type: TokenType.RIGHT_BRACE, line, column }); + break; + case '[': + tokens.push({ type: TokenType.LEFT_BRACKET, line, column }); + break; + case ']': + tokens.push({ type: TokenType.RIGHT_BRACKET, line, column }); + break; + case ';': + tokens.push({ type: TokenType.SEMICOLON, line, column }); + break; + case ',': + tokens.push({ type: TokenType.COMMA, line, column }); + break; + case '.': + tokens.push({ type: TokenType.DOT, line, column }); + break; + case ':': + tokens.push({ type: TokenType.ASSIGNMENT, line, column }); + break; + + case '=': + if (input[current + 1] === '=') { + tokens.push({ type: TokenType.EQUALS, line, column }); + current++; + column++; + } else { + // Single = is used for equality comparison in assertions + tokens.push({ type: TokenType.EQUALS, line, column }); + } + break; + case '<': + if (input[current + 1] === '=') { + tokens.push({ type: TokenType.LESS_EQUAL, line, column }); + current++; + column++; + } else { + tokens.push({ type: TokenType.LESS_THAN, line, column }); + } + break; + case '>': + if (input[current + 1] === '=') { + tokens.push({ type: TokenType.GREATER_EQUAL, line, column }); + current++; + column++; + } else { + tokens.push({ type: TokenType.GREATER_THAN, line, column }); + } + break; + case '!': + if (input[current + 1] === '=') { + tokens.push({ type: TokenType.NOT_EQUAL, line, column }); + current++; + column++; + } else { + throw new Error(`Unexpected character: ${char} at line ${line}, column ${column}`); + } + break; + default: + throw new Error(`Unexpected character: ${char} at line ${line}, column ${column}`); + } + + current++; + column++; + } + + return tokens; +} </code></pre> + </article> + </section> + + + + +</div> + +<br class="clear"> + +<footer> + Generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.4</a> on Tue Jul 29 2025 23:15:00 GMT-0400 (Eastern Daylight Time) using the Minami theme. +</footer> + +<script>prettyPrint();</script> +<script src="scripts/linenumber.js"></script> +</body> +</html> |