wip

.vscode/extensions.json

@@ -3,8 +3,6 @@
   // for the documentation about the extensions.json format
   "recommendations": [
     "dbaeumer.vscode-eslint",
-    "connor4312.esbuild-problem-matchers",
-    "ms-vscode.extension-test-runner",
     "sumneko.lua"
   ]
 }

.vscode/launch.json

@@ -16,7 +16,7 @@
 				"${workspaceFolder}/dutchies-dcs-scripting-tools/dist/**/*.js",
 				"${workspaceFolder}/compiler/dist/**/*.js"
 			],
-			"preLaunchTask": "${defaultBuildTask}"
+			"preLaunchTask": "compile"
 		}
 	]
 }

.vscode/tasks.json

@@ -4,14 +4,9 @@
 	"version": "2.0.0",
 	"tasks": [
 		{
-            "label": "watch",
+			"label": "compile",
-            "dependsOn": [
+			"type": "npm",
-                "npm: watch:tsc",
+			"script": "compile",
-                "npm: watch:esbuild"
-            ],
-            "presentation": {
-                "reveal": "never"
-            },
 			"group": {
 				"kind": "build",
 				"isDefault": true
@@ -21,53 +16,17 @@
 			}
 		},
 		{
+			"label": "watch",
 			"type": "npm",
-            "script": "watch:esbuild",
+			"script": "watch",
-            "group": "build",
-            "problemMatcher": "$esbuild-watch",
 			"isBackground": true,
-            "label": "npm: watch:esbuild",
 			"presentation": {
-                "group": "watch",
 				"reveal": "never"
 			},
+			"group": "build",
 			"options": {
 				"cwd": "${workspaceFolder}/dutchies-dcs-scripting-tools"
 			}
-        },
-		{
-            "type": "npm",
-            "script": "watch:tsc",
-            "group": "build",
-            "problemMatcher": "$tsc-watch",
-            "isBackground": true,
-            "label": "npm: watch:tsc",
-            "presentation": {
-                "group": "watch",
-                "reveal": "never"
-            },
-            "options": {
-                "cwd": "${workspaceFolder}/dutchies-dcs-scripting-tools"
-            }
-        },
-		{
-			"type": "npm",
-			"script": "watch-tests",
-			"problemMatcher": "$tsc-watch",
-			"isBackground": true,
-			"presentation": {
-				"reveal": "never",
-				"group": "watchers"
-			},
-			"group": "build"
-		},
-		{
-			"label": "tasks: watch-tests",
-			"dependsOn": [
-				"npm: watch",
-				"npm: watch-tests"
-			],
-			"problemMatcher": []
 		}
 	]
 }

compiler/src/CodeBlocks.ts

@@ -0,0 +1,320 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { CompilationError } from './CompilationError';
+
+/*
+    Block types.
+
+    Require is a special case as it's technically a function call, but it's a special use case.
+*/
+export enum BlockType {
+    Line,
+    Function,
+    If,
+    While,
+    For,
+    Do,
+    Return,
+    Require,
+    File
+}
+
+export abstract class CodeBlock {
+    protected childBlocks: CodeBlock[] = [];
+    private parentBlock?: CodeBlock;
+    private blockType: BlockType;
+
+    public readonly sourceLineNumber?: number;
+
+    constructor(sourceLineNumber: number, blockType: BlockType, parentBlock?: CodeBlock){
+        this.sourceLineNumber = sourceLineNumber;
+        this.parentBlock = parentBlock;
+        this.blockType = blockType;
+    }
+
+    public getChildren(): CodeBlock[] {
+        return this.childBlocks;
+    }
+
+    abstract toLines(): string[];
+
+    getParentBlock(): CodeBlock | undefined {
+        return this.parentBlock;
+    }
+
+    static createFromFile(luaFilePath: string, onError?: (error: CompilationError) => void): LuaFile {
+
+        if (!fs.existsSync(luaFilePath)) {
+            throw new Error(`File not found: ${luaFilePath}`);
+        }
+        
+        const fileContent = fs.readFileSync(luaFilePath, 'utf-8');
+
+        let leftCursor = 0;
+        let lineCounter = 0;
+        const file : LuaFile = new LuaFile();
+        let currentBlock : CodeBlock = file;
+        
+        let currentWord = '';
+        let currentBlockString = '';
+        
+        while(leftCursor < fileContent.length){
+            const currentChar : string = fileContent[leftCursor];
+            currentWord += currentChar;
+            currentBlockString += currentChar;
+            const nextChar = leftCursor < fileContent.length - 1 ? fileContent[leftCursor + 1] : '';
+
+            if(currentChar === '\n'){
+                lineCounter++;
+            }
+
+            if(currentChar === '-' && nextChar === '-'){
+                currentBlockString = currentBlockString.slice(0, -1); // Remove the '-' from the block string
+                const nextNextChar = leftCursor < fileContent.length - 2 ? fileContent[leftCursor + 2] : '';
+                if(nextNextChar === '['){
+                    // Multiline comment, skip to closing ]]
+                    leftCursor += 3; // Skip the --[
+                    while(leftCursor < fileContent.length && !(fileContent[leftCursor] === ']' && fileContent[leftCursor + 1] === ']')){
+                        if(fileContent[leftCursor] === '\n'){
+                            lineCounter++;
+                        }
+                        leftCursor++;
+                    }
+                    leftCursor += 2; // Skip the closing ]]
+                } else {
+                     // Comment line, skip to end of line
+                    while(leftCursor < fileContent.length && fileContent[leftCursor] !== '\n'){
+                        leftCursor++;
+                    }
+                    if(fileContent[leftCursor] === '\n'){
+                        lineCounter++;
+                    }
+                }
+                currentWord = '';
+                continue;
+            }
+
+            if(currentChar === '"' || currentChar === "'"){
+                // String literal, skip to closing quote
+                const quoteType = currentChar;
+                leftCursor++;
+                while(leftCursor < fileContent.length && fileContent[leftCursor] !== quoteType){
+                    // Add string but don't process it for keywords
+                    currentBlockString += fileContent[leftCursor];
+                    leftCursor++;
+                }
+                currentBlockString += quoteType; // Add the closing quote
+                leftCursor++; // Skip the closing quote
+                currentWord = '';
+                continue;
+            }
+
+            if(currentChar === "\n") {
+                // Process line
+                currentBlockString = currentBlockString.trimEnd(); // Remove trailing whitespace
+                const lineBlock = new LineBlock(lineCounter, currentBlockString, currentBlock);
+                currentBlock.childBlocks.push(lineBlock);
+                currentBlockString = '';
+
+                if(currentBlock.blockType === BlockType.Return){
+                    // TODO: multi line return blocks
+                    //Return blocks don't end on end, but on a new line 
+                    currentBlock = currentBlock.getParentBlock()!;
+                }
+            }
+
+            if(currentChar === ")" && currentBlock.blockType === BlockType.Require){
+                currentBlock = currentBlock.getParentBlock()!;
+            } 
+
+            if(nextChar.trim() === '' || nextChar === '('){
+                // End of a word, check for keywords
+                const trimmedWord = currentWord.trim();
+                let blockToAdd : CodeBlock | null = null;
+                if(trimmedWord === 'if'){
+                    blockToAdd = new IfBlock(lineCounter, currentBlock);
+                }
+                else if(trimmedWord === 'while'){
+                    blockToAdd = new WhileBlock(lineCounter, currentBlock);
+                }
+                else if(trimmedWord === 'for'){
+                    blockToAdd = new ForBlock(lineCounter, currentBlock);
+                }
+                else if(trimmedWord === 'do'){
+                    blockToAdd = new DoBlock(lineCounter, currentBlock);
+                }
+                else if(trimmedWord === 'return'){
+                    blockToAdd = new ReturnBlock(lineCounter, currentBlock);
+                }
+                else if(trimmedWord === 'function') {
+                    blockToAdd = new FunctionBlock(lineCounter, currentBlock);
+                }
+                else if (trimmedWord === 'require') {
+                    blockToAdd = new RequireBlock(lineCounter, currentBlock, leftCursor - currentWord.length, leftCursor);
+                } else if(trimmedWord === 'end'){
+                    const parent = currentBlock.getParentBlock();
+                    if(!parent){
+                        onError?.({
+                            filePath: luaFilePath,
+                            line: fileContent.substring(0, leftCursor).split('\n').length - 1,
+                            message: "Unexpected 'end' without matching block start"
+                        });
+                    }
+                    currentBlock = currentBlock.getParentBlock()!
+                } 
+
+                if(blockToAdd){
+                    currentBlock.childBlocks.push(blockToAdd);
+                    currentBlock = blockToAdd;
+                }
+
+                currentWord = '';
+            }
+            leftCursor++;
+        }
+        return file;
+    }
+}
+
+export class LineBlock extends CodeBlock {
+
+    constructor(sourceLineNumber: number, private line: string, parent?: CodeBlock){
+        super(sourceLineNumber, BlockType.Line, parent);
+    }
+    
+    toLines(): string[] {
+        return [this.line];
+    }    
+}
+
+export class IfBlock extends CodeBlock {
+    
+    constructor(sourceLineNumber: number, parent: CodeBlock){
+        super(sourceLineNumber, BlockType.If, parent);
+    }
+
+    toLines(): string[] {
+        const lines: string[] = [];
+        for(const child of this.childBlocks){
+            lines.push(...child.toLines());
+        }
+        return lines;
+    }
+}
+
+export class WhileBlock extends CodeBlock {
+
+    constructor(sourceLineNumber: number, parent: CodeBlock){
+        super(sourceLineNumber, BlockType.While, parent);
+    }
+
+    toLines(): string[] {
+        const lines: string[] = [];
+        for(const child of this.childBlocks){
+            lines.push(...child.toLines());
+        }
+        return lines;
+    }
+}
+
+export class ForBlock extends CodeBlock {
+
+    constructor(sourceLineNumber: number, parent: CodeBlock){
+        super(sourceLineNumber, BlockType.For, parent);
+    }
+
+    toLines(): string[] {
+        const lines: string[] = [];
+        for(const child of this.childBlocks){
+            lines.push(...child.toLines());
+        }
+        return lines;
+    }
+}
+
+
+export class LuaFile extends CodeBlock {
+    
+    constructor() {
+        super(0, BlockType.File);
+    }
+
+    toLines(): string[] {
+        const lines: string[] = [];
+        for(const child of this.childBlocks){
+            lines.push(...child.toLines());
+        }
+        return lines;
+    }
+}
+
+export class DoBlock extends CodeBlock {
+
+    constructor(sourceLineNumber: number, parent: CodeBlock){
+        super(sourceLineNumber, BlockType.Do, parent);
+    }
+
+    toLines(): string[] {
+        const lines: string[] = [];
+        for(const child of this.childBlocks){
+            lines.push(...child.toLines());
+        }
+        return lines;
+    }
+}
+
+export class ReturnBlock extends CodeBlock {
+
+    constructor(sourceLineNumber: number, parent: CodeBlock){
+        super(sourceLineNumber, BlockType.Return, parent);
+    }
+
+    toLines(): string[] {
+        const lines: string[] = [];
+        for(const child of this.childBlocks){
+            lines.push(...child.toLines());
+        }
+        return lines;
+    }
+}
+
+export class FunctionBlock extends CodeBlock {
+
+    constructor(sourceLineNumber: number, parent: CodeBlock){
+        super(sourceLineNumber, BlockType.Function, parent);
+    }
+
+    toLines(): string[] {
+        const lines: string[] = [];
+        for(const child of this.childBlocks){
+            lines.push(...child.toLines());
+        }
+        return lines;
+    }
+}
+
+export class RequireBlock extends CodeBlock {
+
+    constructor(sourceLineNumber: number, parent: CodeBlock, public readonly charStart?: number, public readonly charEnd?: number){
+        super(sourceLineNumber, BlockType.Require, parent);
+    }
+
+    toLines(): string[] {
+        const lines: string[] = [];
+        for(const child of this.childBlocks){
+            lines.push(...child.toLines());
+        }
+        return lines;
+    }
+
+    getRequiredString(): string {
+        if(this.childBlocks.length === 0){
+            return '';
+        }
+        const firstChild = this.childBlocks[0];
+        if(firstChild instanceof LineBlock){
+            return firstChild.toLines()[0].trim().replace(/['"]/g, '');
+        }
+        return '';
+    }
+}

compiler/src/CompilationError.ts

@@ -0,0 +1,9 @@
+
+
+export interface CompilationError {
+    filePath: string;
+    line: number;
+    charStart?: number;
+    charEnd?: number;
+    message: string;
+}

compiler/src/ScriptCompiler.ts

@@ -1,13 +1,7 @@
 import * as fs from 'fs';
 import * as path from 'path';
-
+import { LuaFile, BlockType, CodeBlock, RequireBlock } from './CodeBlocks';
-export interface CompilationError {
+import { CompilationError } from './CompilationError';
-    filePath: string;
-    line: number;
-    charStart?: number;
-    charEnd?: number;
-    message: string;
-}
 
 export interface ScriptCompilerOptions {
     sourcePath: string,
@@ -23,6 +17,8 @@ export interface ICompilationLogger {
     writeLine(message: string): void;
 }
 
+export { CompilationError };
+
 class Metrics {
     public totalLinesRead : number = 0;
     public totalLinesWritten: number = 0;
@@ -66,9 +62,9 @@ export class ScriptCompiler {
             if (entry.isFile() && entry.name.endsWith('.lua')) {
                 const fullPath = path.join(entry.parentPath, entry.name);
                 const relativePath = path.relative(this.options.sourcePath, fullPath);
-                const content = fs.readFileSync(fullPath, 'utf-8');
+                const luaFile = LuaFile.createFromFile(fullPath, this.options.onError);
+                const parsedFile = new ParsedFile(fileReferenceToLuaVariable(relativePath), luaFile, fullPath);
 
-                const parsedFile = this.parseFile(relativePath, content, fullPath, metricsMeter);
                 parsedFiles.set(parsedFile.fileKey, parsedFile);
                 metricsMeter.filesRead++;
             }
@@ -83,6 +79,8 @@ export class ScriptCompiler {
             this.options.onError
         );
         
+        writer.logDependencyTree();
+
         const writeStart = Date.now();
         writer.write(includeDevScript, metricsMeter);
         const writeEnd = Date.now();
@@ -95,199 +93,6 @@ export class ScriptCompiler {
         metricsMeter.totalTimeMs = (end-start);
         metricsMeter.log(this.logger);
     }
-
-    private reportError(filePath: string, line: number, charStart: number | undefined, charEnd: number | undefined, message: string): void {
-        if (this.options.onError) {
-            this.options.onError({ filePath, line, charStart, charEnd, message });
-        }
-    }
-
-    private parseFile(filePath: string, content: string, fullPath: string, metricsMeter: Metrics): ParsedFile {
-        const dependencies: Dependency[] = [];
-        const newLines: string[] = [`do --${filePath}`];
-
-        content = stripLuaMultilineComments(content);
-        const lines = content.split('\n').map(line => line.replace(/--.*$/, ''));
-        
-        const blockStack : string[] = [];
-        let isInFunction = false;
-        let foundModuleLevelReturn = false;
-        let expectingDo = false;
-        
-        const blockFound = (blockType: string): void => {
-            blockStack.push(blockType);
-            if (blockType === 'function') {
-                isInFunction = true;
-            }
-        };
-
-        const blockClosed = (): void => {
-            const closedBlock = blockStack.pop();
-            if (closedBlock === 'function') {
-                isInFunction = blockStack.includes('function');
-            }
-        };
-
-        for (let i = 0; i < lines.length; i++) {
-            metricsMeter.totalLinesRead++;
-            let line = lines[i];
-            const trimmedLine = line.trim();
-
-            // Skip comments and blank lines
-            if (trimmedLine === '' || trimmedLine.startsWith('--')) {
-                continue;
-            }
-
-            // If we found module-level return, only allow 'end' statements after it
-            if (foundModuleLevelReturn) {
-                // Check for 'end' keyword
-                if (/\bend\b/.test(trimmedLine)) {
-                    const endMatches = trimmedLine.match(/\bend\b/g);
-                    if (endMatches) {
-                        for (let j = 0; j < endMatches.length; j++) {
-                            blockClosed();
-                        }
-                    }
-                    newLines.push(line);
-                } else {
-                    this.reportError(fullPath, i, undefined, undefined, `Code found after module-level return: ${trimmedLine}`);
-                    newLines.push(line); // Continue processing despite error
-                }
-                continue;
-            }
-
-            // Handle require statements
-            const requireMatch = line.match(/require\(['"](.+?)['"]\)/);
-            if (requireMatch) {
-                const textMatch = requireMatch[1];
-                const requiredModule = fileReferenceToLuaVariable(textMatch);
-
-                dependencies.push(new Dependency(textMatch, i, requireMatch.index ?? 0, (requireMatch.index ?? 0) + requireMatch[0].length));
-                line = line.replace(requireMatch[0], requiredModule);
-            }
-
-            // Track block keywords AND returns - need to process in order they appear
-            const keywords = [
-                { regex: /\bfunction\b/, type: 'function' },
-                { regex: /\bif\b/, type: 'if' },
-                { regex: /\bfor\b/, type: 'for' },
-                { regex: /\bwhile\b/, type: 'while' },
-                { regex: /\bdo\b/, type: 'do' },
-                { regex: /\bend\b/, type: 'end' },
-                { regex: /\breturn\b/, type: 'return' }  // Add return to the list!
-            ];
-
-            // Find positions of all keywords in the line
-            const foundKeywords: Array<{ position: number, type: string }> = [];
-            for (const kw of keywords) {
-                const matches = [...trimmedLine.matchAll(new RegExp(kw.regex, 'g'))];
-                for (const match of matches) {
-                    if (match.index !== undefined) {
-                        foundKeywords.push({ position: match.index, type: kw.type });
-                    }
-                }
-            }
-
-            // Sort by position to process in order
-            foundKeywords.sort((a, b) => a.position - b.position);
-
-            // Process keywords in order
-            for (const kw of foundKeywords) {
-                if (kw.type === 'end') {
-                    blockClosed();
-                    expectingDo = false;
-                } else if (kw.type === 'for' || kw.type === 'while') {
-                    blockFound(kw.type);
-                    expectingDo = true;
-                } else if (kw.type === 'do') {
-                    if (!expectingDo) {
-                        // Standalone do block
-                        blockFound('do');
-                    }
-                    expectingDo = false;
-                } else if (kw.type === 'return') {
-                    // Handle return in sequence
-                    if (!isInFunction && !foundModuleLevelReturn) {
-                        // Extract the return value (everything after 'return')
-                        const afterReturnPos = kw.position + 6; // 'return' is 6 chars
-                        const afterReturn = trimmedLine.substring(afterReturnPos).trim();
-                        
-                        if (afterReturn === '') {
-                            this.reportError(fullPath, i, afterReturnPos, afterReturnPos, 'Empty return statement at module level');
-                            continue;
-                        }
-
-                        // Check for multiple return values (commas outside of parentheses/braces/brackets)
-                        let parenDepth = 0;
-                        let braceDepth = 0;
-                        let bracketDepth = 0;
-                        let inString = false;
-                        let stringChar = '';
-                        let hasMultipleValues = false;
-                        
-                        for (let j = 0; j < afterReturn.length; j++) {
-                            const char = afterReturn[j];
-                            
-                            if (!inString) {
-                                if (char === '"' || char === "'") {
-                                    inString = true;
-                                    stringChar = char;
-                                } else if (char === '(') {
-                                    parenDepth++;
-                                } else if (char === ')') {
-                                    parenDepth--;
-                                } else if (char === '{') {
-                                    braceDepth++;
-                                } else if (char === '}') {
-                                    braceDepth--;
-                                } else if (char === '[') {
-                                    bracketDepth++;
-                                } else if (char === ']') {
-                                    bracketDepth--;
-                                } else if (char === ',' && parenDepth === 0 && braceDepth === 0 && bracketDepth === 0) {
-                                    hasMultipleValues = true;
-                                    break;
-                                }
-                            } else {
-                                if (char === stringChar && afterReturn[j - 1] !== '\\') {
-                                    inString = false;
-                                }
-                            }
-                        }
-                        
-                        if (hasMultipleValues) {
-                            this.reportError(fullPath, i, afterReturnPos, afterReturnPos + afterReturn.length, `Multiple return values not supported: ${trimmedLine}`);
-                        } else {
-                            // Replace return with assignment
-                            const moduleVariable = fileReferenceToLuaVariable(filePath);
-                            const parts = moduleVariable.split('.');
-                            for (let p = 1; p < parts.length; p++) {
-                                const path = parts.slice(0, p + 1).join('.');
-                                newLines.push(`if not ${path} then ${path} = {} end`);
-                            }
-
-                            line = line.replace(/\breturn\b/, moduleVariable + ' =');
-                            foundModuleLevelReturn = true;
-                        }
-                    }
-                } else {
-                    // function or if
-                    blockFound(kw.type);
-                    expectingDo = false;
-                }
-            }
-
-            newLines.push(line);
-        }
-
-        newLines.push(`end --${filePath}`);
-        return new ParsedFile(filePath, fullPath, newLines, dependencies);
-    }
-}
-
-function stripLuaMultilineComments(content: string): string {
-    // Matches --[[...]], --[=[...]=], --[==[...]==], etc.
-    return content.replace(/--\[(=*)\[[\s\S]*?\]\1\]/g, '');
 }
 
 class Dependency {
@@ -304,18 +109,6 @@ class Dependency {
     }
 }
 
-class ParsedFile {
-    public readonly fileKey: string;
-    
-    constructor(
-        public readonly filePath: string,
-        public readonly fullPath: string,
-        public readonly lines: string[],
-        public readonly dependencies: Dependency[]
-    ) {
-        this.fileKey = fileReferenceToLuaVariable(filePath);
-    }
-}
 
 function fileReferenceToLuaVariable(fileReference: string): string {
     // Remove .lua extension
@@ -348,6 +141,37 @@ function fileReferenceToLuaVariable(fileReference: string): string {
     return result;
 }
 
+class ParsedFile {
+    public readonly dependencies: Dependency[] = []
+    
+    constructor(
+        public readonly fileKey: string,
+        public readonly luaFile: LuaFile,
+        public readonly fullPath: string
+    ){
+        this.dependencies = ParsedFile.parseDependencies(luaFile);
+    }
+
+    private static parseDependencies(luaFile: CodeBlock): Dependency[] {
+        // Recursively search for RequireBlocks in the LuaFile and its child blocks
+        const dependencies: Dependency[] = [];
+        const searchBlock = (block: CodeBlock) => {
+            if (block instanceof RequireBlock) {
+                const requiredString = block.getRequiredString();
+                if (requiredString) {
+                    dependencies.push(new Dependency(requiredString.toLowerCase(), block.sourceLineNumber ?? 0, block.charStart ?? 0, block.charEnd ?? 0));
+                }
+            }
+
+            for (const child of block.getChildren()) {
+                searchBlock(child);
+            }
+        }
+        searchBlock(luaFile);
+        return dependencies;
+    }
+}
+
 class Writer {
     constructor(
         public location: string,
@@ -462,8 +286,12 @@ class Writer {
                     });
                 }
             }
-            metrics.totalLinesWritten += parsedFile.lines.length;
+
-            outputLines.push(...parsedFile.lines);
+            outputLines.push("do -- " + parsedFile.fileKey);
+            const lines = parsedFile.luaFile.toLines();
+            metrics.totalLinesWritten += lines.length;
+            outputLines.push(...lines.filter(line => line.trim() !== '')); // Filter out empty lines
+            outputLines.push("end -- " + parsedFile.fileKey);
             writtenFiles.add(parsedFile.fileKey);
             metrics.filesWritten++;
         };

dutchies-dcs-scripting-tools/esbuild.js

@@ -47,6 +47,7 @@ async function main() {
 	});
 	if (watch) {
 		await ctx.watch();
+		console.log('[watch] watching for changes...');
 	} else {
 		await ctx.rebuild();
 		await ctx.dispose();

dutchies-dcs-scripting-tools/src/extension.ts

@@ -138,6 +138,7 @@ async function compileLuaScripts() {
         await compiler.compile(includeDevScript);
     } catch (err) {
         vscode.window.showErrorMessage('Compilation failed: ' + (err as Error).message);
+        logger.error('Compilation failed: ' + (err as Error).message + '\n' + (err as Error).stack);
     }
     
     // Update diagnostics