diff --git a/PowerShellSyntax.tmLanguage b/PowerShellSyntax.tmLanguage index ed2ddf0..29ba3f5 100644 --- a/PowerShellSyntax.tmLanguage +++ b/PowerShellSyntax.tmLanguage @@ -288,7 +288,7 @@ match - (?<!\w|-|\.)((?i:begin|break|catch|clean|continue|data|default|define|do|dynamicparam|else|elseif|end|exit|finally|for|from|if|in|inlinescript|parallel|param|process|return|sequence|switch|throw|trap|try|until|var|while)|%|\?)(?!\w) + (?<!\w|-|\.)((?i:begin|break|catch|clean|continue|data|default|define|dispose|do|dynamicparam|else|elseif|end|exit|finally|for|from|if|in|inlinescript|parallel|param|process|return|sequence|switch|throw|trap|try|until|var|while)|%|\?)(?!\w) name keyword.control.powershell diff --git a/examples/advancedFunction.ps1 b/examples/advancedFunction.ps1 index 4bd6c56..949caea 100644 --- a/examples/advancedFunction.ps1 +++ b/examples/advancedFunction.ps1 @@ -93,4 +93,7 @@ function Verb-Noun Clean { } + Dispose + { + } } diff --git a/package-lock.json b/package-lock.json index e75d8ea..d1a65ee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,32 +1,65 @@ { "name": "EditorSyntax", "version": "1.0.0", - "lockfileVersion": 1, + "lockfileVersion": 3, "requires": true, - "dependencies": { - "atom-grammar-test": { + "packages": { + "": { + "name": "EditorSyntax", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "atom-grammar-test": "^0.6.3", + "fast-plist": "0.1.2", + "vscode-oniguruma": "^2.0.1", + "vscode-textmate": "^9.3.2" + }, + "engines": { + "atom": "*", + "node": "*" + } + }, + "node_modules/atom-grammar-test": { "version": "0.6.4", "resolved": "https://registry.npmjs.org/atom-grammar-test/-/atom-grammar-test-0.6.4.tgz", "integrity": "sha1-2KU1A9H+k5mX9Ji3SirDEARKfU4=", - "requires": { - "chevrotain": "0.18.0", - "escape-string-regexp": "1.0.5" + "dependencies": { + "chevrotain": "^0.18.0", + "escape-string-regexp": "^1.0.5" + }, + "engines": { + "atom": ">=1.0.0 <2.0.0" } }, - "chevrotain": { + "node_modules/chevrotain": { "version": "0.18.0", "resolved": "https://registry.npmjs.org/chevrotain/-/chevrotain-0.18.0.tgz", "integrity": "sha1-sodxTjFZC64sXR4vYRZz7+xHnYA=" }, - "escape-string-regexp": { + "node_modules/escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "engines": { + "node": ">=0.8.0" + } }, - "fast-plist": { + "node_modules/fast-plist": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/fast-plist/-/fast-plist-0.1.2.tgz", "integrity": "sha1-pFr/NFGWAG1AbKbNzQX2kFHvNbg=" + }, + "node_modules/vscode-oniguruma": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-2.0.1.tgz", + "integrity": "sha512-poJU8iHIWnC3vgphJnrLZyI3YdqRlR27xzqDmpPXYzA93R4Gk8z7T6oqDzDoHjoikA2aS82crdXFkjELCdJsjQ==", + "license": "MIT" + }, + "node_modules/vscode-textmate": { + "version": "9.3.2", + "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-9.3.2.tgz", + "integrity": "sha512-n2uGbUcrjhUEBH16uGA0TvUfhWwliFZ1e3+pTjrkim1Mt7ydB41lV08aUvsi70OlzDWp6X7Bx3w/x3fAXIsN0Q==", + "license": "MIT" } } } diff --git a/package.json b/package.json index ef3ada8..6c47639 100644 --- a/package.json +++ b/package.json @@ -12,8 +12,10 @@ "node": "*" }, "dependencies": { + "atom-grammar-test": "^0.6.3", "fast-plist": "0.1.2", - "atom-grammar-test": "^0.6.3" + "vscode-oniguruma": "^2.0.1", + "vscode-textmate": "^9.3.2" }, "scripts": { "build-grammar": "node ./tools/build-grammar.js ./PowerShellSyntax.tmLanguage ./grammars/powershell.tmLanguage.json" diff --git a/parse.js b/parse.js new file mode 100644 index 0000000..1901763 --- /dev/null +++ b/parse.js @@ -0,0 +1,3 @@ +const fs = require('fs'); +const grammar = require('./grammars/powershell.tmLanguage.json'); +console.log(Object.keys(grammar)); diff --git a/run-tests.js b/run-tests.js new file mode 100644 index 0000000..1739847 --- /dev/null +++ b/run-tests.js @@ -0,0 +1,2 @@ +const test = require('atom-grammar-test'); +test('./grammars/powershell.tmLanguage.json', './spec/testfiles/syntax_test_Function.ps1'); diff --git a/spec/testfiles/syntax_test_Function.ps1 b/spec/testfiles/syntax_test_Function.ps1 index f527630..33bf957 100644 --- a/spec/testfiles/syntax_test_Function.ps1 +++ b/spec/testfiles/syntax_test_Function.ps1 @@ -373,4 +373,7 @@ function Verb-Noun { Clean { # <- keyword.control.powershell } + Dispose { + # <- keyword.control.powershell + } } diff --git a/test-grammar.js b/test-grammar.js new file mode 100644 index 0000000..df82384 --- /dev/null +++ b/test-grammar.js @@ -0,0 +1,4 @@ +const fs = require('fs'); +const plist = require('fast-plist'); +const grammar = plist.parse(fs.readFileSync('PowerShellSyntax.tmLanguage', 'utf8')); +// well, fast-plist only parses plist. We need a regex engine. diff --git a/test-tokenizer.js b/test-tokenizer.js new file mode 100644 index 0000000..1d0def6 --- /dev/null +++ b/test-tokenizer.js @@ -0,0 +1,47 @@ +const fs = require('fs'); +const path = require('path'); +const vsctm = require('vscode-textmate'); +const oniguruma = require('vscode-oniguruma'); + +const wasmBin = fs.readFileSync(path.join(__dirname, 'node_modules', 'vscode-oniguruma', 'release', 'onig.wasm')); + +const vscodeOnigurumaLib = oniguruma.loadWASM(wasmBin.buffer).then(() => { + return { + createOnigScanner(patterns) { return new oniguruma.OnigScanner(patterns); }, + createOnigString(s) { return new oniguruma.OnigString(s); } + }; +}); + +const registry = new vsctm.Registry({ + onigLib: vscodeOnigurumaLib, + loadGrammar: async (scopeName) => { + if (scopeName === 'source.powershell') { + const grammarPath = path.join(__dirname, 'grammars', 'powershell.tmLanguage.json'); + const grammarContent = fs.readFileSync(grammarPath, 'utf8'); + return vsctm.parseRawGrammar(grammarContent, grammarPath); + } + return null; + } +}); + +registry.loadGrammar('source.powershell').then(grammar => { + const text = fs.readFileSync(path.join(__dirname, 'spec', 'testfiles', 'syntax_test_Function.ps1'), 'utf8'); + const lines = text.split(/\r\n|\n/); + let ruleStack = vsctm.INITIAL; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const lineTokens = grammar.tokenizeLine(line, ruleStack); + ruleStack = lineTokens.ruleStack; + + if (line.includes('Clean')) { + console.log(`Line ${i + 1}: ${line}`); + lineTokens.tokens.forEach(token => { + const tokenText = line.substring(token.startIndex, token.endIndex); + if (tokenText.trim()) { + console.log(` Token: "${tokenText}"`); + console.log(` Scopes: ${token.scopes.join(', ')}`); + } + }); + } + } +}); diff --git a/test-tokenizer2.js b/test-tokenizer2.js new file mode 100644 index 0000000..83ac127 --- /dev/null +++ b/test-tokenizer2.js @@ -0,0 +1,40 @@ +const fs = require('fs'); +const path = require('path'); +const vsctm = require('vscode-textmate'); +const oniguruma = require('vscode-oniguruma'); + +const wasmBin = fs.readFileSync(path.join(__dirname, 'node_modules', 'vscode-oniguruma', 'release', 'onig.wasm')); + +oniguruma.loadWASM(wasmBin.buffer).then(() => { + const registry = new vsctm.Registry({ + onigLib: Promise.resolve({ + createOnigScanner(patterns) { return new oniguruma.OnigScanner(patterns); }, + createOnigString(s) { return new oniguruma.OnigString(s); } + }), + loadGrammar: async (scopeName) => { + const grammarPath = path.join(__dirname, 'grammars', 'powershell.tmLanguage.json'); + return vsctm.parseRawGrammar(fs.readFileSync(grammarPath, 'utf8'), grammarPath); + } + }); + + registry.loadGrammar('source.powershell').then(grammar => { + const text = fs.readFileSync(path.join(__dirname, 'spec', 'testfiles', 'syntax_test_Function.ps1'), 'utf8'); + const lines = text.split(/\r\n|\n/); + let ruleStack = vsctm.INITIAL; + for (let i = lines.length - 15; i < lines.length; i++) { + const line = lines[i]; + if (!line) continue; + const lineTokens = grammar.tokenizeLine(line, ruleStack); + ruleStack = lineTokens.ruleStack; + + console.log(`Line ${i + 1}: ${line}`); + lineTokens.tokens.forEach(token => { + const tokenText = line.substring(token.startIndex, token.endIndex); + if (tokenText.trim()) { + console.log(` Token: "${tokenText}"`); + console.log(` Scopes: ${token.scopes.join(', ')}`); + } + }); + } + }); +});