feat: Lexer, tokenize

Signed-off-by: Lexus Drumgold <[email protected]>

Author: unicornware

.codecov.yml

@@ -89,4 +89,7 @@ ignore:
   - '!src/index.ts'
 
 profiling:
-  critical_files_paths: []
+  critical_files_paths:
+    - src/constructs/eof.ts
+    - src/constructs/initialize.ts
+    - src/lexer.ts

.commitlintrc.ts

@@ -18,7 +18,7 @@ import { scopes } from '@flex-development/commitlint-config'
 const config: UserConfig = {
   extends: ['@flex-development'],
   rules: {
-    'scope-enum': [Severity.Error, 'always', scopes(['chore'])]
+    'scope-enum': [Severity.Error, 'always', scopes(['chore', 'constructs'])]
   }
 }
 

.dictionary.txt

@@ -21,10 +21,13 @@ mkbuild
 mlly
 nocheck
 nvmrc
+onreturn
 pathe
 pkgs
 preid
 shfmt
+succ
+tokenizes
 unstub
 vates
 vfile

.dprint.jsonc

@@ -35,7 +35,7 @@
       },
       {
         "command": "node ./dprint/shfmt.mjs {{file_path}}",
-        "exts": ["sh", "txt", "zsh"],
+        "exts": ["sh", "zsh"],
         "fileNames": [
           ".editorconfig",
           ".env",

.github/infrastructure.yml

@@ -41,7 +41,7 @@ branches:
           - context: test (20)
           - context: typescript (5.3.3)
           - context: typescript (5.4.5)
-          - context: typescript (5.5.0-beta)
+          - context: typescript (5.5.2)
           - context: typescript (latest)
         strict: true
       restrictions: null

.github/workflows/ci.yml

@@ -331,7 +331,7 @@ jobs:
       - id: test
         if: steps.test-files-check.outputs.files_exists == 'true'
         name: Run tests
-        run: yarn test:cov --segfault-retry=3
+        run: yarn test:cov
       - id: codecov
         name: Upload coverage report to Codecov
         if: steps.test-files-check.outputs.files_exists == 'true'

README.md

@@ -2,7 +2,7 @@
 
 [![github release](https://img.shields.io/github/v/release/flex-development/vfile-lexer.svg?include_prereleases&sort=semver)](https://github.com/flex-development/vfile-lexer/releases/latest)
 [![npm](https://img.shields.io/npm/v/@flex-development/vfile-lexer.svg)](https://npmjs.com/package/@flex-development/vfile-lexer)
-[![codecov](https://codecov.io/gh/flex-development/vfile-lexer/graph/badge.svg?token=)](https://codecov.io/gh/flex-development/vfile-lexer)
+[![codecov](https://codecov.io/gh/flex-development/vfile-lexer/graph/badge.svg?token=iA1BvaucoZ)](https://codecov.io/gh/flex-development/vfile-lexer)
 [![module type: esm](https://img.shields.io/badge/module%20type-esm-brightgreen)](https://github.com/voxpelli/badges-cjs-esm)
 [![license](https://img.shields.io/github/license/flex-development/vfile-lexer.svg)](LICENSE.md)
 [![conventional commits](https://img.shields.io/badge/-conventional%20commits-fe5196?logo=conventional-commits&logoColor=ffffff)](https://conventionalcommits.org/)

__fixtures__/hello.txt

@@ -0,0 +1,2 @@
+const 你好 = "hello 👋";
+console.log(\u4f60\u597d); // hello 👋

__fixtures__/inline-tag.txt

@@ -0,0 +1 @@
+{@linkcode Code}

__fixtures__/numerics.txt

@@ -0,0 +1,20 @@
+0
+0n
+1
+1n
+2
+2n
+3
+3n
+4
+4n
+5
+5n
+6
+6n
+7
+7n
+8
+8n
+9
+9n