From 3c151c1ed802e1110b11f4d836cdb3d8220b84b3 Mon Sep 17 00:00:00 2001
From: BorysLevytskyi <borys.levytskyi@gmail.com>
Date: Fri, 5 May 2023 17:40:13 +0200
Subject: [PATCH] tokenize

---
 src/expression/ExpressionParser.test.ts | 114 ++++++++++++++----------
 1 file changed, 65 insertions(+), 49 deletions(-)

diff --git a/src/expression/ExpressionParser.test.ts b/src/expression/ExpressionParser.test.ts
index da36764..9c6f88e 100644
--- a/src/expression/ExpressionParser.test.ts
+++ b/src/expression/ExpressionParser.test.ts
@@ -2,67 +2,83 @@ import { NumericLiteral } from "typescript";
 import ExpressionOperand from "./ExpressionOperand";
 import { ScalarOperand } from "./expression";
 import { ExpressionInputItem } from "./expression-interfaces";
+import { type } from "os";
 
 const NUMBER_REGEX = /./;
 
-class ExpressionParser
-{
-    input: string;
-    pos: number;
-    
-    constructor(input: string) {
-        this.input = input;
-        this.pos = 0;
-    }
+type TokenDef = {
+    regex: RegExp,
+    type: string,
+    skip?: boolean
+}
 
-    tryParse () : ExpressionInputItem[] | null {
-        return null;
-    } 
+const decimalRegex = /^-?\d+/;
+const hexRegex = /^-?0x[0-9,a-f]+/i;
+const binRegex = /^-?0b[0-1]+/i;
+const operatorRegex = /^<<|>>|<<<|\&|\|\^|~/;
 
-    parse() : ExpressionInputItem[] {
-        const s = this.input;
-        const p = this.pos;
-        const items : ExpressionInputItem[] = [];
-        
-        while(p < s.length) {
-            
-           
-           
+type Token = {
+    value: string,
+    type: string,
+}
+
+const TOKENS: TokenDef[] = [
+    { regex: /^>>>/, type: "operator-urightshift"},
+    { regex: /^>>/, type: "operator-rightshift"},
+    { regex: /^<</, type: "operator-rightshift"},
+    { regex: /^\^/, type: "operator-xor"},
+    { regex: /^\|/, type: "operator-or"},
+    { regex: /^\&/, type: "operator-and"},
+    { regex: /^~/, type: "operator-not"},
+    { regex: /^-/, type: "operator-subtraction"},
+    { regex: hexRegex, type: "scalar-hex" },
+    { regex: binRegex, type: "scalar-binary" },
+    { regex: decimalRegex, type: "scalar-decimal" },
+    { regex: /^\s+/, type: "whitespace", skip: true }
+];
+
+function tokenize(input: string): string[] {
+    let cur = input;
+    const found: string[] = [];
+
+    while (true) {
+
+        try {
+            const firstToken = findFirst(TOKENS, t => t.regex.test(cur));
+
+            const match = firstToken.regex.exec(cur)!;
+            const value = match[0];
+
+            if (!firstToken.skip)
+                found.push(value);
+
+            if (cur.length == value.length)
+                break;
+
+            cur = cur.substring(value.length);
         }
-
-        return items;
-    }
-
-    parseSingle(): ExpressionInputItem {
-        const s = this.input;
-        const p = this.pos;
-
-        const ch = s[p];
-
-        if(ch == '~')
-        {
-            this.move(1); 
-            const inner = this.parseSingle()
-            return new ExpressionOperand("~" + inner, inner, "~");
+        catch (ex) {
+            console.log("cur: \"" + cur + "\"");
+            throw ex;
         }
-
-        const ss = s.substring(p);
-        const scalar = ScalarOperand.tryParse(ss);
-
-        if(scalar == null)
-            throw new Error(`Expected scalar value at pos ${p}: ${ss}`);
-
-        return {} as ExpressionInputItem;
     }
 
-    move(step : number) {
-        this.pos += step;
+    return found;
+}
+
+function findFirst<T>(arr: T[], predicate: (item: T) => boolean): T {
+    for (var i = 0; i < arr.length; i++) {
+        if (predicate(arr[i]))
+            return arr[i];
     }
 
-    
+    throw new Error("Array doesn't have item that satisfies given predicate");
 }
 
 it("parses simple expression", () => {
-    const input = "1+1";
-    
+    expect(tokenize("~-0>>>1>>2|3&4^5")).toEqual(["~", "-", "0", ">>>", "1", ">>", "2", "|", "3", "&", "4", "^", "5"]);
+    expect(tokenize("-1>>>1>>2")).toEqual(["-", "1", ">>>", "1", ">>", "2"]);
+    expect(tokenize("-1")).toEqual(["-", "1"]);
+    expect(tokenize("123 0b0101 0xaf")).toEqual(["123", "0b0101", "0xaf"]);
+    expect(tokenize("123 987")).toEqual(["123", "987"]);
 })
\ No newline at end of file