scanner: scanNumber fix (IMAG, RAT, UNIT)

scanner/scanner.go

@@ -54,8 +54,8 @@ type Scanner struct {
 	rdOffset   int  // reading offset (position after current character)
 	lineOffset int  // current line offset
 	nParen     int
+	unitVal    string
 	insertSemi bool // insert a semicolon before next newline
-	needUnit   bool
 
 	// public state - ok to modify
 	ErrorCount int // number of errors encountered
@@ -503,18 +503,19 @@ func (s *Scanner) scanNumber() (token.Token, string) {
 		s.error(s.offset, "hexadecimal mantissa requires a 'p' exponent")
 	}
 
-	// suffix 'i'
-	if s.ch == 'i' {
-		tok = token.IMAG
-		s.next()
-	} else if s.ch == 'r' {
-		tok = token.RAT
-		s.next()
-	} else if isLetter(s.ch) {
-		s.needUnit = true
+	if isLetter(s.ch) {
+		id := s.scanIdentifier()
+		switch id {
+		case "i":
+			tok = token.IMAG
+		case "r":
+			tok = token.RAT
+		default:
+			s.unitVal = id
+		}
 	}
 
-	lit := string(s.src[offs:s.offset])
+	lit := string(s.src[offs : s.offset-len(s.unitVal)])
 	if tok == token.INT && invalid >= 0 {
 		s.errorf(invalid, "invalid digit %q in %s", lit[invalid-offs], litname(prefix))
 	}
@@ -831,10 +832,11 @@ scanAgain:
 
 	// determine token value
 	insertSemi := false
-	if s.needUnit { // number with unit
+	if s.unitVal != "" { // number with unit
 		insertSemi = true
-		tok, lit = token.UNIT, s.scanIdentifier()
-		s.needUnit = false
+		pos -= token.Pos(len(s.unitVal))
+		tok, lit = token.UNIT, s.unitVal
+		s.unitVal = ""
 		goto done
 	}
 	switch ch := s.ch; {

tpl/scanner/_testdata/num/go.expect

@@ -0,0 +1,15 @@
+1 IDENT echo
+6 INT 1_002
+11 ; 
+
+12 IDENT echo
+17 INT 2
+18 + 
+19 IMAG 3i
+21 ; 
+
+22 IDENT echo
+27 FLOAT 2.
+29 IDENT string
+35 ; 
+

tpl/scanner/_testdata/num/gop.expect

@@ -0,0 +1,15 @@
+1 IDENT echo
+6 INT 1_002
+11 ; 
+
+12 IDENT echo
+17 INT 2
+18 + 
+19 IMAG 3i
+21 ; 
+
+22 IDENT echo
+27 FLOAT 2.
+29 UNIT string
+35 ; 
+

tpl/scanner/_testdata/num/in.gop

@@ -0,0 +1,3 @@
+echo 1_002
+echo 2+3i
+echo 2.string

tpl/scanner/_testdata/num/tpl.expect

@@ -0,0 +1,15 @@
+1 IDENT echo
+6 INT 1_002
+11 ; 
+
+12 IDENT echo
+17 INT 2
+18 + 
+19 IMAG 3i
+21 ; 
+
+22 IDENT echo
+27 FLOAT 2.
+29 UNIT string
+35 ; 
+

tpl/scanner/_testdata/unit/go.expect

@@ -2,32 +2,57 @@
 8 STRING "time"
 14 ; 
 
-16 IDENT echo
+16 IDENT x
+18 := 
 21 STRING "Let's wait for a second"
 46 ; 
 
-47 IDENT time
-51 . 
-52 IDENT sleep
-58 INT 1
-59 IDENT s
-60 ; 
-
-61 IDENT time
-65 . 
-66 IDENT sleep
-72 FLOAT 1.
-74 IDENT s
-75 ; 
-
-76 IDENT time
-80 . 
-81 IDENT sleep
-87 FLOAT 1.2
-90 IDENT s
-91 ; 
-
-92 IDENT echo
-97 STRING "Hello, world"
-111 ; 
+47 IDENT echo
+52 IDENT x
+53 ; 
 
+55 IDENT wait
+60 INT 0x10
+64 IDENT s
+65 ; 
+
+66 IDENT wait
+71 FLOAT .1
+73 IDENT s
+74 ; 
+
+75 IDENT wait
+80 FLOAT 1.
+82 IDENT s
+83 ; 
+
+84 IDENT wait
+89 FLOAT 1.e7
+93 IDENT s
+94 ; 
+
+95 IDENT wait
+100 FLOAT 1.2
+103 IDENT s
+104 ; 
+
+106 IDENT time
+110 . 
+111 IDENT sleep
+117 IMAG 100i
+121 IDENT m
+122 ; 
+
+124 IDENT spend
+130 FLOAT 10.0
+134 IDENT rmb
+137 ; 
+
+138 IDENT spend
+144 FLOAT 10.
+147 IDENT rmb
+150 ; 
+
+152 IDENT echo
+157 STRING "Hello, world"
+171 ... 

tpl/scanner/_testdata/unit/gop.expect

@@ -2,32 +2,59 @@
 8 STRING "time"
 14 ; 
 
-16 IDENT echo
+16 IDENT x
+18 := 
 21 STRING "Let's wait for a second"
 46 ; 
 
-47 IDENT time
-51 . 
-52 IDENT sleep
-58 INT 1
-59 UNIT s
-60 ; 
-
-61 IDENT time
-65 . 
-66 IDENT sleep
-72 FLOAT 1.
-74 UNIT s
-75 ; 
-
-76 IDENT time
-80 . 
-81 IDENT sleep
-87 FLOAT 1.2
-90 UNIT s
-91 ; 
-
-92 IDENT echo
-97 STRING "Hello, world"
-111 ; 
+47 IDENT echo
+52 IDENT x
+53 ; 
+
+55 IDENT wait
+60 INT 0x10
+64 UNIT s
+65 ; 
+
+66 IDENT wait
+71 FLOAT .1
+73 UNIT s
+74 ; 
+
+75 IDENT wait
+80 FLOAT 1.
+82 UNIT s
+83 ; 
+
+84 IDENT wait
+89 FLOAT 1.e7
+93 UNIT s
+94 ; 
+
+95 IDENT wait
+100 FLOAT 1.2
+103 UNIT s
+104 ; 
+
+106 IDENT time
+110 . 
+111 IDENT sleep
+117 INT 100
+120 UNIT im
+122 ; 
+
+124 IDENT spend
+130 FLOAT 10.0
+134 UNIT rmb
+137 ; 
+
+138 IDENT spend
+144 FLOAT 10.
+147 UNIT rmb
+150 ; 
+
+152 IDENT echo
+157 STRING "Hello, world"
+171 ... 
+174 ; 
 

tpl/scanner/_testdata/unit/in.gop

@@ -1,7 +1,17 @@
 import "time"
 
-echo "Let's wait for a second"
-time.sleep 1s
-time.sleep 1.s
-time.sleep 1.2s
-echo "Hello, world"
+x := "Let's wait for a second"
+echo x
+
+wait 0x10s
+wait .1s
+wait 1.s
+wait 1.e7s
+wait 1.2s
+
+time.sleep 100im
+
+spend 10.0rmb
+spend 10.rmb
+
+echo "Hello, world"...

tpl/scanner/_testdata/unit/tpl.expect

@@ -2,32 +2,57 @@
 8 STRING "time"
 14 ; 
 
-16 IDENT echo
+16 IDENT x
+18 := 
 21 STRING "Let's wait for a second"
 46 ; 
 
-47 IDENT time
-51 . 
-52 IDENT sleep
-58 INT 1
-59 UNIT s
-60 ; 
-
-61 IDENT time
-65 . 
-66 IDENT sleep
-72 FLOAT 1.
-74 UNIT s
-75 ; 
-
-76 IDENT time
-80 . 
-81 IDENT sleep
-87 FLOAT 1.2
-90 UNIT s
-91 ; 
-
-92 IDENT echo
-97 STRING "Hello, world"
-111 ; 
+47 IDENT echo
+52 IDENT x
+53 ; 
 
+55 IDENT wait
+60 INT 0x10
+64 UNIT s
+65 ; 
+
+66 IDENT wait
+71 FLOAT .1
+73 UNIT s
+74 ; 
+
+75 IDENT wait
+80 FLOAT 1.
+82 UNIT s
+83 ; 
+
+84 IDENT wait
+89 FLOAT 1.e7
+93 UNIT s
+94 ; 
+
+95 IDENT wait
+100 FLOAT 1.2
+103 UNIT s
+104 ; 
+
+106 IDENT time
+110 . 
+111 IDENT sleep
+117 INT 100
+120 UNIT im
+122 ; 
+
+124 IDENT spend
+130 FLOAT 10.0
+134 UNIT rmb
+137 ; 
+
+138 IDENT spend
+144 FLOAT 10.
+147 UNIT rmb
+150 ; 
+
+152 IDENT echo
+157 STRING "Hello, world"
+171 ...