Improve whitespace handling in Lexer & Parser, warn on bad whitespace around binops

effekt/jvm/src/test/scala/effekt/LexerTests.scala

@@ -12,7 +12,7 @@ class LexerTests extends munit.FunSuite {
 
   def assertTokensEq(prog: String, expected: TokenKind*)(using Location): Unit = {
     val tokens = Lexer.lex(StringSource(prog, ""))
-    assertEquals(tokens.map { t => t.kind }, expected.toVector)
+    assertEquals(tokens.map { t => t.kind }.filterNot { k => k == Space }, expected.toVector)
   }
 
   def assertSuccess(prog: String)(using Location): Unit =

effekt/shared/src/main/scala/effekt/Lexer.scala

@@ -303,11 +303,11 @@ class Lexer(source: Source) extends Iterator[Token] {
   private def skipWhitespace(): Unit =
     while !atEndOfInput do
       currentChar match {
-        case ' ' | '\t' => advance()
+        case ' ' | '\t' => return // Stop here, let spaces be handled as a token
         case '\n' => return // Stop here, let newline be handled as a token
         case '\r' =>
           if nextChar == '\n' then return // Stop here for \r\n
-          else advance() // Treat standalone \r as whitespace
+          else advance() // Skip standalone \r
         case _ => return
       }
 
@@ -351,6 +351,11 @@ class Lexer(source: Source) extends Iterator[Token] {
       advance()
     }
 
+  private def advanceSpaces(): TokenKind = {
+    advanceWhile { (curr, _) => curr == ' ' || curr == '\t' }
+    TokenKind.Space
+  }
+
   private def peekAhead(offset: Int): Char =
     val targetIndex = position.offset + offset
     if targetIndex < source.content.length then
@@ -380,6 +385,8 @@ class Lexer(source: Source) extends Iterator[Token] {
     }
 
     (currentChar, nextChar) match {
+      case (' ',     _) => advanceSpaces()
+      case ('\t',    _) => advanceSpaces()
       case ('\n',    _) => advanceWith(TokenKind.Newline)
       case ('\r', '\n') => advance2With(TokenKind.Newline)
 

effekt/shared/src/main/scala/effekt/Parser.scala

@@ -114,6 +114,7 @@ class Parser(tokens: Seq[Token], source: Source) {
 
   // always points to the latest non-space position
   var position: Int = 0
+  spaces() // HACK: eat spaces before we begin!
 
   def recover(tokenKind: TokenKind, tokenPosition: Int): TokenKind = tokenKind match {
     case TokenKind.Error(err) =>
@@ -141,11 +142,19 @@ class Parser(tokens: Seq[Token], source: Source) {
 
   def peek: Token = tokens(position).failOnErrorToken(position)
 
-  /**
-   * Negative lookahead
-   */
-  def lookbehind(offset: Int): Token =
-    tokens(position - offset)
+  def sawNewlineLast: Boolean = {
+    @tailrec
+    def go(position: Int): Boolean =
+      if position < 0 then fail("Unexpected start of file")
+
+      tokens(position).failOnErrorToken(position) match {
+        case token if isSpace(token.kind) && token.kind != Newline => go(position - 1)
+        case token if token.kind == Newline => true
+        case _ => false
+      }
+
+    go(position - 1)
+  }
 
   /**
    * Peeks n tokens ahead of the current one.
@@ -317,7 +326,7 @@ class Parser(tokens: Seq[Token], source: Source) {
 
     // \n   while
     //      ^
-    case _ => lookbehind(1).kind == Newline
+    case _ => sawNewlineLast
   }
   def semi(): Unit = peek.kind match {
     // \n   ; while
@@ -329,7 +338,7 @@ class Parser(tokens: Seq[Token], source: Source) {
 
     // \n   while
     //      ^
-    case _ if lookbehind(1).kind == Newline => ()
+    case _ if sawNewlineLast => ()
 
     case _ => fail("Expected terminator: `;` or a newline")
   }
@@ -1001,6 +1010,7 @@ class Parser(tokens: Seq[Token], source: Source) {
     nonterminal:
       var left = nonTerminal()
       while (ops.contains(peek.kind)) {
+         checkBinaryOpWhitespace()
          val op = next()
          val right = nonTerminal()
          left = binaryOp(left, op, right)
@@ -1053,6 +1063,17 @@ class Parser(tokens: Seq[Token], source: Source) {
   def TypeTuple(tps: Many[Type]): Type =
     TypeRef(IdRef(List("effekt"), s"Tuple${tps.size}", tps.span.synthesized), tps, tps.span.synthesized)
 
+  // Check that the current token is surrounded by whitespace. If not, soft fail.
+  private def checkBinaryOpWhitespace(): Unit = {
+    // position points to the operator token in the raw token array
+    val wsBefore = position     > 0             && isSpace(tokens(position - 1).kind)
+    val wsAfter  = position + 1 < tokens.length && isSpace(tokens(position + 1).kind)
+
+    if (!wsBefore || !wsAfter) {
+      softFail(s"Missing whitespace around binary operator", position, position)
+    }
+  }
+
   /**
    * This is a compound production for
    *  - member selection <EXPR>.<NAME>
@@ -1101,7 +1122,7 @@ class Parser(tokens: Seq[Token], source: Source) {
   // argument lists cannot follow a linebreak:
   //   foo      ==    foo;
   //   ()             ()
-  def isArguments: Boolean = lookbehind(1).kind != Newline && (peek(`(`) || peek(`[`) || peek(`{`))
+  def isArguments: Boolean = !sawNewlineLast && (peek(`(`) || peek(`[`) || peek(`{`))
   def arguments(): (List[ValueType], List[ValueArg], List[Term]) =
     if (!isArguments) fail("at least one argument section (types, values, or blocks)", peek.kind)
     (maybeTypeArgs().unspan, maybeValueArgs(), maybeBlockArgs())

examples/benchmarks/are_we_fast_yet/nbody.effekt

@@ -92,9 +92,9 @@ def run(n: Int) = {
     def offsetMomentum(body: Body, px: Double, py: Double, pz: Double) =
       Body(
         body.x, body.y, body.z,
-        0.0 -(px / SOLAR_MASS),
-        0.0 -(py / SOLAR_MASS),
-        0.0 -(pz / SOLAR_MASS),
+        0.0 - (px / SOLAR_MASS),
+        0.0 - (py / SOLAR_MASS),
+        0.0 - (pz / SOLAR_MASS),
         body.mass)
 
     bodies.unsafeSet(0, sun.offsetMomentum(px, py, pz))

examples/llvm/gids.effekt

@@ -22,6 +22,6 @@ def main() = region r {
         println(g())
         println(h())
     } with GID {
-        def gid() = { n = n+1; resume(n) }
+        def gid() = { n = n + 1; resume(n) }
     }
 }

examples/llvm/localfunctionasargument.effekt

@@ -3,10 +3,10 @@ def hof(x: Int){ f: Int => Int }: Int = {
 }
 
 def foo(a: Int) = {
-    def bar(b: Int): Int = b*a
+    def bar(b: Int): Int = b * a
     hof(5){bar}
 }
 
-def main() = { 
+def main() = {
     println(foo(2))
 }

examples/llvm/polymorphic_failtooption.effekt

@@ -22,7 +22,7 @@ def safeDiv(x: Int, y: Int): Int / Fail = {
     if (y == 0){
         do Fail(); 0
     } else {
-        x/y
+        x / y
     }
 }
 

examples/llvm/polymorphism_data.effekt

@@ -15,24 +15,24 @@ def map[A,B](l: List[A]){ f: A => B }: List[B] = {
 
 def show(i: Int): String = {
   def impl_digit(i: Int): String = {
-    if(i==0){"0"} else
-    if(i==1){"1"} else
-    if(i==2){"2"} else
-    if(i==3){"3"} else
-    if(i==4){"4"} else
-    if(i==5){"5"} else
-    if(i==6){"6"} else
-    if(i==7){"7"} else
-    if(i==8){"8"} else
-    if(i==9){"9"} else
+    if(i == 0){"0"} else
+    if(i == 1){"1"} else
+    if(i == 2){"2"} else
+    if(i == 3){"3"} else
+    if(i == 4){"4"} else
+    if(i == 5){"5"} else
+    if(i == 6){"6"} else
+    if(i == 7){"7"} else
+    if(i == 8){"8"} else
+    if(i == 9){"9"} else
     "?"
   }
   def impl(i: Int): String = {
-    if(i<10){impl_digit(i)}else{
-      impl(i/10) ++ impl_digit(mod(i,10))
+    if(i < 10){impl_digit(i)}else{
+      impl(i / 10) ++ impl_digit(mod(i, 10))
     }
   }
-  if(i==0) {
+  if(i == 0) {
     "0"
   } else {
     impl(i)

examples/neg/parsing/bin_op_spaces.check

@@ -0,0 +1,12 @@
+[error] examples/neg/parsing/bin_op_spaces.effekt:10:13: Missing whitespace around binary operator
+  val _ = 1 +2
+            ^
+[error] examples/neg/parsing/bin_op_spaces.effekt:11:12: Missing whitespace around binary operator
+  val _ = 1+ 2
+           ^
+[error] examples/neg/parsing/bin_op_spaces.effekt:12:12: Missing whitespace around binary operator
+  val _ = 1+2
+           ^
+[error] examples/neg/parsing/bin_op_spaces.effekt:16:1: Expected expression but got }
+}
+^

examples/neg/parsing/bin_op_spaces.effekt

@@ -0,0 +1,16 @@
+def main() = {
+  // OK
+  val _ = 1 + 2
+  val _ = 1  + 2
+  val _ = 1 +  2
+  val _ = 1 +
+    2
+
+  // FAIL
+  val _ = 1 +2
+  val _ = 1+ 2
+  val _ = 1+2
+
+  // NOTE: this is here on purpose to showcase error recovery!
+  (
+}

examples/pos/doubles.effekt

@@ -2,7 +2,7 @@ def main() = {
     println(1.2 + 2.0 * 3.0 / 4.0 + 1.4);
     println(toInt(1.5));
     println(round(1.5));
-    println(toDouble(2)+ 0.1);
+    println(toDouble(2) + 0.1);
     println(floor(1.7));
     println(ceil(2.1));
 }

examples/pos/issue661.effekt

@@ -15,7 +15,7 @@ def my_map[T, U] {action: () => Unit / {Iter[T]}} {f: T => U}: Unit / {Iter[U]}
   }
 
 def main() =
-  try my_map { iter([1, 2, 3]) } { x => x+1 }
+  try my_map { iter([1, 2, 3]) } { x => x + 1 }
   with Iter[Int] {
     def yield(x) = { println(x); resume(()) }
   }

libraries/common/random.effekt

@@ -170,7 +170,7 @@ def main() = {
       val b = randomInt(0, 9)
       val c = randomInt(0, 9)
       println(a.show ++ " " ++ b.show ++ " " ++ c.show)
-      println(a*100 + b*10 + c)
+      println(a * 100 + b * 10 + c)
     }
   }