From 9da00babc9b539de5104986059de63abc7596053 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nibaldo=20Gonz=C3=A1lez?= <nibgonz@gmail.com>
Date: Mon, 25 Jun 2018 17:38:35 -0400
Subject: [PATCH 01/10] Add unicode escapes, fix escapes & integer suffixes,
 and allow non-ASCII identifiers

---
 grammars/rust.cson    | 55 ++++++++++++++++++++++++++++++-------------
 spec/rust-spec.coffee | 12 +++++-----
 2 files changed, 44 insertions(+), 23 deletions(-)

diff --git a/grammars/rust.cson b/grammars/rust.cson
index 2b70a7b..a682c65 100644
--- a/grammars/rust.cson
+++ b/grammars/rust.cson
@@ -39,12 +39,26 @@
   }
   'escaped_character': {
     'name': 'constant.character.escape.rust'
-    'match': '\\\\(x[0-9A-Fa-f]{2}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.)'
+    'match': '\\\\([trn0\'\"\\\\]|x[0-9A-Fa-f]{2}|$)'
+  }
+  'unicode_escaped_character': {
+    'name': 'constant.character.escape.unicode.rust'
+    'match': '\\\\u\\{([0-9A-Fa-f]_*){1,6}\\}'
   }
   'string_literal': {
     'comment': 'Double-quote string literal'
     'name': 'string.quoted.double.rust'
-    'begin': 'b?"'
+    'begin': '"'
+    'end': '"'
+    'patterns': [
+      { 'include': '#escaped_character' }
+      { 'include': '#unicode_escaped_character' }
+    ]
+  }
+  'byte_string_literal': {
+    'comment': 'Double-quote byte string literal'
+    'name': 'string.byte.quoted.double.rust'
+    'begin': 'b"'
     'end': '"'
     'patterns': [
       { 'include': '#escaped_character' }
@@ -59,7 +73,7 @@
   'sigils': {
     'comment': 'Sigil'
     'name': 'keyword.operator.sigil.rust'
-    'match': '[&*](?=[a-zA-Z0-9_\\(\\[\\|\\"]+)'
+    'match': '[&*](?=[a-zA-Z0-9_\\(\\[\\|\\"\\x80-\\xFF]+)'
   }
   'self': {
     'comment': 'Self variable'
@@ -99,14 +113,14 @@
   'lifetime': {
     'comment': 'Named lifetime'
     'name': 'storage.modifier.lifetime.rust'
-    'match': '\'([a-zA-Z_][a-zA-Z0-9_]*)\\b'
+    'match': '\'([a-zA-Z_\\x80-\\xFF][a-zA-Z0-9_\\x80-\\xFF]*)\\b'
     'captures': {
       '1': { 'name': 'entity.name.lifetime.rust' }
     }
   }
   'ref_lifetime': {
     'comment': 'Reference with named lifetime'
-    'match': '&(\'([a-zA-Z_][a-zA-Z0-9_]*))\\b'
+    'match': '&(\'([a-zA-Z_\\x80-\\xFF][a-zA-Z0-9_\\x80-\\xFF]*))\\b'
     'captures': {
       '1': { 'name': 'storage.modifier.lifetime.rust' }
       '2': { 'name': 'entity.name.lifetime.rust' }
@@ -145,7 +159,7 @@
   'type': {
     'comment': 'A type'
     'name': 'entity.name.type.rust'
-    'match': '\\b([A-Za-z][_A-Za-z0-9]*|_[_A-Za-z0-9]+)\\b'
+    'match': '\\b([A-Za-z\\x80-\\xFF][_A-Za-z0-9\\x80-\\xFF]*|_[_A-Za-z0-9\\x80-\\xFF]+)\\b'
   }
   'type_params': {
     'comment': 'Type parameters'
@@ -211,6 +225,7 @@
     'end': '\\]'
     'patterns': [
       { 'include': '#string_literal' }
+      { 'include': '#byte_string_literal' }
       { 'include': '#block_doc_comment' }
       { 'include': '#block_comment' }
       { 'include': '#line_doc_comment' }
@@ -221,9 +236,15 @@
   {
     'comment': 'Single-quote string literal (character)'
     'name': 'string.quoted.single.rust'
-    'match': 'b?\'([^\'\\\\]|\\\\(x[0-9A-Fa-f]{2}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.))\''
+    'match': '\'([^\'\\\\]|\\\\([trn0\'\"\\\\]|x[0-9A-Fa-f]{2}|u\\{([0-9A-Fa-f]_*){1,6}\\}))\''
+  }
+  {
+    'comment': 'Single-quote byte string literal (character)'
+    'name': 'string.byte.quoted.single.rust'
+    'match': 'b\'([^\'\\\\]|\\\\([trn0\'\"\\\\]|x[0-9A-Fa-f]{2}))\''
   }
   { 'include': '#string_literal' }
+  { 'include': '#byte_string_literal' }
   { 'include': '#raw_string_literal' }
   # Numbers
   {
@@ -244,22 +265,22 @@
   {
     'comment': 'Integer literal (decimal)'
     'name': 'constant.numeric.integer.decimal.rust'
-    'match': '\\b[0-9][0-9_]*([ui](8|16|32|64|128|s|size))?\\b'
+    'match': '\\b[0-9][0-9_]*([ui](8|16|32|64|128|size))?\\b'
   }
   {
     'comment': 'Integer literal (hexadecimal)'
     'name': 'constant.numeric.integer.hexadecimal.rust'
-    'match': '\\b0x[a-fA-F0-9_]+([ui](8|16|32|64|128|s|size))?\\b'
+    'match': '\\b0x[a-fA-F0-9_]+([ui](8|16|32|64|128|size))?\\b'
   }
   {
     'comment': 'Integer literal (octal)'
     'name': 'constant.numeric.integer.octal.rust'
-    'match': '\\b0o[0-7_]+([ui](8|16|32|64|128|s|size))?\\b'
+    'match': '\\b0o[0-7_]+([ui](8|16|32|64|128|size))?\\b'
   }
   {
     'comment': 'Integer literal (binary)'
     'name': 'constant.numeric.integer.binary.rust'
-    'match': '\\b0b[01_]+([ui](8|16|32|64|128|s|size))?\\b'
+    'match': '\\b0b[01_]+([ui](8|16|32|64|128|size))?\\b'
   }
   # Language
   {
@@ -354,21 +375,21 @@
   # Function and macro calls
   {
     'comment': 'Invokation of a macro'
-    'match': '\\b([a-zA-Z_][a-zA-Z0-9_]*\\!)\\s*[({\\[]'
+    'match': '\\b([a-zA-Z_\\x80-\\xFF][a-zA-Z0-9_\\x80-\\xFF]*\\!)\\s*[({\\[]'
     'captures': {
       '1': { 'name': 'entity.name.function.macro.rust' }
     }
   }
   {
     'comment': 'Function call'
-    'match': '\\b([A-Za-z][A-Za-z0-9_]*|_[A-Za-z0-9_]+)\\s*\\('
+    'match': '\\b([A-Za-z\\x80-\\xFF][A-Za-z0-9_\\x80-\\xFF]*|_[A-Za-z0-9_\\x80-\\xFF]+)\\s*\\('
     'captures': {
       '1': { 'name': 'entity.name.function.rust' }
     }
   }
   {
     'comment': 'Function call with type parameters'
-    'begin': '\\b([A-Za-z][A-Za-z0-9_]*|_[A-Za-z0-9_]+)\\s*(::)(?=\\s*<.*>\\s*\\()'
+    'begin': '\\b([A-Za-z\\x80-\\xFF][A-Za-z0-9_\\x80-\\xFF]*|_[A-Za-z0-9_\\x80-\\xFF]+)\\s*(::)(?=\\s*<.*>\\s*\\()'
     'end': '\\('
     'captures': {
       '1': { 'name': 'entity.name.function.rust' }
@@ -381,7 +402,7 @@
   # Function definition
   {
     'comment': 'Function definition'
-    'begin': '\\b(fn)\\s+([A-Za-z][A-Za-z0-9_]*|_[A-Za-z0-9_]+)'
+    'begin': '\\b(fn)\\s+([A-Za-z\\x80-\\xFF][A-Za-z0-9_\\x80-\\xFF]*|_[A-Za-z0-9_\\x80-\\xFF]+)'
     'end': '[\\{;]'
     'beginCaptures': {
       '1': { 'name': 'keyword.other.fn.rust' }
@@ -413,7 +434,7 @@
   # Type declaration
   {
     'comment': 'Type declaration'
-    'begin': '\\b(enum|struct|trait|union)\\s+([a-zA-Z_][a-zA-Z0-9_]*)'
+    'begin': '\\b(enum|struct|trait|union)\\s+([a-zA-Z_\\x80-\\xFF][a-zA-Z0-9_\\x80-\\xFF]*)'
     'end': '[\\{\\(;]'
     'beginCaptures': {
       '1': { 'name': 'storage.type.rust' }
@@ -433,7 +454,7 @@
   # Type alias
   {
     'comment': 'Type alias'
-    'begin': '\\b(type)\\s+([a-zA-Z_][a-zA-Z0-9_]*)'
+    'begin': '\\b(type)\\s+([a-zA-Z_\\x80-\\xFF][a-zA-Z0-9_\\x80-\\xFF]*)'
     'end': ';'
     'beginCaptures': {
       '1': { 'name': 'storage.type.rust' }
diff --git a/spec/rust-spec.coffee b/spec/rust-spec.coffee
index 0b736bf..b88ce58 100644
--- a/spec/rust-spec.coffee
+++ b/spec/rust-spec.coffee
@@ -103,11 +103,11 @@ describe 'Rust grammar', ->
     {tokens} = grammar.tokenizeLine('text "string\\nwith\\x20escaped\\"characters" text')
     expect(tokens[0]).toEqual value: 'text ', scopes: ['source.rust']
     expect(tokens[2]).toEqual value: 'string', scopes: ['source.rust', 'string.quoted.double.rust']
-    expect(tokens[3]).toEqual value: '\\n', scopes: ['source.rust', 'string.quoted.double.rust', 'constant.character.escape.rust']
+    expect(tokens[3]).toEqual value: '\\n', scopes: ['source.rust', 'string.quoted.double.rust', 'constant.character.escape.rust', 'constant.character.escape.unicode.rust']
     expect(tokens[4]).toEqual value: 'with', scopes: ['source.rust', 'string.quoted.double.rust']
-    expect(tokens[5]).toEqual value: '\\x20', scopes: ['source.rust', 'string.quoted.double.rust', 'constant.character.escape.rust']
+    expect(tokens[5]).toEqual value: '\\x20', scopes: ['source.rust', 'string.quoted.double.rust', 'constant.character.escape.rust', 'constant.character.escape.unicode.rust']
     expect(tokens[6]).toEqual value: 'escaped', scopes: ['source.rust', 'string.quoted.double.rust']
-    expect(tokens[7]).toEqual value: '\\"', scopes: ['source.rust', 'string.quoted.double.rust', 'constant.character.escape.rust']
+    expect(tokens[7]).toEqual value: '\\"', scopes: ['source.rust', 'string.quoted.double.rust', 'constant.character.escape.rust', 'constant.character.escape.unicode.rust']
     expect(tokens[8]).toEqual value: 'characters', scopes: ['source.rust', 'string.quoted.double.rust']
     expect(tokens[10]).toEqual value: ' text', scopes: ['source.rust']
 
@@ -139,7 +139,7 @@ describe 'Rust grammar', ->
   it 'tokenizes byte strings', ->
     {tokens} = grammar.tokenizeLine('text b"This is a bytestring" text')
     expect(tokens[0]).toEqual value: 'text ', scopes: ['source.rust']
-    expect(tokens[2]).toEqual value: 'This is a bytestring', scopes: ['source.rust', 'string.quoted.double.rust']
+    expect(tokens[2]).toEqual value: 'This is a bytestring', scopes: ['source.rust', 'string.byte.quoted.double.rust']
     expect(tokens[4]).toEqual value: ' text', scopes: ['source.rust']
 
   it 'tokenizes raw byte strings', ->
@@ -170,13 +170,13 @@ describe 'Rust grammar', ->
   it 'tokenizes bytes character', ->
     {tokens} = grammar.tokenizeLine('text b\'b\' text')
     expect(tokens[0]).toEqual value: 'text ', scopes: ['source.rust']
-    expect(tokens[1]).toEqual value: 'b\'b\'', scopes: ['source.rust', 'string.quoted.single.rust']
+    expect(tokens[1]).toEqual value: 'b\'b\'', scopes: ['source.rust', 'string.byte.quoted.single.rust']
     expect(tokens[2]).toEqual value: ' text', scopes: ['source.rust']
 
   it 'tokenizes escaped bytes characters', ->
     {tokens} = grammar.tokenizeLine('text b\'\\x20\' text')
     expect(tokens[0]).toEqual value: 'text ', scopes: ['source.rust']
-    expect(tokens[1]).toEqual value: 'b\'\\x20\'', scopes: ['source.rust', 'string.quoted.single.rust']
+    expect(tokens[1]).toEqual value: 'b\'\\x20\'', scopes: ['source.rust', 'string.byte.quoted.single.rust']
     expect(tokens[2]).toEqual value: ' text', scopes: ['source.rust']
 
   #

From 1937f97564efc888cdcb534308498cdb26f6d567 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nibaldo=20Gonz=C3=A1lez?= <nibgonz@gmail.com>
Date: Mon, 25 Jun 2018 17:58:58 -0400
Subject: [PATCH 02/10] Add missing sigils

---
 grammars/rust.cson | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/grammars/rust.cson b/grammars/rust.cson
index a682c65..c40ed41 100644
--- a/grammars/rust.cson
+++ b/grammars/rust.cson
@@ -73,7 +73,7 @@
   'sigils': {
     'comment': 'Sigil'
     'name': 'keyword.operator.sigil.rust'
-    'match': '[&*](?=[a-zA-Z0-9_\\(\\[\\|\\"\\x80-\\xFF]+)'
+    'match': '[&*~@](?=[a-zA-Z0-9_\\(\\[\\|\\"\\x80-\\xFF]+)'
   }
   'self': {
     'comment': 'Self variable'

From 3644852c149ce6571025e6a04d3e0d760d4baf4c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nibaldo=20Gonz=C3=A1lez?= <nibgonz@gmail.com>
Date: Mon, 25 Jun 2018 18:00:44 -0400
Subject: [PATCH 03/10] Add missing sigils

---
 spec/rust-spec.coffee | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/spec/rust-spec.coffee b/spec/rust-spec.coffee
index b88ce58..010cd83 100644
--- a/spec/rust-spec.coffee
+++ b/spec/rust-spec.coffee
@@ -327,9 +327,11 @@ describe 'Rust grammar', ->
     expect(tokens[2]).toEqual value: ' text', scopes: ['source.rust']
 
   it 'tokenizes sigils', ->
-    {tokens} = grammar.tokenizeLine('*var &var')
+    {tokens} = grammar.tokenizeLine('*var &var ~var @var')
     expect(tokens[0]).toEqual value: '*', scopes: ['source.rust', 'keyword.operator.sigil.rust']
     expect(tokens[2]).toEqual value: '&', scopes: ['source.rust', 'keyword.operator.sigil.rust']
+    expect(tokens[3]).toEqual value: '~', scopes: ['source.rust', 'keyword.operator.sigil.rust']
+    expect(tokens[4]).toEqual value: '@', scopes: ['source.rust', 'keyword.operator.sigil.rust']
 
   #
   # Core

From adce1fe782b2b9854ab1dbfee3ae0c57a341da52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nibaldo=20Gonz=C3=A1lez?= <nibgonz@gmail.com>
Date: Mon, 25 Jun 2018 18:35:09 -0400
Subject: [PATCH 04/10] Fix spec

---
 spec/rust-spec.coffee | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/spec/rust-spec.coffee b/spec/rust-spec.coffee
index 010cd83..14c9a11 100644
--- a/spec/rust-spec.coffee
+++ b/spec/rust-spec.coffee
@@ -103,11 +103,11 @@ describe 'Rust grammar', ->
     {tokens} = grammar.tokenizeLine('text "string\\nwith\\x20escaped\\"characters" text')
     expect(tokens[0]).toEqual value: 'text ', scopes: ['source.rust']
     expect(tokens[2]).toEqual value: 'string', scopes: ['source.rust', 'string.quoted.double.rust']
-    expect(tokens[3]).toEqual value: '\\n', scopes: ['source.rust', 'string.quoted.double.rust', 'constant.character.escape.rust', 'constant.character.escape.unicode.rust']
+    expect(tokens[3]).toEqual value: '\\n', scopes: ['source.rust', 'string.quoted.double.rust', 'constant.character.escape.rust']
     expect(tokens[4]).toEqual value: 'with', scopes: ['source.rust', 'string.quoted.double.rust']
-    expect(tokens[5]).toEqual value: '\\x20', scopes: ['source.rust', 'string.quoted.double.rust', 'constant.character.escape.rust', 'constant.character.escape.unicode.rust']
+    expect(tokens[5]).toEqual value: '\\x20', scopes: ['source.rust', 'string.quoted.double.rust', 'constant.character.escape.rust']
     expect(tokens[6]).toEqual value: 'escaped', scopes: ['source.rust', 'string.quoted.double.rust']
-    expect(tokens[7]).toEqual value: '\\"', scopes: ['source.rust', 'string.quoted.double.rust', 'constant.character.escape.rust', 'constant.character.escape.unicode.rust']
+    expect(tokens[7]).toEqual value: '\\"', scopes: ['source.rust', 'string.quoted.double.rust', 'constant.character.escape.rust']
     expect(tokens[8]).toEqual value: 'characters', scopes: ['source.rust', 'string.quoted.double.rust']
     expect(tokens[10]).toEqual value: ' text', scopes: ['source.rust']
 
@@ -330,8 +330,8 @@ describe 'Rust grammar', ->
     {tokens} = grammar.tokenizeLine('*var &var ~var @var')
     expect(tokens[0]).toEqual value: '*', scopes: ['source.rust', 'keyword.operator.sigil.rust']
     expect(tokens[2]).toEqual value: '&', scopes: ['source.rust', 'keyword.operator.sigil.rust']
-    expect(tokens[3]).toEqual value: '~', scopes: ['source.rust', 'keyword.operator.sigil.rust']
-    expect(tokens[4]).toEqual value: '@', scopes: ['source.rust', 'keyword.operator.sigil.rust']
+    expect(tokens[4]).toEqual value: '~', scopes: ['source.rust', 'keyword.operator.sigil.rust']
+    expect(tokens[6]).toEqual value: '@', scopes: ['source.rust', 'keyword.operator.sigil.rust']
 
   #
   # Core

From c29da07e9bfab867e5767d2eaf340b54417b51c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nibaldo=20Gonz=C3=A1lez?= <nibgonz@gmail.com>
Date: Tue, 11 Dec 2018 02:21:02 -0300
Subject: [PATCH 05/10] Remove deprecated sigils & add test

---
 grammars/rust.cson    |  2 +-
 spec/rust-spec.coffee | 13 ++++++++++---
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/grammars/rust.cson b/grammars/rust.cson
index c40ed41..a682c65 100644
--- a/grammars/rust.cson
+++ b/grammars/rust.cson
@@ -73,7 +73,7 @@
   'sigils': {
     'comment': 'Sigil'
     'name': 'keyword.operator.sigil.rust'
-    'match': '[&*~@](?=[a-zA-Z0-9_\\(\\[\\|\\"\\x80-\\xFF]+)'
+    'match': '[&*](?=[a-zA-Z0-9_\\(\\[\\|\\"\\x80-\\xFF]+)'
   }
   'self': {
     'comment': 'Self variable'
diff --git a/spec/rust-spec.coffee b/spec/rust-spec.coffee
index 14c9a11..fa55691 100644
--- a/spec/rust-spec.coffee
+++ b/spec/rust-spec.coffee
@@ -327,11 +327,9 @@ describe 'Rust grammar', ->
     expect(tokens[2]).toEqual value: ' text', scopes: ['source.rust']
 
   it 'tokenizes sigils', ->
-    {tokens} = grammar.tokenizeLine('*var &var ~var @var')
+    {tokens} = grammar.tokenizeLine('*var &var')
     expect(tokens[0]).toEqual value: '*', scopes: ['source.rust', 'keyword.operator.sigil.rust']
     expect(tokens[2]).toEqual value: '&', scopes: ['source.rust', 'keyword.operator.sigil.rust']
-    expect(tokens[4]).toEqual value: '~', scopes: ['source.rust', 'keyword.operator.sigil.rust']
-    expect(tokens[6]).toEqual value: '@', scopes: ['source.rust', 'keyword.operator.sigil.rust']
 
   #
   # Core
@@ -765,3 +763,12 @@ describe 'Rust grammar', ->
     expect(tokens[0][5]).toEqual value: 'a', scopes: ['source.rust', 'meta.type_params.rust', 'storage.modifier.lifetime.rust', 'entity.name.lifetime.rust']
     expect(tokens[0][13]).toEqual value: "'", scopes: ['source.rust', 'meta.type_params.rust', 'storage.modifier.lifetime.rust']
     expect(tokens[0][14]).toEqual value: 'a', scopes: ['source.rust', 'meta.type_params.rust', 'storage.modifier.lifetime.rust', 'entity.name.lifetime.rust']
+
+  it 'tokenizes non-ASCII identifiers', ->
+    {tokens} = grammar.tokenizeLine("*Ωµó\n'hellóñαωΑΩµo\nhellóñαωΑΩµo!();\nhellóñαωΑΩµo();\nhellóñαωΑΩµo::Hello();\ntype hellóñαωΑΩµo;")
+    expect(tokens[0][0]).toEqual value: '*', scopes: ['source.rust', 'keyword.operator.sigil.rust']
+    expect(tokens[1][1]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'entity.name.lifetime.rust']
+    expect(tokens[2][0]).toEqual value: 'hellóñαωΑΩµo!', scopes: ['source.rust', 'entity.name.function.macro.rust']
+    expect(tokens[3][0]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'entity.name.function.rust']
+    expect(tokens[4][0]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'entity.name.function.rust']
+    expect(tokens[4][2]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'entity.name.type.rust']

From 7b42399e0a04ad24b866fa0358b496cd8939b270 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nibaldo=20Gonz=C3=A1lez?= <nibgonz@gmail.com>
Date: Tue, 11 Dec 2018 02:48:20 -0300
Subject: [PATCH 06/10] Fix spec

---
 spec/rust-spec.coffee | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/spec/rust-spec.coffee b/spec/rust-spec.coffee
index 434034e..6691b84 100644
--- a/spec/rust-spec.coffee
+++ b/spec/rust-spec.coffee
@@ -764,14 +764,18 @@ describe 'Rust grammar', ->
     expect(tokens[0][13]).toEqual value: "'", scopes: ['source.rust', 'meta.type_params.rust', 'storage.modifier.lifetime.rust']
     expect(tokens[0][14]).toEqual value: 'a', scopes: ['source.rust', 'meta.type_params.rust', 'storage.modifier.lifetime.rust', 'entity.name.lifetime.rust']
 
+  #
+  # non-ASCII identifiers
+  #
+
   it 'tokenizes non-ASCII identifiers', ->
-    {tokens} = grammar.tokenizeLine("*Ωµó\n'hellóñαωΑΩµo\nhellóñαωΑΩµo!();\nhellóñαωΑΩµo();\nhellóñαωΑΩµo::Hello();\ntype hellóñαωΑΩµo;")
+    tokens = grammar.tokenizeLine('''*Ωµó\n'hellóñαωΑΩµo\nhellóñαωΑΩµo!();\nhellóñαωΑΩµo();\nhellóñαωΑΩµo::Hello();\ntype hellóñαωΑΩµo;''')
     expect(tokens[0][0]).toEqual value: '*', scopes: ['source.rust', 'keyword.operator.sigil.rust']
     expect(tokens[1][1]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'entity.name.lifetime.rust']
     expect(tokens[2][0]).toEqual value: 'hellóñαωΑΩµo!', scopes: ['source.rust', 'entity.name.function.macro.rust']
     expect(tokens[3][0]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'entity.name.function.rust']
     expect(tokens[4][0]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'entity.name.function.rust']
-    expect(tokens[4][2]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'entity.name.type.rust']
+    expect(tokens[5][1]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'entity.name.type.rust']
 
   #
   # impl type modifier

From 20d00950a285c50bf65be0159c170c153fe9c9da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nibaldo=20Gonz=C3=A1lez?= <nibgonz@gmail.com>
Date: Tue, 11 Dec 2018 02:53:14 -0300
Subject: [PATCH 07/10] Fix spec

---
 spec/rust-spec.coffee | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/rust-spec.coffee b/spec/rust-spec.coffee
index 6691b84..82a2073 100644
--- a/spec/rust-spec.coffee
+++ b/spec/rust-spec.coffee
@@ -769,7 +769,7 @@ describe 'Rust grammar', ->
   #
 
   it 'tokenizes non-ASCII identifiers', ->
-    tokens = grammar.tokenizeLine('''*Ωµó\n'hellóñαωΑΩµo\nhellóñαωΑΩµo!();\nhellóñαωΑΩµo();\nhellóñαωΑΩµo::Hello();\ntype hellóñαωΑΩµo;''')
+    tokens = grammar.tokenizeLine("*Ωµó\n'hellóñαωΑΩµo\nhellóñαωΑΩµo!();\nhellóñαωΑΩµo();\nhellóñαωΑΩµo::Hello();\ntype hellóñαωΑΩµo;")
     expect(tokens[0][0]).toEqual value: '*', scopes: ['source.rust', 'keyword.operator.sigil.rust']
     expect(tokens[1][1]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'entity.name.lifetime.rust']
     expect(tokens[2][0]).toEqual value: 'hellóñαωΑΩµo!', scopes: ['source.rust', 'entity.name.function.macro.rust']

From 49db71bb8bea56987d216e70a82841e329c3ce34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nibaldo=20Gonz=C3=A1lez?= <nibgonz@gmail.com>
Date: Tue, 11 Dec 2018 02:54:13 -0300
Subject: [PATCH 08/10] Fix spec

---
 spec/rust-spec.coffee | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/rust-spec.coffee b/spec/rust-spec.coffee
index 82a2073..4261287 100644
--- a/spec/rust-spec.coffee
+++ b/spec/rust-spec.coffee
@@ -769,7 +769,7 @@ describe 'Rust grammar', ->
   #
 
   it 'tokenizes non-ASCII identifiers', ->
-    tokens = grammar.tokenizeLine("*Ωµó\n'hellóñαωΑΩµo\nhellóñαωΑΩµo!();\nhellóñαωΑΩµo();\nhellóñαωΑΩµo::Hello();\ntype hellóñαωΑΩµo;")
+    tokens = grammar.tokenizeLines("*Ωµó\n'hellóñαωΑΩµo\nhellóñαωΑΩµo!();\nhellóñαωΑΩµo();\nhellóñαωΑΩµo::Hello();\ntype hellóñαωΑΩµo;")
     expect(tokens[0][0]).toEqual value: '*', scopes: ['source.rust', 'keyword.operator.sigil.rust']
     expect(tokens[1][1]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'entity.name.lifetime.rust']
     expect(tokens[2][0]).toEqual value: 'hellóñαωΑΩµo!', scopes: ['source.rust', 'entity.name.function.macro.rust']

From d224887fd59d24b4466d60330686cbed9e487b6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nibaldo=20Gonz=C3=A1lez?= <nibgonz@gmail.com>
Date: Tue, 11 Dec 2018 03:13:47 -0300
Subject: [PATCH 09/10] Try to fix errors in previous commits

---
 spec/rust-spec.coffee | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/spec/rust-spec.coffee b/spec/rust-spec.coffee
index 4261287..b100202 100644
--- a/spec/rust-spec.coffee
+++ b/spec/rust-spec.coffee
@@ -769,13 +769,12 @@ describe 'Rust grammar', ->
   #
 
   it 'tokenizes non-ASCII identifiers', ->
-    tokens = grammar.tokenizeLines("*Ωµó\n'hellóñαωΑΩµo\nhellóñαωΑΩµo!();\nhellóñαωΑΩµo();\nhellóñαωΑΩµo::Hello();\ntype hellóñαωΑΩµo;")
+    tokens = grammar.tokenizeLines("*Ωµó\n'hellóñαωΑΩµo\nhellóñαωΑΩµo!();\nhellóñαωΑΩµo();\ntype hellóñαωΑΩµo;")
     expect(tokens[0][0]).toEqual value: '*', scopes: ['source.rust', 'keyword.operator.sigil.rust']
-    expect(tokens[1][1]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'entity.name.lifetime.rust']
+    expect(tokens[1][1]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'storage.modifier.lifetime.rust']
     expect(tokens[2][0]).toEqual value: 'hellóñαωΑΩµo!', scopes: ['source.rust', 'entity.name.function.macro.rust']
     expect(tokens[3][0]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'entity.name.function.rust']
-    expect(tokens[4][0]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'entity.name.function.rust']
-    expect(tokens[5][1]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'entity.name.type.rust']
+    expect(tokens[4][2]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'entity.name.type.rust']
 
   #
   # impl type modifier

From a5c791f74af668170af2ce8aaa787a541e905d79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nibaldo=20Gonz=C3=A1lez?= <nibgonz@gmail.com>
Date: Tue, 11 Dec 2018 03:20:45 -0300
Subject: [PATCH 10/10] Fixes errors in previous commits

---
 spec/rust-spec.coffee | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/rust-spec.coffee b/spec/rust-spec.coffee
index b100202..a590dd3 100644
--- a/spec/rust-spec.coffee
+++ b/spec/rust-spec.coffee
@@ -771,7 +771,7 @@ describe 'Rust grammar', ->
   it 'tokenizes non-ASCII identifiers', ->
     tokens = grammar.tokenizeLines("*Ωµó\n'hellóñαωΑΩµo\nhellóñαωΑΩµo!();\nhellóñαωΑΩµo();\ntype hellóñαωΑΩµo;")
     expect(tokens[0][0]).toEqual value: '*', scopes: ['source.rust', 'keyword.operator.sigil.rust']
-    expect(tokens[1][1]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'storage.modifier.lifetime.rust']
+    expect(tokens[1][1]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'storage.modifier.lifetime.rust', 'entity.name.lifetime.rust']
     expect(tokens[2][0]).toEqual value: 'hellóñαωΑΩµo!', scopes: ['source.rust', 'entity.name.function.macro.rust']
     expect(tokens[3][0]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'entity.name.function.rust']
     expect(tokens[4][2]).toEqual value: 'hellóñαωΑΩµo', scopes: ['source.rust', 'entity.name.type.rust']