Skip to content

Commit 0ca11b3

Browse files
johnedquinnjohnedquinn
committed
Adds non-reserved keywords
1 parent a279276 commit 0ca11b3

File tree

4 files changed

+267
-23
lines changed

4 files changed

+267
-23
lines changed

‎partiql-parser/src/lexer.rs

Lines changed: 70 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -510,6 +510,8 @@ pub enum Token<'input> {
510510
Ion(&'input str),
511511

512512
// Keywords
513+
#[regex("(?i:Acyclic)")]
514+
Acyclic,
513515
#[regex("(?i:All)")]
514516
All,
515517
#[regex("(?i:Asc)")]
@@ -522,6 +524,8 @@ pub enum Token<'input> {
522524
At,
523525
#[regex("(?i:Between)")]
524526
Between,
527+
#[regex("(?i:Both)")]
528+
Both,
525529
#[regex("(?i:By)")]
526530
By,
527531
#[regex("(?i:Case)")]
@@ -534,6 +538,8 @@ pub enum Token<'input> {
534538
Desc,
535539
#[regex("(?i:Distinct)")]
536540
Distinct,
541+
#[regex("(?i:Domain)")]
542+
Domain,
537543
#[regex("(?i:Else)")]
538544
Else,
539545
#[regex("(?i:End)")]
@@ -570,6 +576,8 @@ pub enum Token<'input> {
570576
Last,
571577
#[regex("(?i:Lateral)")]
572578
Lateral,
579+
#[regex("(?i:Leading)")]
580+
Leading,
573581
#[regex("(?i:Left)")]
574582
Left,
575583
#[regex("(?i:Like)")]
@@ -602,10 +610,14 @@ pub enum Token<'input> {
602610
Pivot,
603611
#[regex("(?i:Preserve)")]
604612
Preserve,
613+
#[regex("(?i:Public)")]
614+
Public,
605615
#[regex("(?i:Right)")]
606616
Right,
607617
#[regex("(?i:Select)")]
608618
Select,
619+
#[regex("(?i:Simple)")]
620+
Simple,
609621
#[regex("(?i:Table)")]
610622
Table,
611623
#[regex("(?i:Time)")]
@@ -614,12 +626,18 @@ pub enum Token<'input> {
614626
Timestamp,
615627
#[regex("(?i:Then)")]
616628
Then,
629+
#[regex("(?i:Trail)")]
630+
Trail,
631+
#[regex("(?i:Trailing)")]
632+
Trailing,
617633
#[regex("(?i:True)")]
618634
True,
619635
#[regex("(?i:Union)")]
620636
Union,
621637
#[regex("(?i:Unpivot)")]
622638
Unpivot,
639+
#[regex("(?i:User)")]
640+
User,
623641
#[regex("(?i:Using)")]
624642
Using,
625643
#[regex("(?i:Value)")]
@@ -642,17 +660,20 @@ impl<'input> Token<'input> {
642660
pub fn is_keyword(&self) -> bool {
643661
matches!(
644662
self,
645-
Token::All
663+
Token::Acyclic
664+
| Token::All
646665
| Token::Asc
647666
| Token::And
648667
| Token::As
649668
| Token::At
650669
| Token::Between
670+
| Token::Both
651671
| Token::By
652672
| Token::Cross
653673
| Token::Date
654674
| Token::Desc
655675
| Token::Distinct
676+
| Token::Domain
656677
| Token::Escape
657678
| Token::Except
658679
| Token::First
@@ -668,6 +689,7 @@ impl<'input> Token<'input> {
668689
| Token::Join
669690
| Token::Last
670691
| Token::Lateral
692+
| Token::Leading
671693
| Token::Left
672694
| Token::Like
673695
| Token::Limit
@@ -684,14 +706,19 @@ impl<'input> Token<'input> {
684706
| Token::Partial
685707
| Token::Pivot
686708
| Token::Preserve
709+
| Token::Public
687710
| Token::Right
688711
| Token::Select
712+
| Token::Simple
689713
| Token::Table
690714
| Token::Time
691715
| Token::Timestamp
692716
| Token::Then
717+
| Token::Trail
718+
| Token::Trailing
693719
| Token::Union
694720
| Token::Unpivot
721+
| Token::User
695722
| Token::Using
696723
| Token::Value
697724
| Token::Values
@@ -748,18 +775,21 @@ impl<'input> fmt::Display for Token<'input> {
748775
Token::EmbeddedIonQuote => write!(f, "<ION>"),
749776
Token::Ion(txt) => write!(f, "<{}:ION>", txt),
750777

751-
Token::All
778+
Token::Acyclic
779+
| Token::All
752780
| Token::Asc
753781
| Token::And
754782
| Token::As
755783
| Token::At
756784
| Token::Between
785+
| Token::Both
757786
| Token::By
758787
| Token::Case
759788
| Token::Cross
760789
| Token::Date
761790
| Token::Desc
762791
| Token::Distinct
792+
| Token::Domain
763793
| Token::Else
764794
| Token::End
765795
| Token::Escape
@@ -778,6 +808,7 @@ impl<'input> fmt::Display for Token<'input> {
778808
| Token::Join
779809
| Token::Last
780810
| Token::Lateral
811+
| Token::Leading
781812
| Token::Left
782813
| Token::Like
783814
| Token::Limit
@@ -794,15 +825,20 @@ impl<'input> fmt::Display for Token<'input> {
794825
| Token::Partial
795826
| Token::Pivot
796827
| Token::Preserve
828+
| Token::Public
797829
| Token::Right
798830
| Token::Select
831+
| Token::Simple
799832
| Token::Table
800833
| Token::Time
801834
| Token::Timestamp
802835
| Token::Then
836+
| Token::Trail
837+
| Token::Trailing
803838
| Token::True
804839
| Token::Union
805840
| Token::Unpivot
841+
| Token::User
806842
| Token::Using
807843
| Token::Value
808844
| Token::Values
@@ -1107,6 +1143,38 @@ mod tests {
11071143
Ok(())
11081144
}
11091145

1146+
#[test]
1147+
fn select_non_reserved_keywords() -> Result<(), ParseError<'static, BytePosition>> {
1148+
let query = "SELECT acyclic, BoTh, DOMAIN, SImple, Trail, TRailing, USER\nfrom @\"foo\"";
1149+
let mut offset_tracker = LineOffsetTracker::default();
1150+
let lexer = PartiqlLexer::new(query, &mut offset_tracker);
1151+
let toks: Vec<_> = lexer.collect::<Result<_, _>>()?;
1152+
1153+
assert_eq!(
1154+
vec![
1155+
Token::Select,
1156+
Token::Acyclic,
1157+
Token::Comma,
1158+
Token::Both,
1159+
Token::Comma,
1160+
Token::Domain,
1161+
Token::Comma,
1162+
Token::Simple,
1163+
Token::Comma,
1164+
Token::Trail,
1165+
Token::Comma,
1166+
Token::Trailing,
1167+
Token::Comma,
1168+
Token::User,
1169+
Token::From,
1170+
Token::QuotedAtIdentifier("foo"),
1171+
],
1172+
toks.into_iter().map(|(_s, t, _e)| t).collect::<Vec<_>>()
1173+
);
1174+
assert_eq!(offset_tracker.num_lines(), 2);
1175+
Ok(())
1176+
}
1177+
11101178
#[test]
11111179
fn select_comment_block() -> Result<(), ParseError<'static, BytePosition>> {
11121180
let query = "SELECT /*comment*/ g";

‎partiql-parser/src/parse/mod.rs

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -679,6 +679,55 @@ mod tests {
679679
}
680680
}
681681

682+
// PROOF OF CONCEPT: NON-RESERVED KEYWORDS
683+
mod non_reserved {
684+
use super::*;
685+
686+
#[test]
687+
fn projection_list_trim_spec() {
688+
parse!(r#"SELECT leading FROM t"#);
689+
parse!(r#"SELECT leading, a FROM t"#);
690+
parse!(r#"SELECT leading + trailing, b FROM t"#);
691+
parse!(r#"SELECT both + leading + trailing, a, b, c FROM t"#);
692+
}
693+
694+
#[test]
695+
fn from_source() {
696+
parse!(r#"SELECT leading, trailing, both FROM leading, trailing, both"#);
697+
}
698+
699+
#[test]
700+
fn with_trim() {
701+
parse!(
702+
r#"SELECT leading + trim(leading leading FROM ' hello world'), both FROM leading, trailing, both"#
703+
);
704+
}
705+
706+
#[test]
707+
fn with_order() {
708+
parse!(r#"SELECT order FROM t ORDER BY order + 5"#);
709+
parse!(r#"SELECT order FROM order ORDER BY order + 5"#);
710+
parse!(r#"SELECT ORDER FROM ORDER ORDER BY ORDER + 5"#);
711+
}
712+
713+
#[test]
714+
fn with_gpml() {
715+
parse!(r#"SELECT acyclic, trail, simple FROM t"#);
716+
parse!(r#"AcYcLiC"#);
717+
parse!(r#"TrAiL"#);
718+
parse!(r#"SiMpLe"#);
719+
}
720+
721+
//
722+
#[test]
723+
fn external_customer_request() {
724+
parse!(r#"SELECT user, puBlIC, DOMAIN FROM USER, pUbLIc, domain"#);
725+
parse!(r#"USER"#);
726+
parse!(r#"pUbLIC"#);
727+
parse!(r#"domain"#);
728+
}
729+
}
730+
682731
mod errors {
683732
use super::*;
684733
use crate::error::{LexError, UnexpectedToken, UnexpectedTokenData};

‎partiql-parser/src/parse/partiql.lalrpop

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1025,6 +1025,27 @@ VarRefExpr: ast::Expr = {
10251025
name: ast::SymbolPrimitive { value: ident.to_owned(), case: ast::CaseSensitivity::CaseSensitive },
10261026
qualifier: ast::ScopeQualifier::Unqualified
10271027
},lo..hi)),
1028+
<lo:@L> <ident:NonReservedKeyword> <hi:@R> => ast::Expr::VarRef(state.node(ast::VarRef {
1029+
name: ast::SymbolPrimitive { value: ident.to_owned(), case: ast::CaseSensitivity::CaseInsensitive },
1030+
qualifier: ast::ScopeQualifier::Unqualified
1031+
},lo..hi)),
1032+
}
1033+
1034+
// PROOF OF CONCEPT
1035+
// These are all the proposed non-reserved keywords (except ORDER)
1036+
// ORDER is only in this list to prove that even ORDER can be parsed as a non-reserved keyword.
1037+
#[inline]
1038+
NonReservedKeyword: &'static str = {
1039+
"ACYCLIC" => "ACYCLIC",
1040+
"BOTH" => "BOTH",
1041+
"DOMAIN" => "DOMAIN",
1042+
"LEADING" => "LEADING",
1043+
"ORDER" => "ORDER",
1044+
"PUBLIC" => "PUBLIC",
1045+
"SIMPLE" => "SIMPLE",
1046+
"TRAIL" => "TRAIL",
1047+
"TRAILING" => "TRAILING",
1048+
"USER" => "USER",
10281049
}
10291050

10301051
// ------------------------------------------------------------------------------ //
@@ -1267,18 +1288,21 @@ extern {
12671288
"Ion" => lexer::Token::Ion(<&'input str>),
12681289

12691290
// Keywords
1291+
"ACYCLIC" => lexer::Token::Acyclic,
12701292
"ALL" => lexer::Token::All,
12711293
"ASC" => lexer::Token::Asc,
12721294
"AND" => lexer::Token::And,
12731295
"AS" => lexer::Token::As,
12741296
"AT" => lexer::Token::At,
12751297
"BETWEEN" => lexer::Token::Between,
1298+
"BOTH" => lexer::Token::Both,
12761299
"BY" => lexer::Token::By,
12771300
"CASE" => lexer::Token::Case,
12781301
"CROSS" => lexer::Token::Cross,
12791302
"DATE" => lexer::Token::Date,
12801303
"DESC" => lexer::Token::Desc,
12811304
"DISTINCT" => lexer::Token::Distinct,
1305+
"DOMAIN" => lexer::Token::Domain,
12821306
"ELSE" => lexer::Token::Else,
12831307
"END" => lexer::Token::End,
12841308
"ESCAPE" => lexer::Token::Escape,
@@ -1297,6 +1321,7 @@ extern {
12971321
"JOIN" => lexer::Token::Join,
12981322
"LAST" => lexer::Token::Last,
12991323
"LATERAL" => lexer::Token::Lateral,
1324+
"LEADING" => lexer::Token::Leading,
13001325
"LEFT" => lexer::Token::Left,
13011326
"LIKE" => lexer::Token::Like,
13021327
"LIMIT" => lexer::Token::Limit,
@@ -1313,15 +1338,20 @@ extern {
13131338
"PARTIAL" => lexer::Token::Partial,
13141339
"PIVOT" => lexer::Token::Pivot,
13151340
"PRESERVE" => lexer::Token::Preserve,
1341+
"PUBLIC" => lexer::Token::Public,
13161342
"RIGHT" => lexer::Token::Right,
13171343
"SELECT" => lexer::Token::Select,
1344+
"SIMPLE" => lexer::Token::Simple,
13181345
"TABLE" => lexer::Token::Table,
13191346
"TIME" => lexer::Token::Time,
13201347
"TIMESTAMP" => lexer::Token::Timestamp,
13211348
"THEN" => lexer::Token::Then,
1349+
"TRAIL" => lexer::Token::Trail,
1350+
"TRAILING" => lexer::Token::Trailing,
13221351
"TRUE" => lexer::Token::True,
13231352
"UNION" => lexer::Token::Union,
13241353
"UNPIVOT" => lexer::Token::Unpivot,
1354+
"USER" => lexer::Token::User,
13251355
"USING" => lexer::Token::Using,
13261356
"VALUE" => lexer::Token::Value,
13271357
"VALUES" => lexer::Token::Values,

0 commit comments

Comments
 (0)