Merge pull request #87 from carpie/master

Allow a custom set of characters in a token

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

boolean/boolean.py

@@ -114,7 +114,8 @@ class BooleanAlgebra(object):
     """
 
     def __init__(self, TRUE_class=None, FALSE_class=None, Symbol_class=None,
-                 NOT_class=None, AND_class=None, OR_class=None):
+                 NOT_class=None, AND_class=None, OR_class=None,
+                 allowed_in_token=('.', ':', '_')):
         """
         The types for TRUE, FALSE, NOT, AND, OR and Symbol define the boolean
         algebra elements, operations and Symbol variable. They default to the
@@ -158,6 +159,9 @@ def __init__(self, TRUE_class=None, FALSE_class=None, Symbol_class=None,
             for name, value in tf_nao.items():
                 setattr(obj, name, value)
 
+        # Set the set of characters allowed in tokens
+        self.allowed_in_token = allowed_in_token
+
     def definition(self):
         """
         Return a tuple of this algebra defined elements and types as:
@@ -461,7 +465,7 @@ def tokenize(self, expr):
                 position += 1
                 while position < length:
                     char = expr[position]
-                    if char.isalnum() or char in ('.', ':', '_'):
+                    if char.isalnum() or char in self.allowed_in_token:
                         position += 1
                         tok += char
                     else:

boolean/test_boolean.py

@@ -273,6 +273,17 @@ def build_symbol(current_dotted):
         )
         self.assertEqual(expected, expr)
 
+    def test_allowing_additional_characters_in_tokens(self):
+        algebra = BooleanAlgebra(allowed_in_token=('.', '_', '-', '+'))
+        test_expr = 'l-a AND b+c'
+
+        expr = algebra.parse(test_expr)
+        expected = algebra.AND(
+            algebra.Symbol('l-a'),
+            algebra.Symbol('b+c')
+        )
+        self.assertEqual(expected, expr)
+
     def test_parse_raise_ParseError1(self):
         algebra = BooleanAlgebra()
         expr = 'l-a AND none'