Fixes issue #689 (#705)

* Fixed issue #689

* Fixed indentation

* Trailing whitespace

Author: Hugo-ter-Doest

lib/natural/tokenizers/sentence_tokenizer.js

@@ -34,7 +34,8 @@ util.inherits(SentenceTokenizer, Tokenizer)
 
 SentenceTokenizer.prototype.tokenize = function (text) {
   // break string up in to sentences based on punctation and quotation marks
-  let tokens = text.match(/(?<=\s+|^)["'‘“'"[({⟨]?(.*?[.?!…]|[^.?!…]+)(\s[.?!…])*["'’”'"\])}⟩]?(?=\s+|$)/g)
+  // let tokens = text.match(/(?<=\s+|^)["'‘“'"[({⟨]?.*?[.?!…](\s[.?!…])*["'’”'"\])}⟩]?(?=\s+|$)/g)
+  let tokens = text.match(/(?<=\s+|^)["'‘“'"[({⟨]?(.*?[.?!…]|.+)(\s[.?!…])*["'’”'"\])}⟩]?(?=\s+|$)/g)
 
   DEBUG && console.log('SentenceTokenizer.tokenize: ' + tokens)
 

spec/sentence_tokenizer_spec.js

@@ -159,18 +159,31 @@ describe('sentence_tokenizer', function () {
     ])
   })
 
-  it('should handle text with the ellipsis symbol … and it should handle last sentence without punctuation (issue #648)', function () {
+  it('Should handle text with the ellipsis symbol … (issue #648)', function () {
     expect(
       tokenizer.tokenize('We’re heading for a catastrophic global temperature rise… Fires are blazing from the Amazon to the Arctic.')
     ).toEqual([
       'We’re heading for a catastrophic global temperature rise…',
       'Fires are blazing from the Amazon to the Arctic.'
     ])
+  })
+  it('It should handle last sentence without punctuation (issue #648)', function () {
     expect(
       tokenizer.tokenize('We’re heading for a catastrophic global temperature rise. Fires are blazing from the Amazon to the Arctic')
     ).toEqual([
       'We’re heading for a catastrophic global temperature rise.',
       'Fires are blazing from the Amazon to the Arctic'
     ])
   })
+  it('It should handle the example from issue #689 correctly', function () {
+    const testInput = `
+      This is some test content.
+
+      We're trying to figure out variations in versions of the package.
+    `.trim()
+    expect(tokenizer.tokenize(testInput)).toEqual([
+      'This is some test content.',
+      'We\'re trying to figure out variations in versions of the package.'
+    ])
+  })
 })