Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 19 additions & 22 deletions lib/natural/tokenizers/sentence_tokenizer.js
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,15 @@ function generateUniqueCode (base, index) {
return `{{${base}_${index}}}`
}

// Match any placeholder created by generateUniqueCode, so reverting is one pass
const replacementPlaceholderPattern = new RegExp(`\\{\\{(?:${NUM}|${URI}|${ABBREV})_\\d+\\}\\}`, 'g')
const delimiterPlaceholderPattern = new RegExp(`\\{\\{${DELIM}_\\d+\\}\\}`, 'g')

function revertPlaceholders (text, pattern, map) {
// A replacer function keeps '$' sequences in the original text literal
return text.replace(pattern, match => map.has(match) ? map.get(match) : match)
}

function escapeRegExp (string) {
return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
}
Expand Down Expand Up @@ -105,16 +114,16 @@ class SentenceTokenizer extends Tokenizer {
return [text]
}

const keys = Array.from(this.delimiterMap.keys())
const pattern = new RegExp(`(${keys.map(escapeRegExp).join('|')})`)
const parts = text.split(pattern)

const sentences = []
for (let i = 0; i < parts.length; i += 2) {
const sentence = parts[i]
const placeholder = parts[i + 1] || ''
sentences.push(sentence + placeholder)
let start = 0
for (const match of text.matchAll(delimiterPlaceholderPattern)) {
if (this.delimiterMap.has(match[0])) {
const end = match.index + match[0].length
sentences.push(text.slice(start, end))
start = end
}
}
sentences.push(text.slice(start))

return sentences
}
Expand All @@ -133,23 +142,11 @@ class SentenceTokenizer extends Tokenizer {
}

revertReplacements (text) {
let originalText = text
for (const [placeholder, replacement] of this.replacementMap.entries()) {
const pattern = new RegExp(escapeRegExp(placeholder), 'g')
originalText = originalText.replace(pattern, replacement)
}

return originalText
return revertPlaceholders(text, replacementPlaceholderPattern, this.replacementMap)
}

revertDelimiters (text) {
let originalText = text
for (const [placeholder, replacement] of this.delimiterMap.entries()) {
const pattern = new RegExp(escapeRegExp(placeholder), 'g')
originalText = originalText.replace(pattern, replacement)
}

return originalText
return revertPlaceholders(text, delimiterPlaceholderPattern, this.delimiterMap)
}

tokenize (text) {
Expand Down
5 changes: 5 additions & 0 deletions spec/sentence_tokenizer_spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,11 @@ const testData = [
input: 'My twitter feed can be found at https://twitter.com/user1.',
output: ['My twitter feed can be found at https://twitter.com/user1.']
},
{
it: 'should keep dollar signs in web addresses unchanged',
input: 'See https://example.com/?a=$&b=$1 now. Next one.',
output: ['See https://example.com/?a=$&b=$1 now.', 'Next one.']
},
{
it: 'should handle an ellipsis followed by punctuation',
input: 'Is this the end for our heroes...?',
Expand Down