Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions lib/natural/distance/levenshtein_distance.js
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,33 @@ function LevenshteinDistance (source, target, options) {
return levenshteinDistance(source, target, levOptions)
}

function levenshteinDistanceTwoRows (source, target, options) {
let previous = new Array(target.length + 1)
let current = new Array(target.length + 1)
previous[0] = 0

for (let column = 1; column <= target.length; column++) {
previous[column] = previous[column - 1] + options.insertion_cost
}

for (let row = 1; row <= source.length; row++) {
current[0] = previous[0] + options.deletion_cost
for (let column = 1; column <= target.length; column++) {
const insertion = current[column - 1] + options.insertion_cost
const deletion = previous[column] + options.deletion_cost
const substitution = source[row - 1] === target[column - 1]
? previous[column - 1]
: previous[column - 1] + options.substitution_cost
current[column] = Math.min(insertion, deletion, substitution)
}
const next = previous
previous = current
current = next
}

return previous[target.length]
}

function levenshteinDistance (source, target, options) {
if (isNaN(options.insertion_cost)) options.insertion_cost = 1
if (isNaN(options.deletion_cost)) options.deletion_cost = 1
Expand All @@ -136,6 +163,16 @@ function levenshteinDistance (source, target, options) {

const sourceLength = source.length
const targetLength = target.length
const maximumCost = Number.MAX_VALUE / (sourceLength + targetLength + 1)
// Keep coercing costs and non-string inputs on the original path.
if (!options.search && !options.damerau &&
typeof source === 'string' && typeof target === 'string' &&
typeof options.insertion_cost === 'number' && Math.abs(options.insertion_cost) <= maximumCost &&
typeof options.deletion_cost === 'number' && Math.abs(options.deletion_cost) <= maximumCost &&
typeof options.substitution_cost === 'number' && Math.abs(options.substitution_cost) <= maximumCost) {
return levenshteinDistanceTwoRows(source, target, options)
}

const distanceMatrix = [[{ cost: 0 }]] // the root, has no parent cell

for (let row = 1; row <= sourceLength; row++) {
Expand Down
9 changes: 5 additions & 4 deletions lib/natural/spellcheck/spellcheck.js
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ class Spellcheck {
// Returns all edits that are 1 edit-distance away from the input word
edits (word) {
const alphabet = 'abcdefghijklmnopqrstuvwxyz'
let edits = []
const edits = []
for (let i = 0; i < word.length + 1; i++) {
if (i > 0) edits.push(word.slice(0, i - 1) + word.slice(i, word.length)) // deletes
if (i > 0 && i < word.length + 1) edits.push(word.slice(0, i - 1) + word.slice(i, i + 1) + word.slice(i - 1, i) + word.slice(i + 1, word.length)) // transposes
Expand All @@ -82,8 +82,7 @@ class Spellcheck {
}
}
// Deduplicate edits
edits = edits.filter(function (v, i, a) { return a.indexOf(v) === i })
return edits
return Array.from(new Set(edits))
}

// Returns all edits that are up to "distance" edit distance away from the input word
Expand All @@ -98,7 +97,9 @@ class Spellcheck {
// const edits = this.edits(words[0])
distance2edits[currentDepth + 1] = []
for (const i in words) {
distance2edits[currentDepth + 1] = distance2edits[currentDepth + 1].concat(this.edits(words[i]))
for (const edit of this.edits(words[i])) {
distance2edits[currentDepth + 1].push(edit)
}
}
return this.editsWithMaxDistanceHelper(distanceCounter - 1, distance2edits)
}
Expand Down
53 changes: 53 additions & 0 deletions spec/levenshtein_spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,23 @@ import {
LevenshteinDistanceSearch as levenshteinDistanceSearch
} from 'lib/natural/distance/levenshtein_distance'

function referenceDistance (source: string, target: string, insertion: number, deletion: number, substitution: number): number {
const matrix: number[][] = [[0]]
for (let row = 1; row <= source.length; row++) matrix[row] = [matrix[row - 1][0] + deletion]
for (let column = 1; column <= target.length; column++) matrix[0][column] = matrix[0][column - 1] + insertion
for (let row = 1; row <= source.length; row++) {
for (let column = 1; column <= target.length; column++) {
const insert = matrix[row][column - 1] + insertion
const remove = matrix[row - 1][column] + deletion
const replace = source[row - 1] === target[column - 1]
? matrix[row - 1][column - 1]
: matrix[row - 1][column - 1] + substitution
matrix[row][column] = Math.min(insert, remove, replace)
}
}
return matrix[source.length][target.length]
}

describe('levenshtein_distance', function () {
describe('options.search = true', function () {
it('should find cheapest substring', function () {
Expand Down Expand Up @@ -63,6 +80,42 @@ describe('levenshtein_distance', function () {
})

describe('default / options.search = false', function () {
it('matches a full matrix for randomized strings and costs', function () {
let seed = 123456789
const random = (): number => {
seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0
return seed / 0x100000000
}
const costs = [-2, -0.5, 0, 0.25, 1, 2, 4]
const alphabet = 'abc😀'
for (let trial = 0; trial < 600; trial++) {
let source = ''
let target = ''
for (let i = 0, length = Math.floor(random() * 9); i < length; i++) source += alphabet[Math.floor(random() * alphabet.length)]
for (let i = 0, length = Math.floor(random() * 9); i < length; i++) target += alphabet[Math.floor(random() * alphabet.length)]
const insertion = costs[Math.floor(random() * costs.length)]
const deletion = costs[Math.floor(random() * costs.length)]
const substitution = costs[Math.floor(random() * costs.length)]
expect(levenshteinDistance(source, target, {
insertion_cost: insertion,
deletion_cost: deletion,
substitution_cost: substitution
})).toBe(referenceDistance(source, target, insertion, deletion, substitution))
}
})

it('handles empty and identical strings and surrogate code units', function () {
expect(levenshteinDistance('', '')).toBe(0)
expect(levenshteinDistance('', '😀')).toBe(2)
expect(levenshteinDistance('😀', '')).toBe(2)
expect(levenshteinDistance('😀', '😀')).toBe(0)
expect(levenshteinDistance('😀', '😃')).toBe(1)
})

it('keeps array input on the original path', function () {
expect(levenshteinDistance(['a', 'b'] as any, 'a')).toBe(1)
})

it('should replace 2', function () {
expect(levenshteinDistance('doctor', 'doktor')).toBe(1)
})
Expand Down
17 changes: 17 additions & 0 deletions spec/spellcheck_spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ describe('spellcheck', function () {
describe('distance 1 corrections', function () {
const spellcheck = new Spellcheck(['something'])

it('returns equal-frequency corrections in edit order', function () {
expect(new Spellcheck(['cat', 'bat', 'rat']).getCorrections('at')).toEqual(['bat', 'cat', 'rat'])
})

it('should fix deletions', function () {
expect(spellcheck.getCorrections('smething')).toContain('something')
})
Expand All @@ -54,6 +58,11 @@ describe('spellcheck', function () {
const spellcheck = new Spellcheck([])
const edits = spellcheck.edits('ab')

it('keeps the first occurrence of each edit in order', function () {
expect(spellcheck.edits('a').slice(0, 5)).toEqual(['aa', 'ba', 'ca', 'da', 'ea'])
expect(spellcheck.edits('a').filter(edit => edit === 'aa')).toEqual(['aa'])
})

it('should compute insertions', function () {
const insertions = ['xab', 'axb', 'abx']
insertions.forEach(insertion => {
Expand Down Expand Up @@ -83,6 +92,14 @@ describe('spellcheck', function () {
describe('edits up to distance', function () {
const spellcheck = new Spellcheck([])

it('keeps depth and word order while retaining duplicates between parents', function () {
const edits = spellcheck.editsWithMaxDistance('a', 2) as unknown as string[][]
expect(edits[0]).toEqual(['a'])
expect(edits[1].slice(0, 3)).toEqual(['aa', 'ba', 'ca'])
expect(edits[2].slice(0, 3)).toEqual(spellcheck.edits('aa').slice(0, 3))
expect(edits[2].filter(edit => edit === 'a').length).toBeGreaterThan(1)
})

it('should correctly compute edits at distance', function () {
const edits = spellcheck.editsWithMaxDistance('abc', 2)
expect(edits[1].indexOf('abzc')).toBeGreaterThan(-1) // 1 insertion
Expand Down