Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 2 additions & 38 deletions lib/natural/ngrams/ngrams.js
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ THE SOFTWARE.

const _ = require('underscore')
const Tokenizer = require('../tokenizers/regexp_tokenizer').WordTokenizer
const padSequence = require('./pad_sequence')
let tokenizer = new Tokenizer()
let frequencies = {}
let nrOfNgrams = 0
Expand Down Expand Up @@ -79,27 +80,9 @@ const ngrams = function (sequence, n, startSymbol, endSymbol, stats) {
sequence = tokenizer.tokenize(sequence)
}

sequence = padSequence(sequence, n, startSymbol, endSymbol)
const count = _.max([0, sequence.length - n + 1])

// Check for left padding
if (typeof startSymbol !== 'undefined' && startSymbol !== null) {
// Create an array of (n) start symbols
const blanks = []
for (let i = 0; i < n; i++) {
blanks.push(startSymbol)
}

// Create the left padding
for (let p = n - 1; p > 0; p--) {
// Create a tuple of (p) start symbols and (n - p) words
const ngram = blanks.slice(0, p).concat(sequence.slice(0, n - p))
result.push(ngram)
if (stats) {
countNgrams(ngram)
}
}
}

// Build the complete ngrams
for (let i = 0; i < count; i++) {
const ngram = sequence.slice(i, i + n)
Expand All @@ -109,25 +92,6 @@ const ngrams = function (sequence, n, startSymbol, endSymbol, stats) {
}
}

// Check for right padding
if (typeof endSymbol !== 'undefined' && endSymbol !== null) {
// Create an array of (n) end symbols
const blanks = []
for (let i = 0; i < n; i++) {
blanks.push(endSymbol)
}

// create the right padding
for (let p = n - 1; p > 0; p--) {
// Create a tuple of (p) start symbols and (n - p) words
const ngram = sequence.slice(sequence.length - p, sequence.length).concat(blanks.slice(0, n - p))
result.push(ngram)
if (stats) {
countNgrams(ngram)
}
}
}

if (stats) {
// Count frequencies
const Nr = {}
Expand Down
32 changes: 2 additions & 30 deletions lib/natural/ngrams/ngrams_zh.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ THE SOFTWARE.
*/

const _ = require('underscore')
const padSequence = require('./pad_sequence')

exports.ngrams = function (sequence, n, startSymbol, endSymbol) {
return ngrams(sequence, n, startSymbol, endSymbol)
Expand All @@ -41,42 +42,13 @@ const ngrams = function (sequence, n, startSymbol, endSymbol) {
sequence = sequence.split('')
}

sequence = padSequence(sequence, n, startSymbol, endSymbol)
const count = _.max([0, sequence.length - n + 1])

// Check for left padding
if (typeof startSymbol !== 'undefined' && startSymbol !== null) {
// Create an array of (n) start symbols
const blanks = []
for (let i = 0; i < n; i++) {
blanks.push(startSymbol)
}

// Create the left padding
for (let p = n - 1; p > 0; p--) {
// Create a tuple of (p) start symbols and (n - p) words
result.push(blanks.slice(0, p).concat(sequence.slice(0, n - p)))
}
}

// Build the complete ngrams
for (let i = 0; i < count; i++) {
result.push(sequence.slice(i, i + n))
}

// Check for right padding
if (typeof endSymbol !== 'undefined' && endSymbol !== null) {
// Create an array of (n) end symbols
const blanks = []
for (let i = 0; i < n; i++) {
blanks.push(endSymbol)
}

// create the right padding
for (let p = n - 1; p > 0; p--) {
// Create a tuple of (p) start symbols and (n - p) words
result.push(sequence.slice(sequence.length - p, sequence.length).concat(blanks.slice(0, n - p)))
}
}

return result
}
39 changes: 39 additions & 0 deletions lib/natural/ngrams/pad_sequence.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
Copyright (c) 2011, 2018 Rob Ellis, Chris Umbel, Hugo W.L. ter Doest

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

// Pad before taking windows so that short sequences can span both boundaries.
module.exports = function padSequence (sequence, n, startSymbol, endSymbol) {
const padLeft = typeof startSymbol !== 'undefined' && startSymbol !== null
const padRight = typeof endSymbol !== 'undefined' && endSymbol !== null
if (!padLeft && !padRight) {
return sequence
}

const left = []
const right = []
for (let i = 1; i < n; i++) {
if (padLeft) left.push(startSymbol)
if (padRight) right.push(endSymbol)
}

return left.concat(sequence, right)
}
123 changes: 123 additions & 0 deletions spec/ngram_padding_spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
/*
Copyright (c) 2019, Rob Ellis, Chris Umbel, Hugo W.L. ter Doest

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

'use strict'

import { NGrams, NGramsZH } from 'lib/natural'

const implementations = [
{ name: 'ngrams', implementation: NGrams, token: 'hello' },
{ name: 'ngrams (zh)', implementation: NGramsZH, token: '你' }
]

implementations.forEach(({ name, implementation, token }) => {
describe(name + ' padding short sequences', function () {
it('should include the trigram spanning both boundary symbols', function () {
expect(implementation.trigrams(token, '[start]', '[end]')).toEqual([
['[start]', '[start]', token],
['[start]', token, '[end]'],
[token, '[end]', '[end]']
])
})

it('should generate complete 4-grams from a two-token array', function () {
const sequence = [token, token]
expect(implementation.ngrams(sequence, 4, '[start]', '[end]')).toEqual([
['[start]', '[start]', '[start]', token],
['[start]', '[start]', token, token],
['[start]', token, token, '[end]'],
[token, token, '[end]', '[end]'],
[token, '[end]', '[end]', '[end]']
])
expect(sequence).toEqual([token, token])
})

it('should omit incomplete ngrams when only left padding is requested', function () {
expect(implementation.ngrams([token], 4, '[start]')).toEqual([
['[start]', '[start]', '[start]', token]
])
})

it('should omit incomplete ngrams when only right padding is requested', function () {
expect(implementation.ngrams([token], 4, null, '[end]')).toEqual([
[token, '[end]', '[end]', '[end]']
])
})

it('should produce boundary-only ngrams for empty sequences with both paddings', function () {
expect(implementation.bigrams('', '[start]', '[end]')).toEqual([
['[start]', '[end]']
])
expect(implementation.trigrams([], '[start]', '[end]')).toEqual([
['[start]', '[start]', '[end]'],
['[start]', '[end]', '[end]']
])
})

it('should return no ngrams when the padded sequence is still too short', function () {
expect(implementation.ngrams([], 3, '[start]')).toEqual([])
expect(implementation.ngrams([], 3, null, '[end]')).toEqual([])
expect(implementation.ngrams([token], 3)).toEqual([])
})

it('should accept empty strings as boundary symbols', function () {
expect(implementation.trigrams([token], '', '')).toEqual([
['', '', token],
['', token, ''],
[token, '', '']
])
})

it('should not pad unigrams', function () {
expect(implementation.ngrams([token], 1, '[start]', '[end]')).toEqual([[token]])
expect(implementation.ngrams([], 1, '[start]', '[end]')).toEqual([])
})
})
})

describe('ngram statistics for short padded sequences', function () {
it('should count the complete padded ngrams', function () {
expect(NGrams.trigrams('hello', '[start]', '[end]', true)).toEqual({
ngrams: [
['[start]', '[start]', 'hello'],
['[start]', 'hello', '[end]'],
['hello', '[end]', '[end]']
],
frequencies: {
'([start], [start], hello)': 1,
'([start], hello, [end])': 1,
'(hello, [end], [end])': 1
},
Nr: { 1: 3 },
numberOfNgrams: 3
})
})

it('should count repeated boundary-only ngrams', function () {
expect(NGrams.trigrams([], '#', '#', true)).toEqual({
ngrams: [['#', '#', '#'], ['#', '#', '#']],
frequencies: { '(#, #, #)': 2 },
Nr: { 2: 1 },
numberOfNgrams: 2
})
})
})