Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -234,8 +234,6 @@ extension CtcKeywordSpotter {
// MARK: - Audio Preparation

private func prepareAudioArray(_ audioSamples: [Float]) throws -> (MLMultiArray, Int) {
let clampedCount = min(audioSamples.count, maxModelSamples)

// Detect expected input rank from the MelSpectrogram model's 'audio' feature description.
// Canary-1b-v2 expects rank 1 [samples], parakeet-ctc-0.6b expects rank 2 [1, samples].
let melModel = models.melSpectrogram
Expand All @@ -246,19 +244,12 @@ extension CtcKeywordSpotter {
let dataType: MLMultiArrayDataType =
audioDesc?.multiArrayConstraint?.dataType == .float16 ? .float16 : .float32

let array: MLMultiArray
if expectedRank == 2 {
// Rank 2: [1, maxSamples]
array = try MLMultiArray(shape: [1, NSNumber(value: maxModelSamples)], dataType: dataType)
} else {
// Rank 1: [maxSamples]
array = try MLMultiArray(shape: [NSNumber(value: maxModelSamples)], dataType: dataType)
}

// Copy actual samples (MLMultiArray is zero-initialized, so padding is implicit).
for i in 0..<clampedCount {
array[i] = NSNumber(value: audioSamples[i])
}
let (array, clampedCount) = try Self.makePaddedAudioArray(
audioSamples,
maxSamples: maxModelSamples,
rank: expectedRank,
dataType: dataType
)

if debugMode {
let midpoint = clampedCount / 2
Expand All @@ -278,6 +269,36 @@ extension CtcKeywordSpotter {
return (array, clampedCount)
}

/// Pad `audioSamples` to `maxSamples` and zero the unused tail.
///
/// `MLMultiArray(shape:dataType:)` does not zero storage. The mel model reads
/// the whole window, so the tail is cleared before the clip is copied.
static func makePaddedAudioArray(
_ audioSamples: [Float],
maxSamples: Int,
rank: Int,
dataType: MLMultiArrayDataType
) throws -> (MLMultiArray, Int) {
let clampedCount = min(audioSamples.count, maxSamples)

let array: MLMultiArray
if rank == 2 {
// Rank 2: [1, maxSamples]
array = try MLMultiArray(shape: [1, NSNumber(value: maxSamples)], dataType: dataType)
} else {
// Rank 1: [maxSamples]
array = try MLMultiArray(shape: [NSNumber(value: maxSamples)], dataType: dataType)
}

// The shape initializer leaves storage uninitialized. Clear it, then copy the clip.
array.reset(to: 0)
for i in 0..<clampedCount {
array[i] = NSNumber(value: audioSamples[i])
}

return (array, clampedCount)
}

private func makeAudioFeatureProvider(array: MLMultiArray, length: Int) throws -> MLFeatureProvider {
let lengthArray = try MLMultiArray(shape: [1], dataType: .int32)
lengthArray[0] = NSNumber(value: length)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
import CoreML
import XCTest

@testable import FluidAudio

/// `CtcKeywordSpotter.makePaddedAudioArray` must zero the tail past the copied clip.
/// `MLMultiArray(shape:dataType:)` does not, and the mel model reads that tail.
final class CtcKeywordSpotterAudioPaddingTests: XCTestCase {

// MARK: - Short clip

func testShortClipFloat32ZerosTheTail() throws {
let maxSamples = 32
let samples: [Float] = [0.25, -3, 2]
XCTAssertTrue(samples[0].isFinite && samples[0] != 0)

// Fresh allocations: a recycled dirty page shows up when the tail is left uninitialized.
for _ in 0..<60 {
for rank in [1, 2] {
try assertPaddedAudio(samples, maxSamples: maxSamples, rank: rank, dataType: .float32)
}
}
}

/// Same contract for `.float16`. Values are exactly representable, and reads use
/// `NSNumber.floatValue` only (macOS x86_64 has no Swift `Float16`).
func testShortClipFloat16ZerosTheTail() throws {
let maxSamples = 24
let samples: [Float] = [1, 0.5, -1.25]
XCTAssertTrue(samples[0].isFinite && samples[0] != 0)

for _ in 0..<60 {
for rank in [1, 2] {
try assertPaddedAudio(samples, maxSamples: maxSamples, rank: rank, dataType: .float16)
}
}
}

// MARK: - Window boundaries

func testInputMatchingWindowHasNoPadding() throws {
let maxSamples = 16
let samples = (0..<maxSamples).map { Float($0) + 0.5 }
for rank in [1, 2] {
try assertPaddedAudio(samples, maxSamples: maxSamples, rank: rank, dataType: .float32)
}
}

func testInputLongerThanWindowKeepsLeadingPrefix() throws {
let maxSamples = 16
let samples = (0..<(maxSamples + 8)).map { Float($0) + 0.5 }
for rank in [1, 2] {
try assertPaddedAudio(samples, maxSamples: maxSamples, rank: rank, dataType: .float32)
}
}

func testEmptyInputIsAllZeros() throws {
let maxSamples = 20
for rank in [1, 2] {
try assertPaddedAudio([], maxSamples: maxSamples, rank: rank, dataType: .float32)
}
}

// MARK: - Contract

private func assertPaddedAudio(
_ samples: [Float],
maxSamples: Int,
rank: Int,
dataType: MLMultiArrayDataType,
file: StaticString = #filePath,
line: UInt = #line
) throws {
let (array, clampedCount) = try CtcKeywordSpotter.makePaddedAudioArray(
samples,
maxSamples: maxSamples,
rank: rank,
dataType: dataType
)
let copiedCount = min(samples.count, maxSamples)

XCTAssertEqual(clampedCount, copiedCount, file: file, line: line)
XCTAssertEqual(array.count, maxSamples, file: file, line: line)
XCTAssertEqual(array.dataType, dataType, file: file, line: line)
let expectedShape = rank == 2 ? [1, maxSamples] : [maxSamples]
XCTAssertEqual(array.shape.map(\.intValue), expectedShape, file: file, line: line)

for index in 0..<array.count {
let value = array[index].floatValue
if index < clampedCount {
XCTAssertEqual(value, samples[index], file: file, line: line)
XCTAssertTrue(value.isFinite, file: file, line: line)
} else {
XCTAssertTrue(value.isFinite, "padding \(index) should be finite", file: file, line: line)
XCTAssertEqual(value, Float(0), "padding \(index) should be 0", file: file, line: line)
}
}
}
}