diff --git a/Sources/FluidAudio/Shared/Download/HFTreeLister.swift b/Sources/FluidAudio/Shared/Download/HFTreeLister.swift index e560d7f94..149d35596 100644 --- a/Sources/FluidAudio/Shared/Download/HFTreeLister.swift +++ b/Sources/FluidAudio/Shared/Download/HFTreeLister.swift @@ -2,9 +2,22 @@ import Foundation /// A file discovered in a HuggingFace repository tree listing. struct RemoteFile: Equatable, Sendable { + enum ContentID: Equatable, Sendable { + case lfsSHA256(String) + case gitBlobSHA1(String) + } + let path: String /// Size in bytes as reported by the tree API; `-1` when not reported. let size: Int + /// LFS content hash, or the Git blob hash for a regular file (not an LFS pointer). + let contentID: ContentID? + + init(path: String, size: Int, contentID: ContentID? = nil) { + self.path = path + self.size = size + self.contentID = contentID + } } /// The one tree-listing implementation for the download stack (#765 Wave 3), @@ -93,7 +106,17 @@ enum HFTreeLister { ) } else if itemType == "file" { guard include(itemPath, false) else { continue } - files.append(RemoteFile(path: itemPath, size: item["size"] as? Int ?? -1)) + let contentID: RemoteFile.ContentID? + if item["lfs"] != nil { + // The top-level oid of an LFS entry hashes its pointer, + // not the resolved content downloaded into the cache. + let lfs = item["lfs"] as? [String: Any] + contentID = (lfs?["oid"] as? String).map { .lfsSHA256($0) } + } else { + contentID = (item["oid"] as? String).map { .gitBlobSHA1($0) } + } + files.append( + RemoteFile(path: itemPath, size: item["size"] as? Int ?? -1, contentID: contentID)) } } diff --git a/Sources/FluidAudio/Shared/Download/ModelCache.swift b/Sources/FluidAudio/Shared/Download/ModelCache.swift index b4575331a..7b8f7e6e1 100644 --- a/Sources/FluidAudio/Shared/Download/ModelCache.swift +++ b/Sources/FluidAudio/Shared/Download/ModelCache.swift @@ -1,3 +1,4 @@ +import CryptoKit import Foundation /// On-disk model-cache knowledge for the download stack (#765 Wave 4): @@ -29,6 +30,181 @@ enum ModelCache { return storedRevision == revision } + /// Inventory every cached compiled bundle and plain file before listing a + /// legacy pinned cache. A revision marker covers all variants in this folder. + static func legacyCacheContents( + at repoPath: URL, revision: String + ) throws -> (bundles: Set, files: Set)? { + let fm = FileManager.default + var isDirectory: ObjCBool = false + guard revision != "main", + !fm.fileExists(atPath: repoPath.appendingPathComponent(revisionMarkerName).path), + fm.fileExists(atPath: repoPath.path, isDirectory: &isDirectory), isDirectory.boolValue + else { return nil } + + let root = repoPath.resolvingSymlinksInPath() + var bundles: Set = root.pathExtension == "mlmodelc" ? [""] : [] + var files: Set = [] + guard let enumerator = fm.enumerator(at: root, includingPropertiesForKeys: nil) else { + return (bundles, files) + } + for case let item as URL in enumerator { + let item = item.resolvingSymlinksInPath() + let relative = String(item.path.dropFirst(root.path.count + 1)) + if item.pathExtension == "mlmodelc" { + bundles.insert(relative) + enumerator.skipDescendants() + continue + } + guard !relative.hasSuffix(".partial"), !relative.hasSuffix(".partial.etag"), + let attributes = try attributesIfPresent(at: item), + attributes[.type] as? FileAttributeType == .typeRegular + else { continue } + files.insert(relative) + } + return (bundles, files) + } + + /// Adopt only after the pinned listing covers every locally cached bundle. + /// Incomplete bundles are removed as a unit so the model-existence gate + /// re-enters download after an interruption. Existing markers remain authoritative. + static func adoptLegacyCache( + at repoPath: URL, revision: String, files: [RemoteFile], subPath: String? = nil + ) throws { + let fm = FileManager.default + guard let contents = try legacyCacheContents(at: repoPath, revision: revision) else { return } + let marker = repoPath.appendingPathComponent(revisionMarkerName) + var listedBundles: Set = [] + var listedFiles: Set = [] + var invalidBundles: Set = [] + var invalidFiles: Set = [] + var keptFiles: [(url: URL, bundle: String?)] = [] + for file in files { + let localPath = localPath(for: file.path, subPath: subPath) + let components = localPath.split(separator: "/") + let bundle = + repoPath.pathExtension == "mlmodelc" + ? "" + : components.firstIndex(where: { $0.hasSuffix(".mlmodelc") }).map { + components[...$0].joined(separator: "/") + } + if let bundle { listedBundles.insert(bundle) } + listedFiles.insert(localPath) + let destination = repoPath.appendingPathComponent(localPath) + guard try hasMatchingContent(file, at: destination) else { + if let bundle { + invalidBundles.insert(bundle) + } else { + invalidFiles.insert(destination) + } + continue + } + keptFiles.append((destination, bundle)) + } + invalidBundles.formUnion(contents.bundles.subtracting(listedBundles)) + for file in contents.files.subtracting(listedFiles) { + invalidFiles.insert(repoPath.appendingPathComponent(file)) + } + var removals = invalidBundles.sorted().map { + $0.isEmpty ? repoPath : repoPath.appendingPathComponent($0) + } + removals.append(contentsOf: invalidFiles) + // Rejected or missing files must not return through finished-partial reuse. + for file in invalidFiles { + removals.append(file.appendingPathExtension("partial")) + removals.append(file.appendingPathExtension("partial.etag")) + } + for file in keptFiles where file.bundle.map({ !invalidBundles.contains($0) }) ?? true { + removals.append(file.url.appendingPathExtension("partial")) + removals.append(file.url.appendingPathExtension("partial.etag")) + } + for path in removals { + do { + try fm.removeItem(at: path) + } catch { + guard isMissingFile(error) else { throw error } + } + } + try fm.createDirectory(at: repoPath, withIntermediateDirectories: true) + try Data((revision + "\n").utf8).write(to: marker, options: .atomic) + } + + /// Size is only a pre-check: an unknown size can still match its content ID. + private static func hasMatchingContent(_ file: RemoteFile, at destination: URL) throws -> Bool { + guard let contentID = file.contentID, + let attributes = try attributesIfPresent(at: destination), + attributes[.type] as? FileAttributeType == .typeRegular, + let size = (attributes[.size] as? NSNumber)?.int64Value, + file.size < 0 || size == Int64(file.size) + else { return false } + + let expected: String + let hexLength: Int + switch contentID { + case .lfsSHA256(let oid): + expected = oid.lowercased() + hexLength = 64 + case .gitBlobSHA1(let oid): + expected = oid.lowercased() + hexLength = 40 + } + guard expected.utf8.count == hexLength, + expected.utf8.allSatisfy({ (48...57).contains($0) || (97...102).contains($0) }) + else { return false } + + do { + switch contentID { + case .lfsSHA256: + return try hashFile(at: destination, using: SHA256()) == expected + case .gitBlobSHA1: + var hasher = Insecure.SHA1() + hasher.update(data: Data("blob \(size)\0".utf8)) + return try hashFile(at: destination, using: hasher) == expected + } + } catch { + guard isMissingFile(error) else { throw error } + return false + } + } + + /// Stream large weights in bounded chunks rather than materializing them in memory. + private static func hashFile(at url: URL, using initialHasher: H) throws -> String { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + var hasher = initialHasher + while let chunk = try handle.read(upToCount: 1_048_576), !chunk.isEmpty { + hasher.update(data: chunk) + } + return hasher.finalize().map { String(format: "%02x", $0) }.joined() + } + + static func localPath(for remotePath: String, subPath: String?) -> String { + guard let subPath, remotePath.hasPrefix("\(subPath)/") else { return remotePath } + return String(remotePath.dropFirst(subPath.count + 1)) + } + + /// Missing-file races are harmless; permission and other I/O errors still propagate. + static func isMissingFile(_ error: Error) -> Bool { + let error = error as NSError + if error.domain == NSCocoaErrorDomain, + error.code == NSFileNoSuchFileError || error.code == NSFileReadNoSuchFileError + { + return true + } + if error.domain == NSPOSIXErrorDomain, error.code == Int(POSIXErrorCode.ENOENT.rawValue) { return true } + guard let underlying = error.userInfo[NSUnderlyingErrorKey] as? Error else { return false } + return isMissingFile(underlying) + } + + private static func attributesIfPresent(at url: URL) throws -> [FileAttributeKey: Any]? { + do { + return try FileManager.default.attributesOfItem(atPath: url.path) + } catch { + guard isMissingFile(error) else { throw error } + return nil + } + } + /// Prepare a managed cache for downloads from one resolved revision. /// Existing files are preserved when the marker matches and replaced when /// the requested revision changes. The marker is written before downloads diff --git a/Sources/FluidAudio/Shared/Download/ModelHub.swift b/Sources/FluidAudio/Shared/Download/ModelHub.swift index 1e9f067e0..f6081a66e 100644 --- a/Sources/FluidAudio/Shared/Download/ModelHub.swift +++ b/Sources/FluidAudio/Shared/Download/ModelHub.swift @@ -502,7 +502,16 @@ public enum ModelHub { } } + let requestedPaths = Set(filesToDownload.map(\.path)) + // Validate all cached variants before stamping the shared marker, + // but only fetch files selected for this caller's download. + filesToDownload = try await filesForLegacyAdoption( + filesToDownload, at: repoPath, repo: repo, revision: revision, subPath: subPath, + includeRepoRootFiles: true, fetch: treeFetch) + try ModelCache.adoptLegacyCache( + at: repoPath, revision: revision, files: filesToDownload, subPath: subPath) try ModelCache.prepareForDownload(at: repoPath, revision: revision) + filesToDownload.removeAll { !requestedPaths.contains($0.path) } logger.info("Found \(filesToDownload.count) files to download") // Compute total known bytes for byte-weighted progress. @@ -613,7 +622,7 @@ public enum ModelHub { let reporter = ProgressReporter(handler: progressHandler, downloadPhaseWeight: 1.0) reporter.listing() let revision = ModelRegistry.mapRevision(repo.remotePath, default: repo.revision) - let filesToDownload: [RemoteFile] = try await HFTreeLister.listTree( + var filesToDownload: [RemoteFile] = try await HFTreeLister.listTree( repoRemotePath: repo.remotePath, revision: revision, startingAt: subdirectory, @@ -621,7 +630,14 @@ public enum ModelHub { fetch: HFTreeLister.fetch(using: listingSession) ) let revisionCache = repoDirectory.appendingPathComponent(subdirectory) + let requestedPaths = Set(filesToDownload.map(\.path)) + filesToDownload = try await filesForLegacyAdoption( + filesToDownload, at: revisionCache, repo: repo, revision: revision, subPath: subdirectory, + includeRepoRootFiles: false, fetch: HFTreeLister.fetch(using: listingSession)) + try ModelCache.adoptLegacyCache( + at: revisionCache, revision: revision, files: filesToDownload, subPath: subdirectory) try ModelCache.prepareForDownload(at: revisionCache, revision: revision) + filesToDownload.removeAll { !requestedPaths.contains($0.path) } let totalFiles = filesToDownload.count logger.info("Found \(totalFiles) files in \(subdirectory)") @@ -669,6 +685,41 @@ public enum ModelHub { logger.info("Downloaded \(subdirectory) from \(repo.folderName)") } + /// A legacy marker covers every cached variant, so extend the calling + /// variant's listing to all local compiled bundles and plain files first. + private static func filesForLegacyAdoption( + _ files: [RemoteFile], at repoPath: URL, repo: Repo, revision: String, subPath: String?, + includeRepoRootFiles: Bool, fetch: HFTreeLister.Fetch + ) async throws -> [RemoteFile] { + guard let contents = try ModelCache.legacyCacheContents(at: repoPath, revision: revision) else { return files } + var additional = try await HFTreeLister.listTree( + repoRemotePath: repo.remotePath, revision: revision, startingAt: subPath ?? "", + include: { path, isDirectory in + let local = ModelCache.localPath(for: path, subPath: subPath) + let inBundle = contents.bundles.contains { + $0.isEmpty || local == $0 || local.hasPrefix($0 + "/") + || (isDirectory && $0.hasPrefix(local + "/")) + } + let plainFile = + contents.files.contains(local) + || (isDirectory && contents.files.contains { $0.hasPrefix(local + "/") }) + return inBundle || plainFile + }, fetch: fetch) + let covered = Set((files + additional).map { ModelCache.localPath(for: $0.path, subPath: subPath) }) + let missingRoots = Set(contents.files.filter { !$0.contains("/") }).subtracting(covered) + // Only repo downloads flatten a subPath and can share root auxiliaries. + // Subdirectory downloads preserve remote paths and never use root files. + if includeRepoRootFiles, subPath != nil, !missingRoots.isEmpty { + additional += try await HFTreeLister.listTree( + repoRemotePath: repo.remotePath, revision: revision, + include: { path, isDirectory in !isDirectory && missingRoots.contains(path) }, fetch: fetch) + } + var merged = files + var paths = Set(files.map(\.path)) + for file in additional where paths.insert(file.path).inserted { merged.append(file) } + return merged + } + /// One file of a subdirectory download, run inside the bounded task group. private static func downloadSubdirectoryFile( _ file: RemoteFile, diff --git a/Tests/FluidAudioTests/Shared/HFTreeListerTests.swift b/Tests/FluidAudioTests/Shared/HFTreeListerTests.swift index 0baf16b1e..63d264b4a 100644 --- a/Tests/FluidAudioTests/Shared/HFTreeListerTests.swift +++ b/Tests/FluidAudioTests/Shared/HFTreeListerTests.swift @@ -59,6 +59,34 @@ final class HFTreeListerTests: XCTestCase { // MARK: - Walking + pruning + func testParsesContentIDsWithoutFallingBackToLFSPointerOID() async throws { + let server = PageServer() + let sha256 = String(repeating: "a", count: 64) + let blobSHA1 = "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" + try server.addPage( + url: treeURL(), + items: [ + ["path": "weight.bin", "type": "file", "size": 5, "oid": blobSHA1, "lfs": ["oid": sha256]], + ["path": "empty.json", "type": "file", "size": 0, "oid": blobSHA1], + ["path": "missing-lfs.bin", "type": "file", "oid": blobSHA1, "lfs": ["size": 5]], + ["path": "malformed-lfs.bin", "type": "file", "oid": blobSHA1, "lfs": "invalid"], + ["path": "missing-id.json", "type": "file"], + ]) + + let files = try await HFTreeLister.listTree( + repoRemotePath: Self.repo, include: { _, _ in true }, fetch: server.fetch) + + XCTAssertEqual( + files, + [ + RemoteFile(path: "weight.bin", size: 5, contentID: .lfsSHA256(sha256)), + RemoteFile(path: "empty.json", size: 0, contentID: .gitBlobSHA1(blobSHA1)), + RemoteFile(path: "missing-lfs.bin", size: -1), + RemoteFile(path: "malformed-lfs.bin", size: -1), + RemoteFile(path: "missing-id.json", size: -1), + ]) + } + func testRecursiveWalkWithPruningAndFileExclusion() async throws { let server = PageServer() try server.addPage( diff --git a/Tests/FluidAudioTests/Shared/ModelCacheLegacyAdoptionTests.swift b/Tests/FluidAudioTests/Shared/ModelCacheLegacyAdoptionTests.swift new file mode 100644 index 000000000..3f10b2087 --- /dev/null +++ b/Tests/FluidAudioTests/Shared/ModelCacheLegacyAdoptionTests.swift @@ -0,0 +1,657 @@ +import Foundation +import XCTest + +@testable import FluidAudio + +/// Filesystem-only cache fixtures; no CoreML model is created or loaded. +final class ModelCacheLegacyAdoptionTests: XCTestCase { + + private var repoPath: URL! + private let revision = Repo.diarizer.revision + + override func setUpWithError() throws { + repoPath = FileManager.default.temporaryDirectory + .appendingPathComponent("cache-legacy-adoption-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: repoPath, withIntermediateDirectories: true) + TreeStubURLProtocol.reset() + } + + override func tearDownWithError() throws { + TreeStubURLProtocol.reset() + try FileManager.default.removeItem(at: repoPath) + } + + private func makeFile(_ path: String, contents: String = "local") throws { + let destination = repoPath.appendingPathComponent(path) + try FileManager.default.createDirectory( + at: destination.deletingLastPathComponent(), withIntermediateDirectories: true) + try Data(contents.utf8).write(to: destination) + } + + private var stubConfiguration: URLSessionConfiguration { + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [TreeStubURLProtocol.self] + return configuration + } + + private func listedFile(path: String, size: Int) -> RemoteFile { + RemoteFile( + path: path, size: size, + contentID: .lfsSHA256("25bf8e1a2393f1108d37029b3df5593236c755742ec93465bbafa9b290bddcf6")) + } + + private func prepare(_ files: [RemoteFile], revision: String? = nil, subPath: String? = nil) throws { + let revision = revision ?? self.revision + try ModelCache.adoptLegacyCache(at: repoPath, revision: revision, files: files, subPath: subPath) + try ModelCache.prepareForDownload(at: repoPath, revision: revision) + } + + private func exists(_ path: String) -> Bool { + FileManager.default.fileExists(atPath: repoPath.appendingPathComponent(path).path) + } + + func testMatchingLegacyFilesArePreservedAndMarkerWritten() throws { + try makeFile("config.json") + try makeFile("nested/extra.json") + try makeFile("unlisted.json") + + try prepare([ + listedFile(path: "config.json", size: 5), + listedFile(path: "nested/extra.json", size: 5), + ]) + + XCTAssertEqual(try Data(contentsOf: repoPath.appendingPathComponent("config.json")), Data("local".utf8)) + XCTAssertEqual(try Data(contentsOf: repoPath.appendingPathComponent("nested/extra.json")), Data("local".utf8)) + XCTAssertFalse(exists("unlisted.json")) + XCTAssertEqual( + try String(contentsOf: repoPath.appendingPathComponent(".fluidaudio-revision"), encoding: .utf8), + revision + "\n") + } + + func testOnlyMismatchedLegacyFileIsRemoved() throws { + try makeFile("config.json") + try makeFile("nested/extra.json", contents: "truncated") + + try prepare([ + listedFile(path: "config.json", size: 5), + listedFile(path: "nested/extra.json", size: 5), + ]) + + XCTAssertTrue(exists("config.json")) + XCTAssertFalse(exists("nested/extra.json")) + XCTAssertTrue(exists("nested")) + XCTAssertTrue(ModelCache.matchesRevision(at: repoPath, revision: revision)) + } + + func testUnknownAndZeroSizedLegacyFilesWithoutIdentityAreRemoved() throws { + try makeFile("unknown.json") + try makeFile("empty.json", contents: "") + try makeFile("config.json") + + try prepare([ + RemoteFile(path: "unknown.json", size: -1), + RemoteFile(path: "empty.json", size: 0), + listedFile(path: "config.json", size: 5), + ]) + + XCTAssertFalse(exists("unknown.json")) + XCTAssertFalse(exists("empty.json")) + XCTAssertTrue(exists("config.json")) + XCTAssertTrue(ModelCache.matchesRevision(at: repoPath, revision: revision)) + } + + func testMissingLegacyFileRemainsMissing() throws { + try makeFile("config.json") + + try prepare([ + listedFile(path: "config.json", size: 5), + listedFile(path: "missing.json", size: 5), + ]) + + XCTAssertTrue(exists("config.json")) + XCTAssertFalse(exists("missing.json")) + XCTAssertTrue(ModelCache.matchesRevision(at: repoPath, revision: revision)) + } + + func testSubPathIsStrippedButRootAuxiliaryFileIsPreserved() throws { + try makeFile("nested/config.json") + try makeFile("vocab.json") + + try prepare( + [ + listedFile(path: "q8/nested/config.json", size: 5), + listedFile(path: "vocab.json", size: 5), + ], subPath: "q8") + + XCTAssertTrue(exists("nested/config.json")) + XCTAssertTrue(exists("vocab.json")) + XCTAssertTrue(ModelCache.matchesRevision(at: repoPath, revision: revision)) + } + + func testDifferentRevisionMarkerStillWipesWholeCache() throws { + try makeFile(".fluidaudio-revision", contents: String(repeating: "b", count: 40)) + try makeFile("nested/config.json") + try makeFile("unlisted.json") + + try prepare([listedFile(path: "nested/config.json", size: 5)]) + + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: repoPath.path), [".fluidaudio-revision"]) + XCTAssertTrue(ModelCache.matchesRevision(at: repoPath, revision: revision)) + } + + func testMatchingRevisionMarkerPreservesFilesWithoutSizeChecks() throws { + try makeFile(".fluidaudio-revision", contents: revision + "\n") + try makeFile("config.json") + try makeFile("weights.bin.partial") + + try prepare([listedFile(path: "config.json", size: 100)]) + + XCTAssertTrue(exists("config.json")) + XCTAssertTrue(exists("weights.bin.partial")) + XCTAssertTrue(ModelCache.matchesRevision(at: repoPath, revision: revision)) + } + + func testUnmarkedMainCacheIsUnchanged() throws { + try makeFile("config.json") + try makeFile("unknown.json") + + try prepare( + [listedFile(path: "config.json", size: 100), listedFile(path: "unknown.json", size: -1)], + revision: "main") + + XCTAssertTrue(exists("config.json")) + XCTAssertTrue(exists("unknown.json")) + XCTAssertFalse(exists(".fluidaudio-revision")) + XCTAssertTrue(ModelCache.matchesRevision(at: repoPath, revision: "main")) + } + + func testRepoDownloadAdoptsMatchingLegacyFiles() async throws { + let repo = Repo.diarizer + let required = ModelNames.getRequiredModelNames(for: repo, variant: nil).sorted() + var trees: [String: [[String: Any]]] = ["": []] + for name in required { + // Directory layout only, matching the existing cache-completeness fixtures. + try makeFile("\(repo.folderName)/\(name)/config.json") + trees["", default: []].append(["path": name, "type": "directory"]) + trees[name] = [TreeStubURLProtocol.fileEntry("\(name)/config.json")] + } + TreeStubURLProtocol.trees = trees + TreeStubURLProtocol.fileBody = Data("fresh".utf8) + + try await ModelHub.download(repo, to: repoPath, configuration: stubConfiguration) + + for name in required { + XCTAssertEqual( + try Data(contentsOf: repoPath.appendingPathComponent("\(repo.folderName)/\(name)/config.json")), + Data("local".utf8)) + } + XCTAssertTrue( + ModelCache.matchesRevision(at: repoPath.appendingPathComponent(repo.folderName), revision: revision)) + } + + private func downloadMetadata() async throws { + TreeStubURLProtocol.trees = [ + "metadata": [ + TreeStubURLProtocol.fileEntry("metadata/config.json"), + ["path": "metadata/nested", "type": "directory"], + ], + "metadata/nested": [TreeStubURLProtocol.fileEntry("metadata/nested/extra.json")], + ] + TreeStubURLProtocol.fileBody = Data("fresh".utf8) + try await ModelHub.download( + .diarizer, subdirectory: "metadata", to: repoPath, + configuration: stubConfiguration) + } + + func testSubdirectoryDownloadAdoptsMatchingLegacyFiles() async throws { + try makeFile("metadata/config.json") + try makeFile("metadata/nested/extra.json") + + try await downloadMetadata() + + XCTAssertEqual( + try Data(contentsOf: repoPath.appendingPathComponent("metadata/config.json")), Data("local".utf8)) + XCTAssertEqual( + try Data(contentsOf: repoPath.appendingPathComponent("metadata/nested/extra.json")), Data("local".utf8)) + XCTAssertTrue(ModelCache.matchesRevision(at: repoPath.appendingPathComponent("metadata"), revision: revision)) + } + + func testSubdirectoryDownloadRefetchesOnlyMismatchedLegacyFile() async throws { + try makeFile("metadata/config.json") + try makeFile("metadata/nested/extra.json", contents: "truncated") + + try await downloadMetadata() + + XCTAssertEqual( + try Data(contentsOf: repoPath.appendingPathComponent("metadata/config.json")), Data("local".utf8)) + XCTAssertEqual( + try Data(contentsOf: repoPath.appendingPathComponent("metadata/nested/extra.json")), Data("fresh".utf8)) + XCTAssertTrue(ModelCache.matchesRevision(at: repoPath.appendingPathComponent("metadata"), revision: revision)) + } + + func testStreamingDownloadValidatesStaleOfflineBundle() async throws { + let repo = Repo.diarizer + let streaming = ModelNames.getRequiredModelNames(for: repo, variant: nil).sorted() + let offline = try XCTUnwrap( + ModelNames.getRequiredModelNames(for: repo, variant: "offline").sorted().first { $0.hasSuffix(".mlmodelc") } + ) + var trees: [String: [[String: Any]]] = ["": []] + for name in streaming + [offline] { + try makeFile("\(repo.folderName)/\(name)/config.json", contents: name == offline ? "truncated" : "local") + trees["", default: []].append(["path": name, "type": "directory"]) + trees[name] = [TreeStubURLProtocol.fileEntry("\(name)/config.json")] + } + TreeStubURLProtocol.trees = trees + TreeStubURLProtocol.fileBody = Data("fresh".utf8) + + try await ModelHub.download(repo, to: repoPath, configuration: stubConfiguration) + + XCTAssertFalse(exists("\(repo.folderName)/\(offline)")) + XCTAssertEqual(TreeStubURLProtocol.fileRequestCount, 0, "variant A must not refetch variant B") + XCTAssertFalse( + ModelCache.allModelsExist(at: repoPath.appendingPathComponent(repo.folderName), models: [offline]), + "variant B's next load must enter download") + for name in streaming { + XCTAssertEqual( + try Data(contentsOf: repoPath.appendingPathComponent("\(repo.folderName)/\(name)/config.json")), + Data("local".utf8)) + } + XCTAssertTrue( + ModelCache.matchesRevision(at: repoPath.appendingPathComponent(repo.folderName), revision: revision)) + } + + func testWrongInnerFileRemovesWholeBundleBeforeWritingMarker() throws { + try makeFile("encoder.mlmodelc/config.json") + try makeFile("encoder.mlmodelc/weights/weight.bin", contents: "truncated") + try makeFile("encoder.mlmodelc/weights/weight.bin.partial", contents: "par") + try makeFile("encoder.mlmodelc/weights/weight.bin.partial.etag", contents: "\"etag\"") + try makeFile("decoder.mlmodelc/config.json") + + try prepare([ + listedFile(path: "encoder.mlmodelc/config.json", size: 5), + listedFile(path: "encoder.mlmodelc/weights/weight.bin", size: 5), + listedFile(path: "decoder.mlmodelc/config.json", size: 5), + ]) + + XCTAssertFalse(exists("encoder.mlmodelc")) + XCTAssertFalse(exists("encoder.mlmodelc/weights/weight.bin.partial")) + XCTAssertFalse(exists("encoder.mlmodelc/weights/weight.bin.partial.etag")) + XCTAssertTrue(exists("decoder.mlmodelc/config.json")) + XCTAssertTrue(ModelCache.matchesRevision(at: repoPath, revision: revision)) + // This is the exact existence gate used by loadModelsOnce before any CoreML loading. + XCTAssertFalse(ModelCache.allModelsExist(at: repoPath, models: ["encoder.mlmodelc", "decoder.mlmodelc"])) + } + + func testMissingInnerFileRemovesWholeBundleBeforeWritingMarker() throws { + try makeFile("encoder.mlmodelc/config.json") + + try prepare([ + listedFile(path: "encoder.mlmodelc/config.json", size: 5), + listedFile(path: "encoder.mlmodelc/weights/weight.bin", size: 5), + ]) + + XCTAssertFalse(exists("encoder.mlmodelc")) + XCTAssertTrue(ModelCache.matchesRevision(at: repoPath, revision: revision)) + XCTAssertFalse(ModelCache.allModelsExist(at: repoPath, models: ["encoder.mlmodelc"])) + } + + func testKeptRejectedAndMissingFilesLoseStaleSidecars() throws { + for file in ["kept.json", "removed.json", "missing.json"] { + if file != "missing.json" { try makeFile(file, contents: file == "kept.json" ? "local" : "stale") } + try makeFile(file + ".partial", contents: "stale") + try makeFile(file + ".partial.etag", contents: "\"etag\"") + } + + try prepare([ + listedFile(path: "kept.json", size: 5), + listedFile(path: "removed.json", size: 5), + listedFile(path: "missing.json", size: 5), + ]) + + XCTAssertTrue(exists("kept.json")) + XCTAssertFalse(exists("kept.json.partial")) + XCTAssertFalse(exists("kept.json.partial.etag")) + XCTAssertFalse(exists("removed.json")) + XCTAssertFalse(exists("missing.json")) + for file in ["removed.json", "missing.json"] { + XCTAssertFalse(exists(file + ".partial")) + XCTAssertFalse(exists(file + ".partial.etag")) + } + XCTAssertTrue(ModelCache.matchesRevision(at: repoPath, revision: revision)) + } + + func testRejectedAndMissingLegacyFilesCannotReuseFinishedStalePartials() async throws { + try makeFile("metadata/rejected.json", contents: "stale") + for file in ["rejected.json", "missing.json"] { + try makeFile("metadata/" + file + ".partial", contents: "stale") + try makeFile("metadata/" + file + ".partial.etag", contents: "\"old-etag\"") + } + TreeStubURLProtocol.trees = [ + "metadata": [ + TreeStubURLProtocol.fileEntry("metadata/rejected.json", contents: "fresh", lfs: false), + TreeStubURLProtocol.fileEntry("metadata/missing.json", contents: "fresh", lfs: false), + ] + ] + TreeStubURLProtocol.fileBody = Data("fresh".utf8) + + try await ModelHub.download( + .diarizer, subdirectory: "metadata", to: repoPath, configuration: stubConfiguration) + + XCTAssertEqual(TreeStubURLProtocol.fileRequestCount, 2, "both paths must fetch fresh bytes") + for file in ["rejected.json", "missing.json"] { + XCTAssertEqual( + try Data(contentsOf: repoPath.appendingPathComponent("metadata/" + file)), Data("fresh".utf8)) + XCTAssertFalse(exists("metadata/" + file + ".partial")) + XCTAssertFalse(exists("metadata/" + file + ".partial.etag")) + } + XCTAssertTrue(ModelCache.matchesRevision(at: repoPath.appendingPathComponent("metadata"), revision: revision)) + } + + func testRepoRefetchesRequestedFileWithoutFetchingRejectedOtherVariant() async throws { + let repo = Repo.diarizer + let streaming = ModelNames.getRequiredModelNames(for: repo, variant: nil).sorted() + let offline = try XCTUnwrap( + ModelNames.getRequiredModelNames(for: repo, variant: "offline").sorted().first { $0.hasSuffix(".mlmodelc") } + ) + var trees: [String: [[String: Any]]] = ["": []] + for name in streaming + [offline] { + try makeFile("\(repo.folderName)/\(name)/config.json", contents: name == offline ? "stale" : "local") + trees["", default: []].append(["path": name, "type": "directory"]) + trees[name] = [TreeStubURLProtocol.fileEntry("\(name)/config.json")] + } + let requested = "plda-parameters.json" + try makeFile("\(repo.folderName)/\(requested)", contents: "stale") + try makeFile("\(repo.folderName)/\(requested).partial", contents: "stale") + try makeFile("\(repo.folderName)/\(requested).partial.etag", contents: "\"old-etag\"") + trees["", default: []].append(TreeStubURLProtocol.fileEntry(requested, contents: "fresh", lfs: false)) + TreeStubURLProtocol.trees = trees + TreeStubURLProtocol.fileBody = Data("fresh".utf8) + + try await ModelHub.download( + repo, to: repoPath, additionalModelNames: [requested], configuration: stubConfiguration) + + XCTAssertEqual(TreeStubURLProtocol.fileRequestCount, 1, "only the caller's rejected file must be fetched") + XCTAssertEqual( + try Data(contentsOf: repoPath.appendingPathComponent("\(repo.folderName)/\(requested)")), Data("fresh".utf8) + ) + XCTAssertFalse(exists("\(repo.folderName)/\(requested).partial")) + XCTAssertFalse(exists("\(repo.folderName)/\(requested).partial.etag")) + XCTAssertFalse(exists("\(repo.folderName)/\(offline)")) + for name in streaming { + XCTAssertEqual( + try Data(contentsOf: repoPath.appendingPathComponent("\(repo.folderName)/\(name)/config.json")), + Data("local".utf8)) + } + XCTAssertTrue( + ModelCache.matchesRevision(at: repoPath.appendingPathComponent(repo.folderName), revision: revision)) + } + + func testConcurrentAdoptersTolerateDisappearingFiles() async throws { + let files = (0..<128).map { listedFile(path: "file\($0).json", size: 5) } + for file in files { + try makeFile(file.path, contents: "truncated") + } + let path = try XCTUnwrap(repoPath) + let revision = revision + + try await withThrowingTaskGroup(of: Void.self) { group in + for _ in 0..<32 { + group.addTask { + try ModelCache.adoptLegacyCache(at: path, revision: revision, files: files) + } + } + try await group.waitForAll() + } + + XCTAssertTrue(ModelCache.matchesRevision(at: path, revision: revision)) + for file in files { XCTAssertFalse(exists(file.path)) } + } + + func testMissingFileErrorClassificationIncludesCocoaAndPOSIX() { + XCTAssertTrue(ModelCache.isMissingFile(CocoaError(.fileNoSuchFile))) + XCTAssertTrue(ModelCache.isMissingFile(CocoaError(.fileReadNoSuchFile))) + XCTAssertTrue(ModelCache.isMissingFile(POSIXError(.ENOENT))) + XCTAssertFalse(ModelCache.isMissingFile(CocoaError(.fileReadNoPermission))) + XCTAssertFalse(ModelCache.isMissingFile(POSIXError(.EACCES))) + } + + func testKeptBundleHasNoStalePartialDownloads() throws { + try makeFile("encoder.mlmodelc/config.json") + try makeFile("encoder.mlmodelc/config.json.partial", contents: "par") + try makeFile("encoder.mlmodelc/config.json.partial.etag", contents: "\"etag\"") + + try prepare([listedFile(path: "encoder.mlmodelc/config.json", size: 5)]) + + XCTAssertTrue(exists("encoder.mlmodelc/config.json")) + XCTAssertFalse(ModelCache.containsPartialDownload(at: repoPath.appendingPathComponent("encoder.mlmodelc"))) + } + + func testPinnedSubPathDownloadValidatesOtherBundlesAndRootAuxiliaryFiles() async throws { + let repo = Repo.parakeetEou160 + let sub = try XCTUnwrap(repo.subPath) + let oldOverrides = ModelRegistry.revisionOverrides + ModelRegistry.revisionOverrides[repo.remotePath] = revision + defer { ModelRegistry.revisionOverrides = oldOverrides } + let bundles = ModelNames.getRequiredModelNames(for: repo, variant: nil).sorted().filter { + $0.hasSuffix(".mlmodelc") + } + var trees: [String: [[String: Any]]] = [sub: []] + for bundle in bundles + ["other.mlmodelc"] { + try makeFile( + "\(repo.folderName)/\(bundle)/config.json", contents: bundle == "other.mlmodelc" ? "truncated" : "local" + ) + trees[sub, default: []].append(["path": "\(sub)/\(bundle)", "type": "directory"]) + trees["\(sub)/\(bundle)"] = [TreeStubURLProtocol.fileEntry("\(sub)/\(bundle)/config.json")] + } + for file in ["vocab.json", "LICENSE"] { try makeFile("\(repo.folderName)/\(file)") } + trees[""] = [ + TreeStubURLProtocol.fileEntry("vocab.json", lfs: false), + TreeStubURLProtocol.fileEntry("LICENSE", lfs: false), + ] + TreeStubURLProtocol.trees = trees + TreeStubURLProtocol.fileBody = Data("fresh".utf8) + + try await ModelHub.download(repo, to: repoPath, configuration: stubConfiguration) + + XCTAssertEqual(TreeStubURLProtocol.fileRequestCount, 0) + XCTAssertFalse(exists("\(repo.folderName)/other.mlmodelc")) + for file in ["vocab.json", "LICENSE"] { + XCTAssertEqual( + try Data(contentsOf: repoPath.appendingPathComponent("\(repo.folderName)/\(file)")), Data("local".utf8)) + } + XCTAssertTrue( + ModelCache.matchesRevision(at: repoPath.appendingPathComponent(repo.folderName), revision: revision)) + } + + func testSubdirectoryLegacyFileNeverUsesRepoRootListing() async throws { + try makeFile("metadata/config.json") + try makeFile("metadata/LICENSE") + TreeStubURLProtocol.trees = [ + "metadata": [TreeStubURLProtocol.fileEntry("metadata/config.json")], + "": [TreeStubURLProtocol.fileEntry("LICENSE")], + ] + TreeStubURLProtocol.fileBody = Data("fresh".utf8) + + try await ModelHub.download( + .diarizer, subdirectory: "metadata", to: repoPath, configuration: stubConfiguration) + + XCTAssertFalse(TreeStubURLProtocol.treeRequests.contains(""), "subdirectory adoption must not list repo root") + XCTAssertFalse(exists("LICENSE"), "repo-root files must not be downloaded into repoDirectory") + XCTAssertFalse(exists("metadata/LICENSE"), "an unrelated root entry must not validate a subdirectory file") + XCTAssertEqual(TreeStubURLProtocol.fileRequestCount, 0) + XCTAssertTrue(ModelCache.matchesRevision(at: repoPath.appendingPathComponent("metadata"), revision: revision)) + } + + func testSameSizeDifferentContentIsRefetched() async throws { + try makeFile("metadata/config.json", contents: "stale") + TreeStubURLProtocol.trees = [ + "metadata": [TreeStubURLProtocol.fileEntry("metadata/config.json", contents: "fresh")] + ] + TreeStubURLProtocol.fileBody = Data("fresh".utf8) + + try await ModelHub.download( + .diarizer, subdirectory: "metadata", to: repoPath, configuration: stubConfiguration) + + XCTAssertEqual(TreeStubURLProtocol.fileRequestCount, 1) + XCTAssertEqual( + try Data(contentsOf: repoPath.appendingPathComponent("metadata/config.json")), Data("fresh".utf8)) + } + + func testMatchingLFSAndGitBlobIdentitiesAreKept() async throws { + try makeFile("metadata/weight.bin") + try makeFile("metadata/config.json") + TreeStubURLProtocol.trees = [ + "metadata": [ + TreeStubURLProtocol.fileEntry("metadata/weight.bin"), + TreeStubURLProtocol.fileEntry("metadata/config.json", lfs: false), + ] + ] + TreeStubURLProtocol.fileBody = Data("fresh".utf8) + + try await ModelHub.download( + .diarizer, subdirectory: "metadata", to: repoPath, configuration: stubConfiguration) + + XCTAssertEqual(TreeStubURLProtocol.fileRequestCount, 0) + for file in ["weight.bin", "config.json"] { + XCTAssertEqual( + try Data(contentsOf: repoPath.appendingPathComponent("metadata/" + file)), Data("local".utf8)) + } + } + + func testMatchingEmptyAndUnknownSizeIdentitiesAreKept() async throws { + try makeFile("metadata/empty.json", contents: "") + try makeFile("metadata/empty.json.partial", contents: "par") + try makeFile("metadata/empty.json.partial.etag", contents: "etag") + try makeFile("metadata/unknown.bin") + TreeStubURLProtocol.trees = [ + "metadata": [ + TreeStubURLProtocol.fileEntry("metadata/empty.json", size: 0, contents: "", lfs: false), + TreeStubURLProtocol.fileEntry("metadata/unknown.bin", size: -1), + ] + ] + TreeStubURLProtocol.fileBody = Data("fresh".utf8) + + try await ModelHub.download( + .diarizer, subdirectory: "metadata", to: repoPath, configuration: stubConfiguration) + + XCTAssertEqual(TreeStubURLProtocol.fileRequestCount, 0) + XCTAssertEqual(try Data(contentsOf: repoPath.appendingPathComponent("metadata/empty.json")), Data()) + XCTAssertFalse(exists("metadata/empty.json.partial")) + XCTAssertFalse(exists("metadata/empty.json.partial.etag")) + XCTAssertEqual( + try Data(contentsOf: repoPath.appendingPathComponent("metadata/unknown.bin")), Data("local".utf8)) + } + + func testListedFileWithoutIdentityIsRefetched() async throws { + try makeFile("metadata/config.json") + TreeStubURLProtocol.trees = [ + "metadata": [["path": "metadata/config.json", "type": "file", "size": 5]] + ] + TreeStubURLProtocol.fileBody = Data("fresh".utf8) + + try await ModelHub.download( + .diarizer, subdirectory: "metadata", to: repoPath, configuration: stubConfiguration) + + XCTAssertEqual(TreeStubURLProtocol.fileRequestCount, 1) + XCTAssertEqual( + try Data(contentsOf: repoPath.appendingPathComponent("metadata/config.json")), Data("fresh".utf8)) + } + + func testMalformedContentIDsAreRejected() throws { + let files = [ + RemoteFile(path: "short.bin", size: 5, contentID: .lfsSHA256("abc")), + RemoteFile(path: "nonhex.bin", size: 5, contentID: .lfsSHA256(String(repeating: "z", count: 64))), + RemoteFile(path: "short.json", size: 5, contentID: .gitBlobSHA1("abc")), + RemoteFile(path: "nonhex.json", size: 5, contentID: .gitBlobSHA1(String(repeating: "z", count: 40))), + ] + for file in files { try makeFile(file.path) } + + try prepare(files) + + for file in files { XCTAssertFalse(exists(file.path)) } + XCTAssertTrue(ModelCache.matchesRevision(at: repoPath, revision: revision)) + } + + func testStreamingHashesMatchAcrossChunkBoundariesWithUnknownSizes() async throws { + // Multibyte text exercises Git's byte-count header as well as multiple reads. + let contents = String(repeating: "cache-é", count: 150_000) + try makeFile("metadata/weight.bin", contents: contents) + try makeFile("metadata/config.json", contents: contents) + TreeStubURLProtocol.trees = [ + "metadata": [ + TreeStubURLProtocol.fileEntry("metadata/weight.bin", size: -1, contents: contents), + TreeStubURLProtocol.fileEntry("metadata/config.json", size: -1, contents: contents, lfs: false), + ] + ] + TreeStubURLProtocol.fileBody = Data("fresh".utf8) + + try await ModelHub.download( + .diarizer, subdirectory: "metadata", to: repoPath, configuration: stubConfiguration) + + XCTAssertEqual(TreeStubURLProtocol.fileRequestCount, 0) + for file in ["weight.bin", "config.json"] { + XCTAssertEqual( + try Data(contentsOf: repoPath.appendingPathComponent("metadata/" + file)), Data(contents.utf8)) + } + } + + func testSkippedLegacyFilesAreValidatedBeforeMarkerWithoutRefetching() async throws { + try makeFile("metadata/config.json") + try makeFile("metadata/skipped.bin", contents: "stale") + TreeStubURLProtocol.trees = [ + "metadata": [ + TreeStubURLProtocol.fileEntry("metadata/config.json"), + TreeStubURLProtocol.fileEntry("metadata/skipped.bin"), + ] + ] + TreeStubURLProtocol.fileBody = Data("fresh".utf8) + + try await ModelHub.download( + .diarizer, subdirectory: "metadata", to: repoPath, + shouldSkip: { $0.hasSuffix("skipped.bin") }, configuration: stubConfiguration) + + XCTAssertFalse(exists("metadata/skipped.bin")) + XCTAssertEqual(TreeStubURLProtocol.fileRequestCount, 0) + XCTAssertTrue(ModelCache.matchesRevision(at: repoPath.appendingPathComponent("metadata"), revision: revision)) + } + + func testRepoAdoptionVerifiesNestedPlainFilesAndInvalidatesNestedBundles() async throws { + let repo = Repo.diarizer + let required = ModelNames.getRequiredModelNames(for: repo, variant: nil).sorted() + var trees: [String: [[String: Any]]] = ["": [["path": "voices", "type": "directory"]]] + for name in required { + try makeFile("\(repo.folderName)/\(name)/config.json") + trees["", default: []].append(["path": name, "type": "directory"]) + trees[name] = [TreeStubURLProtocol.fileEntry("\(name)/config.json")] + } + for file in [ + "voices/x.bin", "voices/kept.bin", "voices/unlisted.bin", "voices/deep/other.mlmodelc/config.json", + ] { + try makeFile("\(repo.folderName)/" + file, contents: file.hasSuffix("kept.bin") ? "local" : "stale") + } + trees["voices"] = [ + TreeStubURLProtocol.fileEntry("voices/x.bin"), + TreeStubURLProtocol.fileEntry("voices/kept.bin"), + ["path": "voices/deep", "type": "directory"], + ] + trees["voices/deep"] = [["path": "voices/deep/other.mlmodelc", "type": "directory"]] + trees["voices/deep/other.mlmodelc"] = [TreeStubURLProtocol.fileEntry("voices/deep/other.mlmodelc/config.json")] + TreeStubURLProtocol.trees = trees + TreeStubURLProtocol.fileBody = Data("fresh".utf8) + + try await ModelHub.download(repo, to: repoPath, configuration: stubConfiguration) + + XCTAssertFalse(exists("\(repo.folderName)/voices/x.bin")) + XCTAssertFalse(exists("\(repo.folderName)/voices/unlisted.bin")) + XCTAssertFalse(exists("\(repo.folderName)/voices/deep/other.mlmodelc")) + XCTAssertTrue(exists("\(repo.folderName)/voices")) + XCTAssertEqual( + try Data(contentsOf: repoPath.appendingPathComponent("\(repo.folderName)/voices/kept.bin")), + Data("local".utf8)) + XCTAssertEqual( + TreeStubURLProtocol.fileRequestCount, 0, "validation-only files must not enter the download loop") + XCTAssertTrue( + ModelCache.matchesRevision(at: repoPath.appendingPathComponent(repo.folderName), revision: revision)) + } + +} diff --git a/Tests/FluidAudioTests/Shared/TreeStubURLProtocol.swift b/Tests/FluidAudioTests/Shared/TreeStubURLProtocol.swift index 28edd99bc..b1b74c248 100644 --- a/Tests/FluidAudioTests/Shared/TreeStubURLProtocol.swift +++ b/Tests/FluidAudioTests/Shared/TreeStubURLProtocol.swift @@ -1,9 +1,10 @@ +import CryptoKit import Foundation // MARK: - Tree-serving URLProtocol stub -/// Serves canned HF `tree/main` JSON per path and a fixed body for every -/// `resolve/main` file request. Thread-safe via a lock; keyed on URL shape. +/// Serves canned HF tree JSON per path and a fixed body for every +/// resolve file request, including pinned revisions. Thread-safe via a lock; keyed on URL shape. /// /// Shared HTTP test double for the `Shared/Download` suite (used by /// `ProgressSequenceTests` and others that drive `ModelHub`'s listing/download @@ -13,6 +14,36 @@ final class TreeStubURLProtocol: URLProtocol { private static let lock = NSLock() nonisolated(unsafe) private static var _trees: [String: [[String: Any]]] = [:] nonisolated(unsafe) private static var _fileBody = Data() + nonisolated(unsafe) private static var _fileRequestCount = 0 + + nonisolated(unsafe) private static var _treeRequests: [String] = [] + + static var treeRequests: [String] { + lock.lock() + defer { lock.unlock() } + return _treeRequests + } + + /// Tree API identities for filesystem fixtures; no model is created or loaded. + static func fileEntry( + _ path: String, size: Int = 5, contents: String = "local", lfs: Bool = true + ) -> [String: Any] { + let data = Data(contents.utf8) + if lfs { + let digest = SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined() + return ["path": path, "type": "file", "size": size, "lfs": ["oid": digest]] + } + var blob = Data("blob \(data.count)\0".utf8) + blob.append(data) + let digest = Insecure.SHA1.hash(data: blob).map { String(format: "%02x", $0) }.joined() + return ["path": path, "type": "file", "size": size, "oid": digest] + } + + static var fileRequestCount: Int { + lock.lock() + defer { lock.unlock() } + return _fileRequestCount + } static var trees: [String: [[String: Any]]] { get { @@ -43,6 +74,10 @@ final class TreeStubURLProtocol: URLProtocol { static func reset() { trees = [:] fileBody = Data() + lock.lock() + _fileRequestCount = 0 + _treeRequests = [] + lock.unlock() } override static func canInit(with request: URLRequest) -> Bool { true } @@ -56,10 +91,13 @@ final class TreeStubURLProtocol: URLProtocol { let path = url.path let payload: Data - if let treeRange = path.range(of: "/tree/main") { - // Listing request: key is everything after "tree/main/" ("" for root). - var key = String(path[treeRange.upperBound...]) - if key.hasPrefix("/") { key.removeFirst() } + if let treeRange = path.range(of: "/tree/") { + // Listing request: drop the revision component ("" for root). + let components = path[treeRange.upperBound...].split(separator: "/") + let key = components.dropFirst().joined(separator: "/") + Self.lock.lock() + Self._treeRequests.append(key) + Self.lock.unlock() guard let items = Self.trees[key], let json = try? JSONSerialization.data(withJSONObject: items) else { @@ -67,7 +105,10 @@ final class TreeStubURLProtocol: URLProtocol { return } payload = json - } else if path.contains("/resolve/main/") { + } else if path.contains("/resolve/") { + Self.lock.lock() + Self._fileRequestCount += 1 + Self.lock.unlock() payload = Self.fileBody } else { respond(status: 404, data: Data())