|
| 1 | +// |
| 2 | +// LanguageDetectionTests.swift |
| 3 | +// damusTests |
| 4 | +// |
| 5 | +// Created for testing language detection false positives. |
| 6 | +// |
| 7 | + |
| 8 | +import XCTest |
| 9 | +import NaturalLanguage |
| 10 | +@testable import damus |
| 11 | + |
| 12 | +final class LanguageDetectionTests: XCTestCase { |
| 13 | + |
| 14 | + // 107 sample texts: (text, isEnglish) |
| 15 | + // |
| 16 | + // English texts should be detected as "en" or nil, never as a foreign language. |
| 17 | + // Foreign texts should be detected as their actual language, not "en". |
| 18 | + // |
| 19 | + // The English set includes 28 phrases confirmed to trigger false positives |
| 20 | + // under the old single-hypothesis 50% threshold logic on iOS 26.2. |
| 21 | + static let sampleTexts: [(text: String, isEnglish: Bool)] = [ |
| 22 | + |
| 23 | + // -- Confirmed false positives under old logic (28 samples) -- |
| 24 | + // Each of these is English but NLLanguageRecognizer's top hypothesis |
| 25 | + // at >= 50% confidence is a non-English language. |
| 26 | + ("gut", true), // → de 98% |
| 27 | + ("vast", true), // → nl 93% |
| 28 | + ("hat", true), // → de 91% |
| 29 | + ("ez", true), // → hu 90% |
| 30 | + ("W", true), // → pl 87% |
| 31 | + ("cap", true), // → ca 83% |
| 32 | + ("sus", true), // → es 82% |
| 33 | + ("cope", true), // → es 81% |
| 34 | + ("no cap", true), // → ca 81% |
| 35 | + ("no me", true), // → es 81% |
| 36 | + ("nah", true), // → id 79% |
| 37 | + ("don\u{2019}t care", true), // → ro 79% |
| 38 | + ("it\u{2019}s joever", true), // → nl 72% |
| 39 | + ("idk", true), // → id 70% |
| 40 | + ("ya know", true), // → id 69% |
| 41 | + ("vet", true), // → nb 65% |
| 42 | + ("halt", true), // → de 65% |
| 43 | + ("nice one", true), // → id 62% |
| 44 | + ("You\u{2019}re funner", true), // → nb 61% |
| 45 | + ("water", true), // → nl 58% |
| 46 | + ("secular", true), // → es 56% |
| 47 | + ("slim", true), // → tr 56% |
| 48 | + ("stem", true), // → nl 55% |
| 49 | + ("gn gn", true), // → id 54% |
| 50 | + ("menu", true), // → id 54% |
| 51 | + ("game over", true), // → nl 53% |
| 52 | + ("general", true), // → es 51% |
| 53 | + ("gm frens", true), // → ca 51% |
| 54 | + |
| 55 | + // -- Additional short English (22 samples) -- |
| 56 | + ("good morning", true), |
| 57 | + ("gm", true), |
| 58 | + ("hello", true), |
| 59 | + ("thanks", true), |
| 60 | + ("let\u{2019}s go", true), |
| 61 | + ("LFG", true), |
| 62 | + ("GM", true), |
| 63 | + ("GN", true), |
| 64 | + ("It\u{2019}s a meme", true), |
| 65 | + ("sure thing", true), |
| 66 | + ("sounds good", true), |
| 67 | + ("that\u{2019}s cool", true), |
| 68 | + ("love this", true), |
| 69 | + ("great post", true), |
| 70 | + ("well said", true), |
| 71 | + ("based", true), |
| 72 | + ("this is the way", true), |
| 73 | + ("facts", true), |
| 74 | + ("probably nothing", true), |
| 75 | + ("don\u{2019}t trust, verify", true), |
| 76 | + ("have fun staying poor", true), |
| 77 | + ("stack sats", true), |
| 78 | + |
| 79 | + // -- Medium/longer English (20 samples) -- |
| 80 | + ("I just had the best coffee this morning", true), |
| 81 | + ("Bitcoin is going to change the world", true), |
| 82 | + ("Has anyone tried the new update?", true), |
| 83 | + ("This is why I love Nostr", true), |
| 84 | + ("The weather is beautiful today", true), |
| 85 | + ("Just deployed a new version of my app", true), |
| 86 | + ("Happy birthday! Hope you have a great day", true), |
| 87 | + ("I\u{2019}ve been thinking about this for a while", true), |
| 88 | + ("Check out this amazing sunset", true), |
| 89 | + ("Bitcoin fixes this lol", true), |
| 90 | + ("Not your keys, not your coins", true), |
| 91 | + ("Does anyone else feel this way?", true), |
| 92 | + ("I can\u{2019}t believe it\u{2019}s already March", true), |
| 93 | + ("Nostr is the super app. Because it\u{2019}s actually an ecosystem of apps, all of which make each other better.", true), |
| 94 | + ("I think the problem with social media is that it incentivizes outrage over thoughtful discussion.", true), |
| 95 | + ("Just finished reading a great book about the history of cryptography. Highly recommend it.", true), |
| 96 | + ("The best part about open protocols is that anyone can build on them without asking for permission.", true), |
| 97 | + ("I spent the whole weekend working on this project and I\u{2019}m really happy with how it turned out.", true), |
| 98 | + ("The internet was supposed to decentralize power but instead it concentrated it in a few companies.", true), |
| 99 | + ("Running a relay is one of the most impactful things you can do for the Nostr network right now.", true), |
| 100 | + |
| 101 | + // -- Short foreign texts (7 samples, all < 11 chars) -- |
| 102 | + // These exercise the non-Latin script bypass in the short-text guard. |
| 103 | + ("Привет", false), // Russian, 6 chars |
| 104 | + ("こんにちは", false), // Japanese, 5 chars |
| 105 | + ("你好", false), // Chinese, 2 chars |
| 106 | + ("مرحبا", false), // Arabic, 5 chars |
| 107 | + ("สวัสดี", false), // Thai, 6 chars |
| 108 | + ("안녕", false), // Korean, 2 chars |
| 109 | + ("Γεια", false), // Greek, 4 chars |
| 110 | + |
| 111 | + // -- Foreign language texts (30 samples) -- |
| 112 | + ("Bonjour, comment allez-vous aujourd\u{2019}hui?", false), |
| 113 | + ("Ich bin ein Berliner und ich liebe diese Stadt", false), |
| 114 | + ("Buenas noches a todos mis amigos", false), |
| 115 | + ("こんにちは、今日はいい天気ですね", false), |
| 116 | + ("今天天气真好，出去走走吧", false), |
| 117 | + ("Привет, как дела? Давно не виделись!", false), |
| 118 | + ("오늘 날씨가 정말 좋네요", false), |
| 119 | + ("สวัสดีครับ วันนี้อากาศดีมาก", false), |
| 120 | + ("مرحبا، كيف حالك اليوم؟", false), |
| 121 | + ("Bom dia! Tudo bem com você?", false), |
| 122 | + ("Ciao, come stai? Tutto bene?", false), |
| 123 | + ("Merhaba, bugün hava çok güzel", false), |
| 124 | + ("Hej, hur mår du idag?", false), |
| 125 | + ("Hei, hvordan har du det i dag?", false), |
| 126 | + ("Tere, kuidas teil läheb?", false), |
| 127 | + ("Γεια σας, πώς είστε σήμερα;", false), |
| 128 | + ("Cześć, jak się masz dzisiaj?", false), |
| 129 | + ("Ahoj, jak se máš dnes?", false), |
| 130 | + ("Hallo, hoe gaat het met je?", false), |
| 131 | + ("Xin chào, bạn khỏe không?", false), |
| 132 | + ("La vie est belle quand on est libre", false), |
| 133 | + ("Das Leben ist schön wenn man frei ist", false), |
| 134 | + ("La vida es bella cuando eres libre", false), |
| 135 | + ("人生は自由であるとき美しい", false), |
| 136 | + ("Жизнь прекрасна, когда ты свободен", false), |
| 137 | + ("인생은 자유로울 때 아름답다", false), |
| 138 | + ("ชีวิตสวยงามเมื่อคุณเป็นอิสระ", false), |
| 139 | + ("الحياة جميلة عندما تكون حرا", false), |
| 140 | + ("A vida é bela quando se é livre", false), |
| 141 | + ("La vita è bella quando sei libero", false), |
| 142 | + ] |
| 143 | + |
| 144 | + // MARK: - Locale pinning |
| 145 | + |
| 146 | + private var savedLanguages: [Any]? |
| 147 | + |
| 148 | + override func setUp() { |
| 149 | + super.setUp() |
| 150 | + savedLanguages = UserDefaults.standard.array(forKey: "AppleLanguages") |
| 151 | + UserDefaults.standard.set(["en"], forKey: "AppleLanguages") |
| 152 | + UserDefaults.standard.synchronize() |
| 153 | + } |
| 154 | + |
| 155 | + override func tearDown() { |
| 156 | + if let saved = savedLanguages { |
| 157 | + UserDefaults.standard.set(saved, forKey: "AppleLanguages") |
| 158 | + } else { |
| 159 | + UserDefaults.standard.removeObject(forKey: "AppleLanguages") |
| 160 | + } |
| 161 | + UserDefaults.standard.synchronize() |
| 162 | + super.tearDown() |
| 163 | + } |
| 164 | + |
| 165 | + // MARK: - Tests |
| 166 | + |
| 167 | + /// Regression test: English phrases must not be detected as foreign. |
| 168 | + /// This test FAILS before the fix and PASSES after. |
| 169 | + func testShortEnglishPhrasesNotDetectedAsForeign() throws { |
| 170 | + try XCTSkipUnless( |
| 171 | + localeToLanguage(Locale.current.identifier) == "en", |
| 172 | + "Test requires English locale" |
| 173 | + ) |
| 174 | + let expectation = XCTestExpectation(description: "Language detection") |
| 175 | + |
| 176 | + DispatchQueue.global().async { |
| 177 | + var failures: [(text: String, detected: String)] = [] |
| 178 | + |
| 179 | + for (text, isEnglish) in Self.sampleTexts { |
| 180 | + guard isEnglish else { continue } |
| 181 | + |
| 182 | + guard let event = NostrEvent( |
| 183 | + content: text, |
| 184 | + keypair: test_keypair, |
| 185 | + createdAt: UInt32(Date().timeIntervalSince1970) |
| 186 | + ) else { |
| 187 | + XCTFail("Could not create event for '\(text)'") |
| 188 | + continue |
| 189 | + } |
| 190 | + |
| 191 | + let lang = event.note_language(test_keypair) |
| 192 | + |
| 193 | + // English text must return "en" or nil, never a foreign language. |
| 194 | + // Returning a foreign language causes the "translate note" button |
| 195 | + // to appear, which is the bug we are fixing. |
| 196 | + if let lang, lang != "en" { |
| 197 | + failures.append((text: text, detected: lang)) |
| 198 | + } |
| 199 | + } |
| 200 | + |
| 201 | + if !failures.isEmpty { |
| 202 | + let report = failures.map { " '\($0.text)' → \($0.detected)" }.joined(separator: "\n") |
| 203 | + XCTFail("\(failures.count) English phrases falsely detected as foreign:\n\(report)") |
| 204 | + } |
| 205 | + |
| 206 | + expectation.fulfill() |
| 207 | + } |
| 208 | + |
| 209 | + wait(for: [expectation], timeout: 30) |
| 210 | + } |
| 211 | + |
| 212 | + /// Foreign language texts must still be detected as non-English. |
| 213 | + func testForeignTextsDetectedCorrectly() { |
| 214 | + let expectation = XCTestExpectation(description: "Language detection") |
| 215 | + |
| 216 | + DispatchQueue.global().async { |
| 217 | + var failures: [(text: String, detected: String?)] = [] |
| 218 | + |
| 219 | + for (text, isEnglish) in Self.sampleTexts { |
| 220 | + guard !isEnglish else { continue } |
| 221 | + |
| 222 | + guard let event = NostrEvent( |
| 223 | + content: text, |
| 224 | + keypair: test_keypair, |
| 225 | + createdAt: UInt32(Date().timeIntervalSince1970) |
| 226 | + ) else { |
| 227 | + XCTFail("Could not create event for '\(text)'") |
| 228 | + continue |
| 229 | + } |
| 230 | + |
| 231 | + let lang = event.note_language(test_keypair) |
| 232 | + |
| 233 | + // Foreign text should not be detected as English. |
| 234 | + if lang == "en" || lang == nil { |
| 235 | + failures.append((text: text, detected: lang)) |
| 236 | + } |
| 237 | + } |
| 238 | + |
| 239 | + if !failures.isEmpty { |
| 240 | + let report = failures.map { " '\($0.text)' → \($0.detected ?? "nil")" }.joined(separator: "\n") |
| 241 | + XCTFail("\(failures.count) foreign phrases incorrectly detected as English or nil:\n\(report)") |
| 242 | + } |
| 243 | + |
| 244 | + expectation.fulfill() |
| 245 | + } |
| 246 | + |
| 247 | + wait(for: [expectation], timeout: 30) |
| 248 | + } |
| 249 | + |
| 250 | + /// Comparison report: prints old vs new detection side by side for all samples. |
| 251 | + func testLanguageDetectionComparison() throws { |
| 252 | + try XCTSkipUnless( |
| 253 | + localeToLanguage(Locale.current.identifier) == "en", |
| 254 | + "Test requires English locale" |
| 255 | + ) |
| 256 | + let expectation = XCTestExpectation(description: "Comparison") |
| 257 | + |
| 258 | + DispatchQueue.global().async { |
| 259 | + var lines: [String] = [] |
| 260 | + lines.append("\n=== LANGUAGE DETECTION BEFORE/AFTER COMPARISON (100 notes) ===") |
| 261 | + lines.append(self.padRow("Text", "Expect", "Old", "New", "Hypotheses")) |
| 262 | + lines.append(String(repeating: "-", count: 110)) |
| 263 | + |
| 264 | + var falsePosBefore = 0 |
| 265 | + var falsePosAfter = 0 |
| 266 | + var falseNegBefore = 0 |
| 267 | + var falseNegAfter = 0 |
| 268 | + let currentLang = localeToLanguage(Locale.current.identifier) ?? "en" |
| 269 | + |
| 270 | + for (text, isEnglish) in Self.sampleTexts { |
| 271 | + let recognizer = NLLanguageRecognizer() |
| 272 | + recognizer.processString(text) |
| 273 | + let hyps = recognizer.languageHypotheses(withMaximum: 3) |
| 274 | + |
| 275 | + // Old logic: single top hypothesis >= 50% |
| 276 | + let oldResult: String? = { |
| 277 | + guard let top = hyps.max(by: { $0.value < $1.value }), |
| 278 | + top.value >= 0.5 else { return nil } |
| 279 | + return localeToLanguage(top.key.rawValue) |
| 280 | + }() |
| 281 | + |
| 282 | + // New logic: call the actual shipped function |
| 283 | + let newResult: String? = { |
| 284 | + guard let event = NostrEvent( |
| 285 | + content: text, |
| 286 | + keypair: test_keypair, |
| 287 | + createdAt: UInt32(Date().timeIntervalSince1970) |
| 288 | + ) else { return nil } |
| 289 | + return event.note_language(test_keypair) |
| 290 | + }() |
| 291 | + |
| 292 | + let hypStr = hyps.sorted(by: { $0.value > $1.value }) |
| 293 | + .map { "\(localeToLanguage($0.key.rawValue) ?? "?"):\(String(format: "%.0f%%", $0.value * 100))" } |
| 294 | + .joined(separator: " ") |
| 295 | + |
| 296 | + let expectStr = isEnglish ? "en" : "other" |
| 297 | + let oldStr = oldResult ?? "nil" |
| 298 | + let newStr = newResult ?? "nil" |
| 299 | + let textCol = String(text.prefix(41)) |
| 300 | + lines.append(self.padRow(textCol, expectStr, oldStr, newStr, hypStr)) |
| 301 | + |
| 302 | + if isEnglish { |
| 303 | + if let o = oldResult, o != "en" { falsePosBefore += 1 } |
| 304 | + if let n = newResult, n != "en" { falsePosAfter += 1 } |
| 305 | + } else { |
| 306 | + if oldResult == "en" || oldResult == nil { falseNegBefore += 1 } |
| 307 | + if newResult == "en" || newResult == nil { falseNegAfter += 1 } |
| 308 | + } |
| 309 | + } |
| 310 | + |
| 311 | + lines.append("\n=== SUMMARY ===") |
| 312 | + lines.append("False positives (English wrongly detected as foreign): \(falsePosBefore) before → \(falsePosAfter) after") |
| 313 | + lines.append("False negatives (foreign wrongly detected as English/nil): \(falseNegBefore) before → \(falseNegAfter) after") |
| 314 | + lines.append("Total samples: \(Self.sampleTexts.count)") |
| 315 | + |
| 316 | + for line in lines { print(line) } |
| 317 | + |
| 318 | + XCTAssertEqual(falsePosAfter, 0, "Expected zero false positives after fix") |
| 319 | + XCTAssertEqual(falseNegAfter, 0, "Expected zero false negatives after fix") |
| 320 | + |
| 321 | + expectation.fulfill() |
| 322 | + } |
| 323 | + |
| 324 | + wait(for: [expectation], timeout: 30) |
| 325 | + } |
| 326 | + |
| 327 | + private func padRow(_ text: String, _ expect: String, _ old: String, _ new: String, _ hyps: String) -> String { |
| 328 | + let col1 = text.padding(toLength: 42, withPad: " ", startingAt: 0) |
| 329 | + let col2 = expect.padding(toLength: 8, withPad: " ", startingAt: 0) |
| 330 | + let col3 = old.padding(toLength: 7, withPad: " ", startingAt: 0) |
| 331 | + let col4 = new.padding(toLength: 7, withPad: " ", startingAt: 0) |
| 332 | + return "\(col1) \(col2) \(col3) \(col4) \(hyps)" |
| 333 | + } |
| 334 | +} |
0 commit comments