/**
* Removes duplicate words from a free-text string containing Unicode characters.
* Preserves word order and the case of the first occurrence.
*
* @param input - The input string containing text and punctuation.
* @returns Space-separated unique words.
*/
function removeDuplicateWords(input: string): string {
if (!input || !input.trim()) {
return '';
}
// 1. \p{L} matches Unicode letters, \p{N} matches digits.
// 'g' flag finds all occurrences, 'u' flag enables Unicode mode.
const wordPattern: RegExp = /[\p{L}\p{N}_]+/gu;
const matches: RegExpMatchArray | null = input.match(wordPattern);
if (!matches) {
return '';
}
// 2. Set for O(1) duplicate tracking.
const seenWords: Set<string> = new Set<string>();
const uniqueWords: string[] = [];
// 3. Keep first occurrence while normalizing case for check.
for (const word of matches) {
const lowerWord: string = word.toLowerCase();
if (!seenWords.has(lowerWord)) {
seenWords.add(lowerWord);
uniqueWords.push(word);
}
}
// 4. Join unique words with a single space.
return uniqueWords.join(' ');
}
// Main
const input: string = "Hello! こんにちは, ,hello こんにちは Bună ziua; Γεια σας Bună ziua *HELLO* Γεια σας";
const result: string = removeDuplicateWords(input);
console.log(result);
/*
run:
Hello こんにちは Bună ziua Γεια σας
*/
