import java.util.ArrayList;
import java.util.HashSet;
import java.util.Arrays;
import java.util.Locale;
import java.util.List;
import java.util.Set;
/**
* RemoveDuplicateWordsFreeText
*
* Removes duplicate words from free text with full Unicode support:
* - Case‑insensitive
* - Preserves original casing of first occurrence
* - Splits on ANY non‑letter (Unicode) sequence
* - Preserves original order
*
* Uses Unicode regex classes:
* \p{L} = any letter in any language
* \P{L} = any character that is NOT a letter
*/
public class RemoveDuplicateWordsFreeText {
/**
* Splits free text into words using Unicode-aware regex.
*
* Regex explanation:
* "\\P{L}+"
* \P{L} → any non-letter character
* + → one or more
*
* This splits on ANY sequence of non-letter characters.
*/
public static List<String> splitWords(String text) {
// Trim whitespace at edges
text = text.trim();
// Unicode-aware split
String[] parts = text.split("\\P{L}+");
return Arrays.asList(parts);
}
/**
* Removes duplicate words while preserving original order and casing.
*/
public static String removeDuplicateWords(String text) {
List<String> words = splitWords(text);
Set<String> seen = new HashSet<>();
List<String> unique = new ArrayList<>();
for (String word : words) {
if (word.isEmpty()) continue;
// Unicode-aware lowercase key
String key = word.toLowerCase(Locale.ROOT);
if (!seen.contains(key)) {
seen.add(key);
unique.add(word); // preserve original casing
}
}
// Reassemble into space-separated string
return String.join(" ", unique);
}
public static void main(String[] args) {
String input =
"Hello, hello! This is a test. A TEST, hello universe... "
+ "UNIVERSE! Hello; *** Is Anybody There?";
String output = removeDuplicateWords(input);
System.out.println(output);
}
}
/*
run:
Hello This is a test universe Anybody There
*/
