/**
Unicode duplicate remover for free text.
- Uses Java's built-in Unicode support
- Removes punctuation
- Lowercases only for comparison
- Preserves original casing
- First occurrence wins
- LinkedHashMap keeps insertion order
*/
import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
import java.util.LinkedHashMap;
import java.util.Map;
public class RemoveUnicodeDuplicates {
// Remove punctuation while keeping original casing
static String cleanPreserveCase(String s) {
StringBuilder out = new StringBuilder();
s.codePoints().forEach(codePoint -> {
if (Character.isLetterOrDigit(codePoint)) {
out.appendCodePoint(codePoint);
}
});
return out.toString();
}
// Normalize a word: remove punctuation + lowercase (for comparison)
static String normalizeWord(String s) {
StringBuilder out = new StringBuilder();
s.codePoints().forEach(codePoint -> {
if (Character.isLetterOrDigit(codePoint)) {
out.appendCodePoint(Character.toLowerCase(codePoint));
}
});
return out.toString();
}
public static void main(String[] args) throws Exception {
// Force System.out to print using UTF-8 encoding
System.setOut(new PrintStream(System.out, true, StandardCharsets.UTF_8.name()));
String input =
"Hello! こんにちは, ,hello こんにちは Bună ziua; Γεια σας Bună ziua *HELLO* Γεια σας";
Map<String, String> unique = new LinkedHashMap<>();
for (String word : input.split("\\s+")) {
String normalized = normalizeWord(word);
String cleaned = cleanPreserveCase(word);
if (!normalized.isEmpty() && !unique.containsKey(normalized)) {
unique.put(normalized, cleaned);
}
}
StringBuilder out = new StringBuilder();
for (String original : unique.values()) {
if (out.length() > 0) {
out.append(" ");
}
out.append(original);
}
System.out.println(out.toString());
}
}
/*
run:
Hello こんにちは Bună ziua Γεια σας
*/
