Question

How to remove duplicate words with Unicode characters from free‑text in C++

Answer 1

#include <iostream>
#include <string>
#include <sstream>
#include <unordered_set>
#include <locale>
#include <codecvt>
#include <vector>

/*
    This program removes duplicate words from free text containing Unicode characters.
    It uses:
      - wstring + UTF‑8 conversion for proper Unicode handling
      - locale("") for system Unicode rules
      - tolower() for case‑folding
      - a hash set to track seen words
      - punctuation stripping
    The algorithm is O(n) and preserves the order of first appearance.
*/

// Convert UTF‑8 → wide string
std::wstring utf8_to_wstring(const std::string& s) {
    std::wstring_convert<std::codecvt_utf8<wchar_t>> conv;
    
    return conv.from_bytes(s);
}

// Convert wide string → UTF‑8
std::string wstring_to_utf8(const std::wstring& ws) {
    std::wstring_convert<std::codecvt_utf8<wchar_t>> conv;
    
    return conv.to_bytes(ws);
}

// Normalize a word: lowercase + remove punctuation
std::wstring normalize_word(const std::wstring& w, const std::locale& loc) {
    std::wstring out;
    for (wchar_t c : w) {
        // Skip punctuation and symbols
        if (std::iswpunct(c) || std::iswspace(c))
            continue;

        // Lowercase using locale rules
        out.push_back(std::towlower(c));
    }

    return out;
}

int main() {
    std::string input =
        "Hello! こんにちは,  ,hello こんにちは Bună ziua; Γεια σας Bună ziua *HELLO* Γεια σας";

    // Convert to wide string for Unicode processing
    std::wstring winput = utf8_to_wstring(input);

    std::locale loc(""); // system locale for Unicode

    std::wstringstream wss(winput);
    std::wstring word;

    std::unordered_set<std::wstring> seen;   // normalized words
    std::vector<std::wstring> output;        // original words (cleaned)

    while (wss >> word) {
        std::wstring normalized = normalize_word(word, loc);

        if (!normalized.empty() && !seen.count(normalized)) {
            seen.insert(normalized);
            output.push_back(normalized); // store normalized clean word
        }
    }

    // Convert back to UTF‑8 for printing
    std::ostringstream out;
    for (size_t i = 0; i < output.size(); ++i) {
        out << wstring_to_utf8(output[i]);
        if (i + 1 < output.size()) out << " ";
    }

    std::cout << out.str() << "\n";
}


/*
run:

Hello こんにちは Bună ziua Γεια σας

*/

 

70+ SQL courses for beginners and professionals

Answer 2

#include <iostream>
#include <string>
#include <sstream>
#include <unordered_set>
#include <vector>
#include <locale>
#include <codecvt>

/*
This program removes duplicate words from free text containing Unicode characters.
It uses:
- UTF‑8 ↔ wide string conversion
- locale("") for Unicode-aware case folding
- punctuation stripping
- an O(n) duplicate-removal algorithm using unordered_set
It preserves the order of first appearance.
*/

// ---------------- UTF‑8 / Wide conversions ----------------

std::wstring utf8_to_wstring(const std::string& s) {
    std::wstring_convert<std::codecvt_utf8<wchar_t>> conv;
    
    return conv.from_bytes(s);
}

std::string wstring_to_utf8(const std::wstring& ws) {
    std::wstring_convert<std::codecvt_utf8<wchar_t>> conv;
    
    return conv.to_bytes(ws);
}

// ---------------- Word normalization ----------------

/*
normalize_word:
- removes punctuation
- lowercases using locale rules
- returns a clean comparable Unicode word
*/
std::wstring normalize_word(const std::wstring& w, const std::locale& loc) {
    std::wstring out;
    for (wchar_t c : w) {
        if (std::iswpunct(c) || std::iswspace(c))
            continue;
        out.push_back(std::towlower(c));
    }
    
    return out;
}

// ---------------- Duplicate removal ----------------

/*
remove_duplicates:
- splits text into words
- normalizes each word
- keeps only first occurrences
- returns a vector of normalized words
*/
std::vector<std::wstring> remove_duplicates(const std::wstring& winput,
                                            const std::locale& loc)
{
    std::wstringstream wss(winput);
    std::wstring word;

    std::unordered_set<std::wstring> seen;
    std::vector<std::wstring> output;

    while (wss >> word) {
        std::wstring normalized = normalize_word(word, loc);
        if (!normalized.empty() && !seen.count(normalized)) {
            seen.insert(normalized);
            output.push_back(normalized);
        }
    }

    return output;
}

// ---------------- Main ----------------

int main() {
    std::string input =
        "Hello! こんにちは,  ,hello こんにちは Bună ziua; Γεια σας Bună ziua HELLO Γεια σας";

    std::locale loc(""); // system locale for Unicode

    // Convert to wide string
    std::wstring winput = utf8_to_wstring(input);

    // Remove duplicates
    std::vector<std::wstring> unique_words = remove_duplicates(winput, loc);

    // Convert back to UTF‑8 for printing
    std::ostringstream out;
    for (size_t i = 0; i < unique_words.size(); ++i) {
        out << wstring_to_utf8(unique_words[i]);
        if (i + 1 < unique_words.size()) out << " ";
    }

    std::cout << out.str() << "\n";
}



/*
run:

Hello こんにちは Bună ziua Γεια σας

*/

 

70+ SQL courses for beginners and professionals