How to remove duplicate words from free‑text in C++

1 Answer

0 votes
#include <iostream>
#include <string>
#include <regex>
#include <unordered_set>
#include <vector>

/**
 * removeDuplicateWordsFreeText
 *
 * Removes duplicate words from free text:
 * - Case‑insensitive
 * - Preserves original casing of first occurrence
 * - Splits on ANY non‑alphabetic sequence
 * - Preserves original order
 *
 * NOTE:
 *   Standard C++ <regex> does NOT reliably support Unicode classes \p{L}, \p{M}.
 *   Therefore we use [:alpha:] which is portable.
 */
std::string removeDuplicateWordsFreeText(const std::string& text)
{
    // Trim whitespace
    auto trim = [](std::string s) {
        auto notSpace = [](unsigned char c){ return !std::isspace(c); };
        s.erase(s.begin(), std::find_if(s.begin(), s.end(), notSpace));
        s.erase(std::find_if(s.rbegin(), s.rend(), notSpace).base(), s.end());
        
        return s;
    };

    std::string cleaned = trim(text);

    // Portable splitter: split on any non‑alphabetic sequence
    std::regex splitter(R"([^[:alpha:]]+)", std::regex::ECMAScript);

    std::sregex_token_iterator it(cleaned.begin(), cleaned.end(), splitter, -1);
    std::sregex_token_iterator end;

    std::unordered_set<std::string> seen;
    std::vector<std::string> unique;

    for (; it != end; ++it) {
        std::string word = *it;
        if (word.empty()) continue;

        // Lowercase key for case‑insensitive comparison
        std::string key = word;
        std::transform(key.begin(), key.end(), key.begin(),
                       [](unsigned char c){ return std::tolower(c); });

        // Portable duplicate check
        if (seen.find(key) == seen.end()) {
            seen.insert(key);
            unique.push_back(word);
        }
    }

    // Join words
    std::string result;
    for (size_t i = 0; i < unique.size(); ++i) {
        if (i > 0) result += " ";
        result += unique[i];
    }

    return result;
}

int main()
{
    std::string input =
        "Hello, hello! This is a test. A TEST, hello universe...   "
        "UNIVERSE! Hello; ***  Is Anybody There?";

    std::string output = removeDuplicateWordsFreeText(input);

    std::cout << output << "\n";
}


/*
run:

Hello This is a test universe Anybody There

*/

 



answered Aug 2 by avibootz
edited Aug 2 by avibootz
...