#include <iostream>
#include <string>
#include <regex>
#include <unordered_set>
#include <vector>
/**
* removeDuplicateWordsFreeText
*
* Removes duplicate words from free text:
* - Case‑insensitive
* - Preserves original casing of first occurrence
* - Splits on ANY non‑alphabetic sequence
* - Preserves original order
*
* NOTE:
* Standard C++ <regex> does NOT reliably support Unicode classes \p{L}, \p{M}.
* Therefore we use [:alpha:] which is portable.
*/
std::string removeDuplicateWordsFreeText(const std::string& text)
{
// Trim whitespace
auto trim = [](std::string s) {
auto notSpace = [](unsigned char c){ return !std::isspace(c); };
s.erase(s.begin(), std::find_if(s.begin(), s.end(), notSpace));
s.erase(std::find_if(s.rbegin(), s.rend(), notSpace).base(), s.end());
return s;
};
std::string cleaned = trim(text);
// Portable splitter: split on any non‑alphabetic sequence
std::regex splitter(R"([^[:alpha:]]+)", std::regex::ECMAScript);
std::sregex_token_iterator it(cleaned.begin(), cleaned.end(), splitter, -1);
std::sregex_token_iterator end;
std::unordered_set<std::string> seen;
std::vector<std::string> unique;
for (; it != end; ++it) {
std::string word = *it;
if (word.empty()) continue;
// Lowercase key for case‑insensitive comparison
std::string key = word;
std::transform(key.begin(), key.end(), key.begin(),
[](unsigned char c){ return std::tolower(c); });
// Portable duplicate check
if (seen.find(key) == seen.end()) {
seen.insert(key);
unique.push_back(word);
}
}
// Join words
std::string result;
for (size_t i = 0; i < unique.size(); ++i) {
if (i > 0) result += " ";
result += unique[i];
}
return result;
}
int main()
{
std::string input =
"Hello, hello! This is a test. A TEST, hello universe... "
"UNIVERSE! Hello; *** Is Anybody There?";
std::string output = removeDuplicateWordsFreeText(input);
std::cout << output << "\n";
}
/*
run:
Hello This is a test universe Anybody There
*/