Imports System
Imports System.Text.RegularExpressions
Imports System.Collections.Generic
Module RemoveDuplicateWordsFreeText
'---------------------------------------------------------------
' Splits free text into words using Unicode-aware regex.
'
' Regex explanation:
' \P{L}+ → any sequence of NON-letter characters
' \p{L} → any Unicode letter (Hebrew, Arabic, Latin, etc.)
'
' This gives correct splitting for multilingual free text.
'---------------------------------------------------------------
Function SplitWords(text As String) As List(Of String)
text = text.Trim()
' Unicode-aware split on non-letter sequences
Dim parts As String() = Regex.Split(text, "\P{L}+")
Return New List(Of String)(parts)
End Function
'---------------------------------------------------------------
' Removes duplicate words while preserving:
' - original order
' - original casing of first occurrence
' - case-insensitive comparison
'
' Uses HashSet for O(1) average lookup time.
'---------------------------------------------------------------
Function RemoveDuplicateWords(text As String) As String
Dim words As List(Of String) = SplitWords(text)
Dim seen As New HashSet(Of String)(StringComparer.OrdinalIgnoreCase)
Dim unique As New List(Of String)()
For Each word In words
If word = "" Then Continue For
' Case-insensitive check via HashSet
If Not seen.Contains(word) Then
seen.Add(word)
unique.Add(word) ' preserve original casing
End If
Next
' Reassemble into a space-separated string
Return String.Join(" ", unique)
End Function
'---------------------------------------------------------------
' Program entry point
'---------------------------------------------------------------
Sub Main()
Dim input As String =
"Hello, hello! This is a test. A TEST, hello universe... " &
"UNIVERSE! Hello; *** Is Anybody There?"
Dim output As String = RemoveDuplicateWords(input)
Console.WriteLine(output)
End Sub
End Module
' run:
'
' Hello This is a test universe Anybody There
'
