Question

How to remove duplicate words with Unicode characters from free‑text in C

Answer 1

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>

/*
    This program removes duplicate words from free‑text containing Unicode characters.
    Notes:
      - C does not understand Unicode, but UTF‑8 bytes can be compared safely.
      - We split on ASCII whitespace.
      - We strip ASCII punctuation.
      - We lowercase only ASCII letters (Unicode case folding is not available in C).
      - We store normalized words and compare them byte‑wise.
      - The algorithm is O(n) using a simple dynamic array.
*/

// Check if a byte is ASCII punctuation
int is_ascii_punct(unsigned char c) {
    return ispunct(c);
}

// Normalize a word: remove ASCII punctuation + lowercase ASCII letters
char *normalize_word(const char *word) {
    size_t len = strlen(word);
    char *out = malloc(len + 1);
    size_t j = 0;

    for (size_t i = 0; i < len; i++) {
        unsigned char c = word[i];

        if (is_ascii_punct(c))
            continue;

        if (c < 128)  // ASCII only
            c = tolower(c);

        out[j++] = c;
    }

    out[j] = '\0';
    
    return out;
}

// Check if a word already exists in the list
int exists(char **list, int count, const char *word) {
    for (int i = 0; i < count; i++) {
        if (strcmp(list[i], word) == 0)
            return 1;
    }
    
    return 0;
}

int main() {
    const char *input =
        "Hello! こんにちは,  ,hello こんにちは Bună ziua; Γεια σας Bună ziua *HELLO* Γεια σας";

    // Copy input so strtok can modify it
    char *buffer = malloc(strlen(input) + 1);
    strcpy(buffer, input);

    char **unique = NULL;
    int count = 0;

    // Split on whitespace
    char *token = strtok(buffer, " \t\n\r");
    while (token) {
        char *norm = normalize_word(token);

        if (strlen(norm) > 0 && !exists(unique, count, norm)) {
            unique = realloc(unique, sizeof(char*) * (count + 1));
            unique[count++] = norm;
        } else {
            free(norm);
        }

        token = strtok(NULL, " \t\n\r");
    }

    // Print result
    for (int i = 0; i < count; i++) {
        printf("%s", unique[i]);
        if (i + 1 < count) printf(" ");
    }
    printf("\n");

    // Cleanup
    for (int i = 0; i < count; i++)
        free(unique[i]);
    free(unique);
    free(buffer);

    return 0;
}


/*
run:

hello こんにちは bună ziua Γεια σας

*/

 

70+ SQL courses for beginners and professionals

Answer 2

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>

/*
    This program removes duplicate words from free text containing Unicode characters.
    Because C does not understand Unicode, we treat UTF‑8 sequences as opaque bytes.
    The algorithm:
      - Split text on ASCII whitespace
      - Strip ASCII punctuation
      - Lowercase ASCII letters
      - Compare UTF‑8 words byte‑by‑byte
      - Preserve first occurrence order
      - Use dynamic arrays and functions for clarity
*/

// ---------------- Utility Functions ----------------

// Check if a byte is ASCII punctuation
int is_ascii_punct(unsigned char c) {
    return ispunct(c);
}

// Lowercase ASCII letters only (Unicode case folding is not available in C)
unsigned char ascii_lower(unsigned char c) {
    if (c < 128)
        return tolower(c);
        
    return c;
}

// ---------------- Word Normalization ----------------

/*
    normalize_word:
      - removes ASCII punctuation
      - lowercases ASCII letters
      - returns a newly allocated normalized word
*/
char *normalize_word(const char *word) {
    size_t len = strlen(word);
    char *out = malloc(len + 1);
    size_t j = 0;

    for (size_t i = 0; i < len; i++) {
        unsigned char c = word[i];

        if (is_ascii_punct(c))
            continue;

        out[j++] = ascii_lower(c);
    }

    out[j] = '\0';
    
    return out;
}

// ---------------- Duplicate Checking ----------------

/*
    exists:
      - checks if a normalized word already exists in the list
*/
int exists(char **list, int count, const char *word) {
    for (int i = 0; i < count; i++) {
        if (strcmp(list[i], word) == 0)
            return 1;
    }
    
    return 0;
}

// ---------------- Duplicate Removal ----------------

/*
    remove_duplicates:
      - splits input text into tokens
      - normalizes each token
      - stores only first occurrences
      - returns array of unique normalized words
*/
char **remove_duplicates(const char *input, int *out_count) {
    char *buffer = malloc(strlen(input) + 1);
    strcpy(buffer, input);

    char **unique = NULL;
    int count = 0;

    char *token = strtok(buffer, " \t\n\r");
    while (token) {
        char *norm = normalize_word(token);

        if (strlen(norm) > 0 && !exists(unique, count, norm)) {
            unique = realloc(unique, sizeof(char*) * (count + 1));
            unique[count++] = norm;
        } else {
            free(norm);
        }

        token = strtok(NULL, " \t\n\r");
    }

    free(buffer);
    *out_count = count;
    
    return unique;
}

// ---------------- Main ----------------

int main() {
    const char *input =
        "Hello! こんにちは,  ,hello こんにちは Bună ziua; Γεια σας Bună ziua *HELLO* Γεια σας";

    int count = 0;
    char **unique_words = remove_duplicates(input, &count);

    for (int i = 0; i < count; i++) {
        printf("%s", unique_words[i]);
        if (i + 1 < count) printf(" ");
    }
    printf("\n");

    for (int i = 0; i < count; i++)
        free(unique_words[i]);
    free(unique_words);

    return 0;
}



/*
run:

hello こんにちは bună ziua Γεια σας

*/

 

70+ SQL courses for beginners and professionals

Answer 3

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <wchar.h>
#include <wctype.h>
#include <locale.h>

/*
    This program removes duplicate words from free text containing Unicode characters.
    It uses:
      - setlocale() to enable Unicode behavior
      - mbstowcs / wcstombs for UTF‑8 ↔ wide conversion
      - iswpunct / towlower for Unicode-aware normalization
      - dynamic arrays for storing unique words
      - O(n) duplicate removal preserving first occurrence order
*/

// Convert UTF‑8 → wide string
wchar_t *utf8_to_wide(const char *input) {
    size_t len = mbstowcs(NULL, input, 0);
    wchar_t *out = malloc((len + 1) * sizeof(wchar_t));
    mbstowcs(out, input, len + 1);
    
    return out;
}

// Convert wide string → UTF‑8
char *wide_to_utf8(const wchar_t *input) {
    size_t len = wcstombs(NULL, input, 0);
    char *out = malloc(len + 1);
    wcstombs(out, input, len + 1);
    return out;
}

// Normalize a word: remove punctuation + lowercase
wchar_t *normalize_word(const wchar_t *word) {
    size_t len = wcslen(word);
    wchar_t *out = malloc((len + 1) * sizeof(wchar_t));
    size_t j = 0;

    for (size_t i = 0; i < len; i++) {
        wchar_t c = word[i];

        if (iswpunct(c) || iswspace(c))
            continue;

        out[j++] = towlower(c);
    }

    out[j] = L'\0';
    
    return out;
}

// Check if a word already exists
int exists(wchar_t **list, int count, const wchar_t *word) {
    for (int i = 0; i < count; i++) {
        if (wcscmp(list[i], word) == 0)
            return 1;
    }
    
    return 0;
}

// Remove duplicates from wide string
wchar_t **remove_duplicates(const wchar_t *input, int *out_count) {
    wchar_t *buffer = malloc((wcslen(input) + 1) * sizeof(wchar_t));
    wcscpy(buffer, input);

    wchar_t **unique = NULL;
    int count = 0;

    wchar_t *token = wcstok(buffer, L" \t\n\r", &buffer);
    while (token) {
        wchar_t *norm = normalize_word(token);

        if (wcslen(norm) > 0 && !exists(unique, count, norm)) {
            unique = realloc(unique, sizeof(wchar_t*) * (count + 1));
            unique[count++] = norm;
        } else {
            free(norm);
        }

        token = wcstok(NULL, L" \t\n\r", &buffer);
    }

    *out_count = count;
    
    return unique;
}

int main() {
    setlocale(LC_ALL, ""); // enable Unicode behavior

    const char *input_utf8 =
        "Hello! こんにちは,  ,hello こんにちは Bună ziua; Γεια σας Bună ziua *HELLO* Γεια σας";

    // Convert to wide string
    wchar_t *input = utf8_to_wide(input_utf8);

    int count = 0;
    wchar_t **unique = remove_duplicates(input, &count);

    // Print results in UTF‑8
    for (int i = 0; i < count; i++) {
        char *utf8 = wide_to_utf8(unique[i]);
        printf("%s", utf8);
        if (i + 1 < count) printf(" ");
        free(utf8);
    }
    printf("\n");

    // Cleanup
    for (int i = 0; i < count; i++)
        free(unique[i]);
    free(unique);
    free(input);

    return 0;
}


/*
run:

hello こんにちは bună ziua γεια σας

*/

 

70+ SQL courses for beginners and professionals

Answer 4

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <wchar.h>
#include <wctype.h>
#include <locale.h>

/*
    Duplicate removal with Unicode wide chars.
    - Normalize words only for comparison
    - Store original words for output
    - Preserve original casing
*/

// Convert UTF‑8 → wide string
wchar_t *utf8_to_wide(const char *input) {
    size_t len = mbstowcs(NULL, input, 0);
    wchar_t *out = malloc((len + 1) * sizeof(wchar_t));
    mbstowcs(out, input, len + 1);
    
    return out;
}

// Convert wide string → UTF‑8
char *wide_to_utf8(const wchar_t *input) {
    size_t len = wcstombs(NULL, input, 0);
    char *out = malloc(len + 1);
    wcstombs(out, input, len + 1);
    
    return out;
}

// Normalize a word for comparison
wchar_t *normalize_word(const wchar_t *word) {
    size_t len = wcslen(word);
    wchar_t *out = malloc((len + 1) * sizeof(wchar_t));
    size_t j = 0;

    for (size_t i = 0; i < len; i++) {
        wchar_t c = word[i];

        if (iswpunct(c) || iswspace(c))
            continue;

        out[j++] = towlower(c);
    }

    out[j] = L'\0';
    
    return out;
}

// Check if normalized word exists
int exists(wchar_t **norm_list, int count, const wchar_t *norm_word) {
    for (int i = 0; i < count; i++) {
        if (wcscmp(norm_list[i], norm_word) == 0)
            return 1;
    }
    
    return 0;
}

int main() {
    setlocale(LC_ALL, "");

    const char *input_utf8 =
        "Hello! こんにちは,  ,hello こんにちは Bună ziua; Γεια σας Bună ziua *HELLO* Γεια σας";

    wchar_t *input = utf8_to_wide(input_utf8);

    wchar_t **original = NULL;   // store original words
    wchar_t **normalized = NULL; // store normalized words
    int count = 0;

    wchar_t *ctx;
    wchar_t *token = wcstok(input, L" \t\n\r", &ctx);

    while (token) {
        wchar_t *norm = normalize_word(token);

        if (wcslen(norm) > 0 && !exists(normalized, count, norm)) {
            normalized = realloc(normalized, sizeof(wchar_t*) * (count + 1));
            original   = realloc(original,   sizeof(wchar_t*) * (count + 1));

            normalized[count] = norm;
            original[count]   = wcsdup(token); // store original casing

            count++;
        } else {
            free(norm);
        }

        token = wcstok(NULL, L" \t\n\r", &ctx);
    }

    // Print original words
    for (int i = 0; i < count; i++) {
        char *utf8 = wide_to_utf8(original[i]);
        printf("%s", utf8);
        if (i + 1 < count) printf(" ");
        free(utf8);
    }
    printf("\n");

    // Cleanup
    for (int i = 0; i < count; i++) {
        free(original[i]);
        free(normalized[i]);
    }
    free(original);
    free(normalized);
    free(input);

    return 0;
}


/*
run:

Hello! こんにちは, Bună ziua; Γεια σας

*/

 

70+ SQL courses for beginners and professionals