codec_ext8.c

Go to the documentation of this file.

/*
 * This file is part of LibParserUtils.
 * Licensed under the MIT License,
 *                http://www.opensource.org/licenses/mit-license.php
 * Copyright 2008 John-Mark Bell <jmb@netsurf-browser.org>
 */

#include <assert.h>
#include <stdlib.h>
#include <string.h>

#include <parserutils/charset/mibenum.h>

#include "charset/codecs/codec_impl.h"
#include "utils/endian.h"
#include "utils/utils.h"

#include "charset/codecs/ext8_tables.h"

static struct {
        uint16_t mib;
        const char *name;
        size_t len;
        uint32_t *table;
} known_charsets[] = {
        { 0, "Windows-1250", SLEN("Windows-1250"), w1250 },
        { 0, "Windows-1251", SLEN("Windows-1251"), w1251 },
        { 0, "Windows-1252", SLEN("Windows-1252"), w1252 },
        { 0, "Windows-1253", SLEN("Windows-1253"), w1253 },
        { 0, "Windows-1254", SLEN("Windows-1254"), w1254 },
        { 0, "Windows-1255", SLEN("Windows-1255"), w1255 },
        { 0, "Windows-1256", SLEN("Windows-1256"), w1256 },
        { 0, "Windows-1257", SLEN("Windows-1257"), w1257 },
        { 0, "Windows-1258", SLEN("Windows-1258"), w1258 },
};

typedef struct charset_ext8_codec {
        parserutils_charset_codec base; 
        uint32_t *table;                
#define READ_BUFSIZE (8)
        uint32_t read_buf[READ_BUFSIZE];        
        size_t read_len;                
#define WRITE_BUFSIZE (8)
        uint32_t write_buf[WRITE_BUFSIZE];      
        size_t write_len;               
} charset_ext8_codec;

static bool charset_ext8_codec_handles_charset(const char *charset);
static parserutils_error charset_ext8_codec_create(const char *charset,
                parserutils_charset_codec **codec);
static parserutils_error charset_ext8_codec_destroy(
                parserutils_charset_codec *codec);
static parserutils_error charset_ext8_codec_encode(
                parserutils_charset_codec *codec,
                const uint8_t **source, size_t *sourcelen,
                uint8_t **dest, size_t *destlen);
static parserutils_error charset_ext8_codec_decode(
                parserutils_charset_codec *codec,
                const uint8_t **source, size_t *sourcelen,
                uint8_t **dest, size_t *destlen);
static parserutils_error charset_ext8_codec_reset(
                parserutils_charset_codec *codec);
static inline parserutils_error charset_ext8_codec_read_char(
                charset_ext8_codec *c,
                const uint8_t **source, size_t *sourcelen,
                uint8_t **dest, size_t *destlen);
static inline parserutils_error charset_ext8_codec_output_decoded_char(
                charset_ext8_codec *c,
                uint32_t ucs4, uint8_t **dest, size_t *destlen);
static inline parserutils_error charset_ext8_from_ucs4(charset_ext8_codec *c,
                uint32_t ucs4, uint8_t **s, size_t *len);
static inline parserutils_error charset_ext8_to_ucs4(charset_ext8_codec *c,
                const uint8_t *s, size_t len, uint32_t *ucs4);

bool charset_ext8_codec_handles_charset(const char *charset)
{
        uint32_t i;
        uint16_t match = parserutils_charset_mibenum_from_name(charset,
                        strlen(charset));

        if (known_charsets[0].mib == 0) {
                for (i = 0; i < N_ELEMENTS(known_charsets); i++) {
                        known_charsets[i].mib =
                                parserutils_charset_mibenum_from_name(
                                                known_charsets[i].name,
                                                known_charsets[i].len);
                }
        }

        for (i = 0; i < N_ELEMENTS(known_charsets); i++) {
                if (known_charsets[i].mib == match)
                        return true;
        }

        return false;
}

parserutils_error charset_ext8_codec_create(const char *charset,
                parserutils_charset_codec **codec)
{
        uint32_t i;
        charset_ext8_codec *c;
        uint16_t match = parserutils_charset_mibenum_from_name(
                        charset, strlen(charset));
        uint32_t *table = NULL;

        for (i = 0; i < N_ELEMENTS(known_charsets); i++) {
                if (known_charsets[i].mib == match) {
                        table = known_charsets[i].table;
                        break;
                }
        }

        assert(table != NULL);

        c = malloc(sizeof(charset_ext8_codec));
        if (c == NULL)
                return PARSERUTILS_NOMEM;

        c->table = table;

        c->read_buf[0] = 0;
        c->read_len = 0;

        c->write_buf[0] = 0;
        c->write_len = 0;

        /* Finally, populate vtable */
        c->base.handler.destroy = charset_ext8_codec_destroy;
        c->base.handler.encode = charset_ext8_codec_encode;
        c->base.handler.decode = charset_ext8_codec_decode;
        c->base.handler.reset = charset_ext8_codec_reset;

        *codec = (parserutils_charset_codec *) c;

        return PARSERUTILS_OK;
}

parserutils_error charset_ext8_codec_destroy (parserutils_charset_codec *codec)
{
        UNUSED(codec);

        return PARSERUTILS_OK;
}

parserutils_error charset_ext8_codec_encode(parserutils_charset_codec *codec,
                const uint8_t **source, size_t *sourcelen,
                uint8_t **dest, size_t *destlen)
{
        charset_ext8_codec *c = (charset_ext8_codec *) codec;
        uint32_t ucs4;
        uint32_t *towrite;
        size_t towritelen;
        parserutils_error error;

        /* Process any outstanding characters from the previous call */
        if (c->write_len > 0) {
                uint32_t *pwrite = c->write_buf;

                while (c->write_len > 0) {
                        error = charset_ext8_from_ucs4(c, pwrite[0],
                                        dest, destlen);
                        if (error != PARSERUTILS_OK) {
                                uint32_t len;
                                assert(error == PARSERUTILS_NOMEM);

                                for (len = 0; len < c->write_len; len++) {
                                        c->write_buf[len] = pwrite[len];
                                }

                                return error;
                        }

                        pwrite++;
                        c->write_len--;
                }
        }

        /* Now process the characters for this call */
        while (*sourcelen > 0) {
                ucs4 = endian_big_to_host(*((uint32_t *) (void *) *source));
                towrite = &ucs4;
                towritelen = 1;

                /* Output current characters */
                while (towritelen > 0) {
                        error = charset_ext8_from_ucs4(c, towrite[0], dest,
                                        destlen);
                        if (error != PARSERUTILS_OK) {
                                uint32_t len;
                                if (error != PARSERUTILS_NOMEM) {
                                        return error;
                                }

                                /* Insufficient output space */
                                assert(towritelen < WRITE_BUFSIZE);

                                c->write_len = towritelen;

                                /* Copy pending chars to save area, for
                                 * processing next call. */
                                for (len = 0; len < towritelen; len++)
                                        c->write_buf[len] = towrite[len];

                                /* Claim character we've just buffered,
                                 * so it's not reprocessed */
                                *source += 4;
                                *sourcelen -= 4;

                                return PARSERUTILS_NOMEM;
                        }

                        towrite++;
                        towritelen--;
                }

                *source += 4;
                *sourcelen -= 4;
        }

        return PARSERUTILS_OK;
}

parserutils_error charset_ext8_codec_decode(parserutils_charset_codec *codec,
                const uint8_t **source, size_t *sourcelen,
                uint8_t **dest, size_t *destlen)
{
        charset_ext8_codec *c = (charset_ext8_codec *) codec;
        parserutils_error error;

        if (c->read_len > 0) {
                /* Output left over from last decode */
                uint32_t *pread = c->read_buf;

                while (c->read_len > 0 && *destlen >= c->read_len * 4) {
                        *((uint32_t *) (void *) *dest) =
                                        endian_host_to_big(pread[0]);

                        *dest += 4;
                        *destlen -= 4;

                        pread++;
                        c->read_len--;
                }

                if (*destlen < c->read_len * 4) {
                        /* Ran out of output buffer */
                        size_t i;

                        /* Shuffle remaining output down */
                        for (i = 0; i < c->read_len; i++)
                                c->read_buf[i] = pread[i];

                        return PARSERUTILS_NOMEM;
                }
        }

        /* Finally, the "normal" case; process all outstanding characters */
        while (*sourcelen > 0) {
                error = charset_ext8_codec_read_char(c,
                                source, sourcelen, dest, destlen);
                if (error != PARSERUTILS_OK) {
                        return error;
                }
        }

        return PARSERUTILS_OK;
}

parserutils_error charset_ext8_codec_reset(parserutils_charset_codec *codec)
{
        charset_ext8_codec *c = (charset_ext8_codec *) codec;

        c->read_buf[0] = 0;
        c->read_len = 0;

        c->write_buf[0] = 0;
        c->write_len = 0;

        return PARSERUTILS_OK;
}


parserutils_error charset_ext8_codec_read_char(charset_ext8_codec *c,
                const uint8_t **source, size_t *sourcelen,
                uint8_t **dest, size_t *destlen)
{
        uint32_t ucs4;
        parserutils_error error;

        /* Convert a single character */
        error = charset_ext8_to_ucs4(c, *source, *sourcelen, &ucs4);
        if (error == PARSERUTILS_OK) {
                /* Read a character */
                error = charset_ext8_codec_output_decoded_char(c,
                                ucs4, dest, destlen);
                if (error == PARSERUTILS_OK || error == PARSERUTILS_NOMEM) {
                        /* output succeeded; update source pointers */
                        *source += 1;
                        *sourcelen -= 1;
                }

                return error;
        } else if (error == PARSERUTILS_NEEDDATA) {
                /* Can only happen if sourcelen == 0 */
                return error;
        } else if (error == PARSERUTILS_INVALID) {
                /* Illegal input sequence */

                /* Strict errormode; simply flag invalid character */
                if (c->base.errormode ==
                                PARSERUTILS_CHARSET_CODEC_ERROR_STRICT) {
                        return PARSERUTILS_INVALID;
                }

                /* output U+FFFD and continue processing. */
                error = charset_ext8_codec_output_decoded_char(c,
                                0xFFFD, dest, destlen);
                if (error == PARSERUTILS_OK || error == PARSERUTILS_NOMEM) {
                        /* output succeeded; update source pointers */
                        *source += 1;
                        *sourcelen -= 1;
                }

                return error;
        }

        return PARSERUTILS_OK;
}

parserutils_error charset_ext8_codec_output_decoded_char(charset_ext8_codec *c,
                uint32_t ucs4, uint8_t **dest, size_t *destlen)
{
        if (*destlen < 4) {
                /* Run out of output buffer */
                c->read_len = 1;
                c->read_buf[0] = ucs4;

                return PARSERUTILS_NOMEM;
        }

        *((uint32_t *) (void *) *dest) = endian_host_to_big(ucs4);
        *dest += 4;
        *destlen -= 4;

        return PARSERUTILS_OK;
}

parserutils_error charset_ext8_from_ucs4(charset_ext8_codec *c,
                uint32_t ucs4, uint8_t **s, size_t *len)
{
        uint8_t out = 0;

        if (*len < 1)
                return PARSERUTILS_NOMEM;

        if (ucs4 < 0x80) {
                /* ASCII */
                out = ucs4;
        } else {
                uint32_t i;

                for (i = 0; i < 128; i++) {
                        if (ucs4 == c->table[i])
                                break;
                }

                if (i == 128) {
                        if (c->base.errormode ==
                                        PARSERUTILS_CHARSET_CODEC_ERROR_STRICT)
                                return PARSERUTILS_INVALID;
                        else
                                out = '?';
                } else {
                        out = 0x80 + i;
                }
        }

        *(*s) = out;
        (*s)++;
        (*len)--;

        return PARSERUTILS_OK;
}

parserutils_error charset_ext8_to_ucs4(charset_ext8_codec *c,
                const uint8_t *s, size_t len, uint32_t *ucs4)
{
        uint32_t out;

        if (len < 1)
                return PARSERUTILS_NEEDDATA;

        if (*s < 0x80) {
                out = *s;
        } else {
                if (c->table[*s - 0x80] == 0xFFFF)
                        return PARSERUTILS_INVALID;

                out = c->table[*s - 0x80];
        }

        *ucs4 = out;

        return PARSERUTILS_OK;
}

const parserutils_charset_handler charset_ext8_codec_handler = {
        charset_ext8_codec_handles_charset,
        charset_ext8_codec_create
};