1
0
mirror of https://github.com/samba-team/samba.git synced 2024-12-24 21:34:56 +03:00
samba-mirror/source4/lib/charcnv.c
Andrew Tridgell 343545a883 r3063: our default dos charset is CP850, but some systems don't have that, so
as a special case, automatically fall back to ASCII if its not found.
(This used to be commit 55aeb33343)
2007-10-10 13:01:55 -05:00

715 lines
18 KiB
C

/*
Unix SMB/CIFS implementation.
Character set conversion Extensions
Copyright (C) Igor Vergeichik <iverg@mail.ru> 2001
Copyright (C) Andrew Tridgell 2001
Copyright (C) Simo Sorce 2001
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#include "includes.h"
/**
* @file
*
* @brief Character-set conversion routines built on our iconv.
*
* @note Samba's internal character set (at least in the 3.0 series)
* is always the same as the one for the Unix filesystem. It is
* <b>not</b> necessarily UTF-8 and may be different on machines that
* need i18n filenames to be compatible with Unix software. It does
* have to be a superset of ASCII. All multibyte sequences must start
* with a byte with the high bit set.
*
* @sa lib/iconv.c
*/
/**
* Return the name of a charset to give to iconv().
**/
static const char *charset_name(charset_t ch)
{
const char *ret = NULL;
if (ch == CH_UTF16) ret = "UTF-16LE";
else if (ch == CH_UNIX) ret = lp_unix_charset();
else if (ch == CH_DOS) ret = lp_dos_charset();
else if (ch == CH_DISPLAY) ret = lp_display_charset();
else if (ch == CH_UTF8) ret = "UTF8";
else if (ch == CH_UTF16BE) ret = "UTF-16BE";
if (!ret || !*ret) ret = "ASCII";
return ret;
}
static smb_iconv_t conv_handles[NUM_CHARSETS][NUM_CHARSETS];
/**
re-initialize iconv conversion descriptors
**/
void init_iconv(void)
{
charset_t c1, c2;
for (c1=0;c1<NUM_CHARSETS;c1++) {
for (c2=0;c2<NUM_CHARSETS;c2++) {
if (conv_handles[c1][c2] != NULL) {
if (conv_handles[c1][c2] != (smb_iconv_t)-1) {
smb_iconv_close(conv_handles[c1][c2]);
}
conv_handles[c1][c2] = NULL;
}
}
}
}
/*
on-demand initialisation of conversion handles
*/
static smb_iconv_t get_conv_handle(charset_t from, charset_t to)
{
const char *n1, *n2;
static int initialised;
/* auto-free iconv memory on exit so valgrind reports are easier
to look at */
if (initialised == 0) {
initialised = 1;
atexit(init_iconv);
}
if (conv_handles[from][to]) {
return conv_handles[from][to];
}
n1 = charset_name(from);
n2 = charset_name(to);
conv_handles[from][to] = smb_iconv_open(n2,n1);
if (conv_handles[from][to] == (smb_iconv_t)-1) {
if ((from == CH_DOS || to == CH_DOS) &&
strcasecmp(charset_name(CH_DOS), "ASCII") != 0) {
DEBUG(0,("dos charset '%s' unavailable - using ASCII\n",
charset_name(CH_DOS)));
lp_set_cmdline("dos charset", "ASCII");
n1 = charset_name(from);
n2 = charset_name(to);
conv_handles[from][to] = smb_iconv_open(n2,n1);
}
}
return conv_handles[from][to];
}
/**
* Convert string from one encoding to another, making error checking etc
*
* @param src pointer to source string (multibyte or singlebyte)
* @param srclen length of the source string in bytes
* @param dest pointer to destination string (multibyte or singlebyte)
* @param destlen maximal length allowed for string
* @returns the number of bytes occupied in the destination
**/
ssize_t convert_string(charset_t from, charset_t to,
void const *src, size_t srclen,
void *dest, size_t destlen)
{
size_t i_len, o_len;
size_t retval;
const char* inbuf = (const char*)src;
char* outbuf = (char*)dest;
smb_iconv_t descriptor;
if (srclen == (size_t)-1)
srclen = strlen(src)+1;
descriptor = get_conv_handle(from, to);
if (descriptor == (smb_iconv_t)-1 || descriptor == (smb_iconv_t)0) {
/* conversion not supported, use as is */
size_t len = MIN(srclen,destlen);
memcpy(dest,src,len);
return len;
}
i_len=srclen;
o_len=destlen;
retval = smb_iconv(descriptor, &inbuf, &i_len, &outbuf, &o_len);
if(retval==(size_t)-1) {
const char *reason;
switch(errno) {
case EINVAL:
reason="Incomplete multibyte sequence";
break;
case E2BIG:
reason="No more room";
if (from == CH_UNIX) {
DEBUG(0,("E2BIG: convert_string(%s,%s): srclen=%d destlen=%d - '%s'\n",
charset_name(from), charset_name(to),
srclen, destlen, (const char *)src));
} else {
DEBUG(0,("E2BIG: convert_string(%s,%s): srclen=%d destlen=%d\n",
charset_name(from), charset_name(to),
srclen, destlen));
}
break;
case EILSEQ:
reason="Illegal multibyte sequence";
break;
}
/* smb_panic(reason); */
}
return destlen-o_len;
}
/**
* Convert between character sets, allocating a new buffer using talloc for the result.
*
* @param srclen length of source buffer.
* @param dest always set at least to NULL
* @note -1 is not accepted for srclen.
*
* @returns Size in bytes of the converted string; or -1 in case of error.
**/
ssize_t convert_string_talloc(TALLOC_CTX *ctx, charset_t from, charset_t to,
void const *src, size_t srclen, void **dest)
{
size_t i_len, o_len, destlen;
size_t retval;
const char *inbuf = (const char *)src;
char *outbuf, *ob;
smb_iconv_t descriptor;
*dest = NULL;
if (src == NULL || srclen == (size_t)-1 || srclen == 0)
return (size_t)-1;
descriptor = get_conv_handle(from, to);
if (descriptor == (smb_iconv_t)-1 || descriptor == (smb_iconv_t)0) {
/* conversion not supported, return -1*/
DEBUG(3, ("convert_string_talloc: conversion from %s to %s not supported!\n",
charset_name(from), charset_name(to)));
return -1;
}
/* it is _very_ rare that a conversion increases the size by
more than 3x */
destlen = srclen;
outbuf = NULL;
convert:
destlen = 2 + (destlen*3);
ob = (char *)talloc_realloc(ctx, outbuf, destlen);
if (!ob) {
DEBUG(0, ("convert_string_talloc: realloc failed!\n"));
talloc_free(outbuf);
return (size_t)-1;
} else {
outbuf = ob;
}
/* we give iconv 2 less bytes to allow us to terminate at the
end */
i_len = srclen;
o_len = destlen-2;
retval = smb_iconv(descriptor,
&inbuf, &i_len,
&outbuf, &o_len);
if(retval == (size_t)-1) {
const char *reason="unknown error";
switch(errno) {
case EINVAL:
reason="Incomplete multibyte sequence";
break;
case E2BIG:
goto convert;
case EILSEQ:
reason="Illegal multibyte sequence";
break;
}
DEBUG(0,("Conversion error: %s(%s)\n",reason,inbuf));
talloc_free(ob);
return (size_t)-1;
}
destlen = (destlen-2) - o_len;
/* guarantee null termination in all charsets */
SSVAL(ob, destlen, 0);
*dest = ob;
return destlen;
}
/**
* Copy a string from a char* unix src to a dos codepage string destination.
*
* @return the number of bytes occupied by the string in the destination.
*
* @param flags can include
* <dl>
* <dt>STR_TERMINATE</dt> <dd>means include the null termination</dd>
* <dt>STR_UPPER</dt> <dd>means uppercase in the destination</dd>
* </dl>
*
* @param dest_len the maximum length in bytes allowed in the
* destination. If @p dest_len is -1 then no maximum is used.
**/
ssize_t push_ascii(void *dest, const char *src, size_t dest_len, int flags)
{
size_t src_len;
ssize_t ret;
if (flags & STR_UPPER) {
char *tmpbuf = strupper_talloc(NULL, src);
if (tmpbuf == NULL) {
return -1;
}
ret = push_ascii(dest, tmpbuf, dest_len, flags & ~STR_UPPER);
talloc_free(tmpbuf);
return ret;
}
/* treat a pstring as "unlimited" length */
if (dest_len == (size_t)-1)
dest_len = sizeof(pstring);
src_len = strlen(src);
if (flags & (STR_TERMINATE | STR_TERMINATE_ASCII))
src_len++;
return convert_string(CH_UNIX, CH_DOS, src, src_len, dest, dest_len);
}
/**
* Copy a string from a unix char* src to an ASCII destination,
* allocating a buffer using talloc().
*
* @param dest always set at least to NULL
*
* @returns The number of bytes occupied by the string in the destination
* or -1 in case of error.
**/
ssize_t push_ascii_talloc(TALLOC_CTX *ctx, char **dest, const char *src)
{
size_t src_len = strlen(src)+1;
*dest = NULL;
return convert_string_talloc(ctx, CH_UNIX, CH_DOS, src, src_len, (void **)dest);
}
/**
* Copy a string from a dos codepage source to a unix char* destination.
*
* The resulting string in "dest" is always null terminated.
*
* @param flags can have:
* <dl>
* <dt>STR_TERMINATE</dt>
* <dd>STR_TERMINATE means the string in @p src
* is null terminated, and src_len is ignored.</dd>
* </dl>
*
* @param src_len is the length of the source area in bytes.
* @returns the number of bytes occupied by the string in @p src.
**/
ssize_t pull_ascii(char *dest, const void *src, size_t dest_len, size_t src_len, int flags)
{
size_t ret;
if (dest_len == (size_t)-1)
dest_len = sizeof(pstring);
if (flags & (STR_TERMINATE | STR_TERMINATE_ASCII)) {
if (src_len == (size_t)-1) {
src_len = strlen(src) + 1;
} else {
size_t len = strnlen(src, src_len);
if (len < src_len)
len++;
src_len = len;
}
}
ret = convert_string(CH_DOS, CH_UNIX, src, src_len, dest, dest_len);
if (dest_len)
dest[MIN(ret, dest_len-1)] = 0;
return src_len;
}
/**
* Copy a string from a char* src to a unicode destination.
*
* @returns the number of bytes occupied by the string in the destination.
*
* @param flags can have:
*
* <dl>
* <dt>STR_TERMINATE <dd>means include the null termination.
* <dt>STR_UPPER <dd>means uppercase in the destination.
* <dt>STR_NOALIGN <dd>means don't do alignment.
* </dl>
*
* @param dest_len is the maximum length allowed in the
* destination. If dest_len is -1 then no maxiumum is used.
**/
ssize_t push_ucs2(void *dest, const char *src, size_t dest_len, int flags)
{
size_t len=0;
size_t src_len = strlen(src);
size_t ret;
if (flags & STR_UPPER) {
char *tmpbuf = strupper_talloc(NULL, src);
if (tmpbuf == NULL) {
return -1;
}
ret = push_ucs2(dest, tmpbuf, dest_len, flags & ~STR_UPPER);
talloc_free(tmpbuf);
return ret;
}
/* treat a pstring as "unlimited" length */
if (dest_len == (size_t)-1)
dest_len = sizeof(pstring);
if (flags & STR_TERMINATE)
src_len++;
if (ucs2_align(NULL, dest, flags)) {
*(char *)dest = 0;
dest = (void *)((char *)dest + 1);
if (dest_len) dest_len--;
len++;
}
/* ucs2 is always a multiple of 2 bytes */
dest_len &= ~1;
ret = convert_string(CH_UNIX, CH_UTF16, src, src_len, dest, dest_len);
if (ret == (size_t)-1) {
return 0;
}
len += ret;
return len;
}
/**
* Copy a string from a unix char* src to a UCS2 destination,
* allocating a buffer using talloc().
*
* @param dest always set at least to NULL
*
* @returns The number of bytes occupied by the string in the destination
* or -1 in case of error.
**/
ssize_t push_ucs2_talloc(TALLOC_CTX *ctx, void **dest, const char *src)
{
size_t src_len = strlen(src)+1;
*dest = NULL;
return convert_string_talloc(ctx, CH_UNIX, CH_UTF16, src, src_len, dest);
}
/**
* Copy a string from a unix char* src to a UTF-8 destination, allocating a buffer using talloc
*
* @param dest always set at least to NULL
*
* @returns The number of bytes occupied by the string in the destination
**/
ssize_t push_utf8_talloc(TALLOC_CTX *ctx, char **dest, const char *src)
{
size_t src_len = strlen(src)+1;
*dest = NULL;
return convert_string_talloc(ctx, CH_UNIX, CH_UTF8, src, src_len, (void **)dest);
}
/**
Copy a string from a ucs2 source to a unix char* destination.
Flags can have:
STR_TERMINATE means the string in src is null terminated.
STR_NOALIGN means don't try to align.
if STR_TERMINATE is set then src_len is ignored if it is -1.
src_len is the length of the source area in bytes
Return the number of bytes occupied by the string in src.
The resulting string in "dest" is always null terminated.
**/
size_t pull_ucs2(char *dest, const void *src, size_t dest_len, size_t src_len, int flags)
{
size_t ret;
if (dest_len == (size_t)-1)
dest_len = sizeof(pstring);
if (ucs2_align(NULL, src, flags)) {
src = (const void *)((const char *)src + 1);
if (src_len > 0)
src_len--;
}
if (flags & STR_TERMINATE) {
if (src_len == (size_t)-1) {
src_len = utf16_len(src);
} else {
src_len = utf16_len_n(src, src_len);
}
}
/* ucs2 is always a multiple of 2 bytes */
if (src_len != (size_t)-1)
src_len &= ~1;
ret = convert_string(CH_UTF16, CH_UNIX, src, src_len, dest, dest_len);
if (dest_len)
dest[MIN(ret, dest_len-1)] = 0;
return src_len;
}
ssize_t pull_ucs2_pstring(char *dest, const void *src)
{
return pull_ucs2(dest, src, sizeof(pstring), -1, STR_TERMINATE);
}
/**
* Copy a string from a UCS2 src to a unix char * destination, allocating a buffer using talloc
*
* @param dest always set at least to NULL
*
* @returns The number of bytes occupied by the string in the destination
**/
ssize_t pull_ucs2_talloc(TALLOC_CTX *ctx, char **dest, const void *src)
{
size_t src_len = utf16_len(src);
*dest = NULL;
return convert_string_talloc(ctx, CH_UTF16, CH_UNIX, src, src_len, (void **)dest);
}
/**
* Copy a string from a UTF-8 src to a unix char * destination, allocating a buffer using talloc
*
* @param dest always set at least to NULL
*
* @returns The number of bytes occupied by the string in the destination
**/
ssize_t pull_utf8_talloc(TALLOC_CTX *ctx, char **dest, const char *src)
{
size_t src_len = strlen(src)+1;
*dest = NULL;
return convert_string_talloc(ctx, CH_UTF8, CH_UNIX, src, src_len, (void **)dest);
}
/**
Copy a string from a char* src to a unicode or ascii
dos codepage destination choosing unicode or ascii based on the
flags in the SMB buffer starting at base_ptr.
Return the number of bytes occupied by the string in the destination.
flags can have:
STR_TERMINATE means include the null termination.
STR_UPPER means uppercase in the destination.
STR_ASCII use ascii even with unicode packet.
STR_NOALIGN means don't do alignment.
dest_len is the maximum length allowed in the destination. If dest_len
is -1 then no maxiumum is used.
**/
ssize_t push_string(void *dest, const char *src, size_t dest_len, int flags)
{
if (flags & STR_ASCII) {
return push_ascii(dest, src, dest_len, flags);
} else if (flags & STR_UNICODE) {
return push_ucs2(dest, src, dest_len, flags);
} else {
smb_panic("push_string requires either STR_ASCII or STR_UNICODE flag to be set");
return -1;
}
}
/**
Copy a string from a unicode or ascii source (depending on
the packet flags) to a char* destination.
Flags can have:
STR_TERMINATE means the string in src is null terminated.
STR_UNICODE means to force as unicode.
STR_ASCII use ascii even with unicode packet.
STR_NOALIGN means don't do alignment.
if STR_TERMINATE is set then src_len is ignored is it is -1
src_len is the length of the source area in bytes.
Return the number of bytes occupied by the string in src.
The resulting string in "dest" is always null terminated.
**/
ssize_t pull_string(char *dest, const void *src, size_t dest_len, size_t src_len, int flags)
{
if (flags & STR_ASCII) {
return pull_ascii(dest, src, dest_len, src_len, flags);
} else if (flags & STR_UNICODE) {
return pull_ucs2(dest, src, dest_len, src_len, flags);
} else {
smb_panic("pull_string requires either STR_ASCII or STR_UNICODE flag to be set");
return -1;
}
}
/*
return the unicode codepoint for the next multi-byte CH_UNIX character
in the string
also return the number of bytes consumed (which tells the caller
how many bytes to skip to get to the next CH_UNIX character)
return INVALID_CODEPOINT if the next character cannot be converted
*/
codepoint_t next_codepoint(const char *str, size_t *size)
{
/* it cannot occupy more than 4 bytes in UTF16 format */
uint8_t buf[4];
smb_iconv_t descriptor;
size_t ilen_orig;
size_t ilen;
size_t olen;
char *outbuf;
if ((str[0] & 0x80) == 0) {
*size = 1;
return (codepoint_t)str[0];
}
/* we assume that no multi-byte character can take
more than 5 bytes. This is OK as we only
support codepoints up to 1M */
ilen_orig = strnlen(str, 5);
ilen = ilen_orig;
descriptor = get_conv_handle(CH_UNIX, CH_UTF16);
if (descriptor == (smb_iconv_t)-1) {
*size = 1;
return INVALID_CODEPOINT;
}
/* this looks a little strange, but it is needed to cope
with codepoints above 64k */
olen = 2;
outbuf = buf;
smb_iconv(descriptor, &str, &ilen, &outbuf, &olen);
if (olen == 2) {
olen = 4;
outbuf = buf;
smb_iconv(descriptor, &str, &ilen, &outbuf, &olen);
if (olen == 4) {
/* we didn't convert any bytes */
*size = 1;
return INVALID_CODEPOINT;
}
olen = 4 - olen;
} else {
olen = 2 - olen;
}
*size = ilen_orig - ilen;
if (olen == 2) {
return (codepoint_t)SVAL(buf, 0);
}
if (olen == 4) {
/* decode a 4 byte UTF16 character manually */
return (codepoint_t)0x10000 +
(buf[2] | ((buf[3] & 0x3)<<8) |
(buf[0]<<10) | ((buf[1] & 0x3)<<18));
}
/* no other length is valid */
return INVALID_CODEPOINT;
}
/*
push a single codepoint into a CH_UNIX string the target string must
be able to hold the full character, which is guaranteed if it is at
least 5 bytes in size. The caller may pass less than 5 bytes if they
are sure the character will fit (for example, you can assume that
uppercase/lowercase of a character will not add more than 1 byte)
return the number of bytes occupied by the CH_UNIX character, or
-1 on failure
*/
ssize_t push_codepoint(char *str, codepoint_t c)
{
smb_iconv_t descriptor;
uint8_t buf[4];
size_t ilen, olen;
const char *inbuf;
if (c < 128) {
*str = c;
return 1;
}
descriptor = get_conv_handle(CH_UTF16, CH_UNIX);
if (descriptor == (smb_iconv_t)-1) {
return -1;
}
if (c < 0x10000) {
ilen = 2;
olen = 5;
inbuf = buf;
SSVAL(buf, 0, c);
smb_iconv(descriptor, &inbuf, &ilen, &str, &olen);
if (ilen != 0) {
return -1;
}
return 5 - olen;
}
c -= 0x10000;
buf[0] = (c>>10) & 0xFF;
buf[1] = (c>>18) | 0xd8;
buf[2] = c & 0xFF;
buf[3] = ((c>>8) & 0x3) | 0xdc;
ilen = 4;
olen = 5;
inbuf = buf;
smb_iconv(descriptor, &inbuf, &ilen, &str, &olen);
if (ilen != 0) {
return -1;
}
return 5 - olen;
}