c++中unicode utf8, gb2312还有url_encoding

最新推荐文章于 2025-07-24 20:33:53 发布

binwind

最新推荐文章于 2025-07-24 20:33:53 发布

阅读量3.5k

点赞数

CC 4.0 BY-SA版权

文章标签： encoding url c++ buffer input output

本文链接：https://blog.youkuaiyun.com/binwind/article/details/1785772

本文详细介绍了Unicode和UTF-8编码的概念及其相互转换的方法。包括如何将Unicode转换为UTF-8，反之亦然，并提供了具体的实现代码。此外，还讲解了URL编码的作用及其实现。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

我这里说的unicode指的是在windows下用wchar_t表示的那个unicode, 也就是L"xxx"中表示的那个unicode,也就是在编译选项里面用unicode 并且_T("XXX") 中的unicode
下面的代码

TCHAR english[] = _T( " Hello " );

TCHAR chinese[] = _T( " 你好 " );

char chineseA[] = " 你好 " ;

char enchlishA[] = " hello " ;

wchar_t chineseW[] = L " 你好 " ;

wchar_t englishW[] = L " hello "

在visual c++中，根本不管utf8, 在没有定义unicode时候编译一下代码
TCHAR和_T就翻译成为了普通的char和“”
在定义了unicode的时候
TCHAR和_T就翻译成为了wchar_t和L""
其中char中指定中文的时候,中文使用gb2312编码中文占2个字节，而指定英文的时候，英文占1个字节，是标准的ascII, 但是gb2312编码国际化不好，就是说各国语言的编码可能有重叠的部分，就是说2字节的东西，同一个2字节的数字可能对应不同的编码中的不同的符号所以才出现了乱码
所以就有了unicode了
在 wchar_t中无论中文英文（还有日文等等）都是同一作为2个字节，好处在于无论中文英文日文一个字符都有一个统一的不同的数字,(unicode = unique code咯～～)其实wchar_t的本质上就是unsigned short

虽然unicode解决了统一编码的问题，但是不好啊，因为他定长的，就好象我们数据压缩中的如果用定长编码
的效率肯定不如变长编码高了，所以utf8就产生了
utf8本质上就是unicode的编程编码，具体的编码解我们可以一下的link
http://zhidao.baidu.com/question/2692826.html
而url_encoding是在网络传输中为了避开一些特殊的字符(比如&，/n等等)，所以使用的编码方式，他是在utf8或者unicode智商的编码，就是说无论我们在网络上传送的是utf8或者是unicode我们都可以通过url_encoding把他在发送端编码并且在接收端解码，（当然了我们也可以用url_encoding来传送2进制流)url_encoding每8位编码一次(就是吧除了a~z,A~Z,1~9还有几个其他符号(我这里说的是转换成为int的字符)转换成%XX的16进制表达方式

也就是说utf8 和 unicode, gb2132是平行的概念，这几个概念是相对的，是在操作系统层的编码，而url_encoding是在这2者上一层次的编码，用来把某段数据数据在网络上发送出去。是在网络层的编码(这段数据可以是utf-8也可以是unicode，也可以是gb2312也可以是其他的如gbk, jp等等的编码)

关于url_encoding我们可以看
http://www.albionresearch.com/misc/urlencode.php
utf8, unicode, urlencoding转换请看代码(unicode的urlencoding我就不想写了。懒得写了， gb2312转uicode可以通过bstr_t类转换，windows也带有了相应的api)
.h文件

class EncodingConvertor {

public:

EncodingConvertor(void);

public:

~EncodingConvertor(void);

public:

bool Unicode2UTF8(const wchar_t *input_unicode,char ** p_output_utf8,unsigned long *length);

bool UTF82Unicode(const char *input_utf8, wchar_t ** p_output_unicode, unsigned long *length);

bool UTF82UrlEncode(const char* input_utf8, char ** p_url_encode, unsigned long *length);

} ;

.cpp文件

include < stdlib.h >

#include " EncodingConvertor.h "

EncodingConvertor::EncodingConvertor( void ) {

}

EncodingConvertor:: ~ EncodingConvertor( void ) {

}

bool EncodingConvertor::Unicode2UTF8( const wchar_t * input_unicode,

char ** p_output_utf8,

unsigned long * length) {

if (input_unicode == NULL) {

return true;

}

int size_d = 8;

int buffer_size = 0;

const wchar_t* p_unicode = input_unicode;

// count for the space need to allocate

wchar_t w_char;

do {

w_char = *p_unicode;

if (w_char < 0x80) {

// utf char size is 1

buffer_size += 1;

} else if (w_char < 0x800) {

// utf char size is 2

buffer_size += 2;

} else if (w_char < 0x10000) {

// utf char size is 3

buffer_size += 3;

} else if (w_char < 0x200000) {

// utf char size is 4

buffer_size += 4;

} else if (w_char < 0x4000000) {

// utf char size is 5

buffer_size += 5;

} else {

// utf char size is 6

buffer_size += 6;

}

p_unicode++;

}

while (w_char != static_cast<char>(0));

// allocate the memory

char* utf8 = new char[buffer_size];

p_unicode = input_unicode;

int index_buffer = 0;

// do the conversion

do {

w_char = *input_unicode; // the unicode char current being converted

input_unicode++;

if (w_char < 0x80) {

// length = 1;

utf8[index_buffer++] = static_cast<char>(w_char);

} else if (w_char < 0x800) {

// length = 2;

utf8[index_buffer++] = 0xc0 | (w_char >> 6);

utf8[index_buffer++] = 0x80 | (w_char & 0x3f);

} else if (w_char < 0x10000) {

// length = 3;

utf8[index_buffer++] = 0xe0 | (w_char >> 12);

utf8[index_buffer++] = 0x80 | ((w_char >> 6) & 0x3f);

utf8[index_buffer++] = 0x80 | (w_char & 0x3f);

} else if (w_char < 0x200000) {

// length = 4;

utf8[index_buffer++] = 0xf0 | (static_cast<int>(w_char) >> 18);

utf8[index_buffer++] = 0x80 | ((w_char >> 12) & 0x3f);

utf8[index_buffer++] = 0x80 | ((w_char >> 6) & 0x3f);

utf8[index_buffer++] = 0x80 | (w_char & 0x3f);

} else if (w_char < 0x4000000) {

// length = 5

utf8[index_buffer++] = 0xf8| (static_cast<int>(w_char) >> 24);

utf8[index_buffer++] = 0x80 | ((static_cast<int>(w_char) >> 18) & 0x3f);

utf8[index_buffer++] = 0x80 | ((w_char >> 12) & 0x3f);

utf8[index_buffer++] = 0x80 | ((w_char >> 6) & 0x3f);

utf8[index_buffer++] = 0x80 | (w_char & 0x3f);

} else { // if(wchar >= 0x4000000)

// all other cases length = 6

utf8[index_buffer++] = 0xfc | (static_cast<int>(w_char) >> 30);

utf8[index_buffer++] = 0x80 | ((static_cast<int>(w_char) >> 24) & 0x3f);

utf8[index_buffer++] = 0x80 | ((static_cast<int>(w_char) >> 18) & 0x3f);

utf8[index_buffer++] = 0x80 | ((w_char >> 12) & 0x3f);

utf8[index_buffer++] = 0x80 | ((w_char >> 6) & 0x3f);

utf8[index_buffer++] = 0x80 | (w_char & 0x3f);

}

while (w_char != static_cast<char>(0));

// set the output length

*length = buffer_size - 1; // ignore last

// set the output charset

*p_output_utf8 = utf8;

return false;

}

bool EncodingConvertor::UTF82Unicode( const char * input_utf8,

wchar_t ** p_output_unicode,

unsigned long * length) {

if (input_utf8 == NULL) { // input wrong.

return false;

}

const char* p_current_char = input_utf8;

unsigned long unicode_length = 0;

char current_char;

// calculate the size to locate

do {

// get the begining char

current_char = *p_current_char;

if ((current_char & 0x80) == 0) {

// 0xxxxxxx

p_current_char++;

} else if ((current_char & 0xe0) == 0xc0) {

// < 110x-xxxx 10xx-xxxx

p_current_char += 2;

} else if ((current_char & 0xf0) == 0xe0) {

// < 1110-xxxx 10xx-xxxx 10xx-xxxx

p_current_char += 3;

} else if ((current_char & 0xf8) == 0xf0) {

// < 1111-0xxx 10xx-xxxx 10xx-xxxx 10xx-xxxx

p_current_char += 4;

} else if ((current_char & 0xfc) == 0xf8) {

// 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx

p_current_char += 5;

} else {

// if((current_char & 0xfe) == 0xfc)

// 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx

p_current_char += 6;

}

unicode_length++;

}

while (current_char != 0);

wchar_t* des = new wchar_t[unicode_length];

unsigned long unicode_index = 0;

p_current_char = input_utf8;

do {

current_char = *p_current_char;

if ((current_char & 0x80) == 0) {

des[unicode_index] = p_current_char[0];

p_current_char++;

} else if ((current_char & 0xE0) == 0xC0) {

// < 110x-xxxx 10xx-xxxx

wchar_t &wide_char = des[unicode_index];

wide_char = (p_current_char[0] & 0x3F) << 6;

wide_char |= (p_current_char[1] & 0x3F);

p_current_char += 2;

} else if ((current_char & 0xF0) == 0xE0) {

// < 1110-xxxx 10xx-xxxx 10xx-xxxx

wchar_t &wide_char = des[unicode_index];

wide_char = (p_current_char[0] & 0x1F) << 12;

wide_char |= (p_current_char[1] & 0x3F) << 6;

wide_char |= (p_current_char[2] & 0x3F);

p_current_char += 3;

} else if ((current_char & 0xF8) == 0xF0) {

// < 1111-0xxx 10xx-xxxx 10xx-xxxx 10xx-xxxx

wchar_t &wide_char = des[unicode_index];

wide_char = (p_current_char[0] & 0x0F) << 18;

wide_char |= (p_current_char[1] & 0x3F) << 12;

wide_char |= (p_current_char[2] & 0x3F) << 6;

wide_char |= (p_current_char[3] & 0x3F);

p_current_char += 4;

} else if ((current_char & 0xfc) == 0xf8) {

// 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx

wchar_t &wide_char = des[unicode_index];

wide_char = (p_current_char[0] & 0x07) << 24;

wide_char |= (p_current_char[1] & 0x3F) << 18;

wide_char |= (p_current_char[2] & 0x3F) << 12;

wide_char |= (p_current_char[3] & 0x3F) << 6;

wide_char |= (p_current_char[4] & 0x3F);

p_current_char += 5;

} else {

// if((*current_char & 0xfe) == 0xfc)

// 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx

wchar_t &wide_char = des[unicode_index];

wide_char = (p_current_char[0] & 0x03) << 30;

wide_char |= (p_current_char[1] & 0x3F) << 24;

wide_char |= (p_current_char[2] & 0x3F) << 18;

wide_char |= (p_current_char[3] & 0x3F) << 12;

wide_char |= (p_current_char[4] & 0x3F) << 6;

wide_char |= (p_current_char[5] & 0x3F);

p_current_char += 6;

}

unicode_index++;

} while (current_char != 0);

*p_output_unicode = des;

*length = unicode_length - 1; // ignore the last

return true;

}

bool EncodingConvertor::UTF82UrlEncode( const char * input_utf8,

char ** p_url_encode,

unsigned long * length) {

unsigned long total_length = 0;

// calculate output size

const char* current_pos = input_utf8;

do {

// the char needs to be encode

if (!((*current_pos) >= 'a' && (*current_pos) <= 'z') &&

!((*current_pos) >= 'A' && (*current_pos) <= 'Z') &&

!((*current_pos) >= '0' && (*current_pos) <= '9') &&

!((*current_pos) == ' ') &&

!((*current_pos) == '.') &&

!((*current_pos) == '-') &&

!((*current_pos) == '_') &&

!((*current_pos) == '~') &&

!((*current_pos) == ''') &&

!((*current_pos) == '(') &&

!((*current_pos) == ')') &&

!((*current_pos) == '*') &&

!((*current_pos) == 0)

) {

total_length += 3;

} else {

total_length++;

}

if ((*current_pos) == 0) { // the end of str

break;

}

current_pos++;

} while (true);

char* output_buffer = new char[total_length];

current_pos = input_utf8;

char* target_current_pos = output_buffer;

unsigned long left_size = total_length;

// do conversion

do {

// the char needs to be encode

if (!((*current_pos) >= 'a' && (*current_pos) <= 'z') &&

!((*current_pos) >= 'A' && (*current_pos) <= 'Z') &&

!((*current_pos) >= '0' && (*current_pos) <= '9') &&

!((*current_pos) == ' ') &&

!((*current_pos) == '.') &&

!((*current_pos) == '-') &&

!((*current_pos) == '_') &&

!((*current_pos) == '~') &&

!((*current_pos) == ''') &&

!((*current_pos) == '(') &&

!((*current_pos) == ')') &&

!((*current_pos) == '*') &&

!((*current_pos) == 0)

) {

unsigned char temp_unsigned_char = *current_pos;

// convert to unsigned to skip the sign bit

::_snprintf_s(target_current_pos,

left_size,

_TRUNCATE ,

"%%%02X",

temp_unsigned_char);

target_current_pos += 3;

left_size -= 3;

} else if ((*current_pos) == ' ') {

// the char blank

*target_current_pos = '+';

target_current_pos++;

left_size--;

} else {

// char that need not change

*target_current_pos = *current_pos;

target_current_pos++;

left_size--;

}

if ((*current_pos) == 0) {

break;

}

current_pos++;

} while (true);

*p_url_encode = output_buffer;

*length = total_length - 1; // ignore last

return true;

}