#include "Python.h"

#include "pycore_pystate.h"

#include "ucnhash.h"

#include <ctype.h>

const char *Py_hexdigits = "0123456789abcdef";

static int _PyCodecRegistry_Init(void); /* Forward */

int PyCodec_Register(PyObject *search_function)

{

PyInterpreterState *interp = _PyInterpreterState_Get();

if (interp->codec_search_path == NULL && _PyCodecRegistry_Init())

goto onError;

if (search_function == NULL) {

PyErr_BadArgument();

goto onError;

}

if (!PyCallable_Check(search_function)) {

PyErr_SetString(PyExc_TypeError, "argument must be callable");

goto onError;

}

return PyList_Append(interp->codec_search_path, search_function);

onError:

return -1;

}

PyObject *normalizestring(const char *string)

{

size_t i;

size_t len = strlen(string);

char *p;

PyObject *v;

if (len > PY_SSIZE_T_MAX) {

PyErr_SetString(PyExc_OverflowError, "string is too large");

return NULL;

}

p = PyMem_Malloc(len + 1);

if (p == NULL)

return PyErr_NoMemory();

for (i = 0; i < len; i++) {

char ch = string[i];

if (ch == ' ')

ch = '-';

else

ch = Py_TOLOWER(Py_CHARMASK(ch));

p[i] = ch;

}

p[i] = '\0';

v = PyUnicode_FromString(p);

PyMem_Free(p);

return v;

}

PyObject *_PyCodec_Lookup(const char *encoding)

{

PyObject *result, *args = NULL, *v;

Py_ssize_t i, len;

if (encoding == NULL) {

PyErr_BadArgument();

goto onError;

}

PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE();

if (interp->codec_search_path == NULL && _PyCodecRegistry_Init())

goto onError;

/* Convert the encoding to a normalized Python string: all

v = normalizestring(encoding);

if (v == NULL)

goto onError;

PyUnicode_InternInPlace(&v);

/* First, try to lookup the name in the registry dictionary */

result = PyDict_GetItemWithError(interp->codec_search_cache, v);

if (result != NULL) {

Py_INCREF(result);

Py_DECREF(v);

return result;

}

else if (PyErr_Occurred()) {

Py_DECREF(v);

return NULL;

}

/* Next, scan the search functions in order of registration */

args = PyTuple_New(1);

if (args == NULL) {

Py_DECREF(v);

return NULL;

}

PyTuple_SET_ITEM(args,0,v);

len = PyList_Size(interp->codec_search_path);

if (len < 0)

goto onError;

if (len == 0) {

PyErr_SetString(PyExc_LookupError,

"no codec search functions registered: "

"can't find encoding");

goto onError;

}

for (i = 0; i < len; i++) {

PyObject *func;

func = PyList_GetItem(interp->codec_search_path, i);

if (func == NULL)

goto onError;

result = PyEval_CallObject(func, args);

if (result == NULL)

goto onError;

if (result == Py_None) {

Py_DECREF(result);

continue;

}

if (!PyTuple_Check(result) || PyTuple_GET_SIZE(result) != 4) {

PyErr_SetString(PyExc_TypeError,

"codec search functions must return 4-tuples");

Py_DECREF(result);

goto onError;

}

break;

}

if (i == len) {

/* XXX Perhaps we should cache misses too ? */

PyErr_Format(PyExc_LookupError,

"unknown encoding: %s", encoding);

goto onError;

}

/* Cache and return the result */

if (PyDict_SetItem(interp->codec_search_cache, v, result) < 0) {

Py_DECREF(result);

goto onError;

}

Py_DECREF(args);

return result;

onError:

Py_XDECREF(args);

return NULL;

}

int _PyCodec_Forget(const char *encoding)

{

PyObject *v;

int result;

PyInterpreterState *interp = _PyInterpreterState_Get();

if (interp->codec_search_path == NULL) {

return -1;

}

/* Convert the encoding to a normalized Python string: all

v = normalizestring(encoding);

if (v == NULL) {

return -1;

}

/* Drop the named codec from the internal cache */

result = PyDict_DelItem(interp->codec_search_cache, v);

Py_DECREF(v);

return result;

}

int PyCodec_KnownEncoding(const char *encoding)

{

PyObject *codecs;

codecs = _PyCodec_Lookup(encoding);

if (!codecs) {

PyErr_Clear();

return 0;

}

else {

Py_DECREF(codecs);

return 1;

}

}

PyObject *args_tuple(PyObject *object,

const char *errors)

{

PyObject *args;

args = PyTuple_New(1 + (errors != NULL));

if (args == NULL)

return NULL;

Py_INCREF(object);

PyTuple_SET_ITEM(args,0,object);

if (errors) {

PyObject *v;

v = PyUnicode_FromString(errors);

if (v == NULL) {

Py_DECREF(args);

return NULL;

}

PyTuple_SET_ITEM(args, 1, v);

}

return args;

}

PyObject *codec_getitem(const char *encoding, int index)

{

PyObject *codecs;

PyObject *v;

codecs = _PyCodec_Lookup(encoding);

if (codecs == NULL)

return NULL;

v = PyTuple_GET_ITEM(codecs, index);

Py_DECREF(codecs);

Py_INCREF(v);

return v;

}

PyObject *codec_makeincrementalcodec(PyObject *codec_info,

const char *errors,

const char *attrname)

{

PyObject *ret, *inccodec;

inccodec = PyObject_GetAttrString(codec_info, attrname);

if (inccodec == NULL)

return NULL;

if (errors)

ret = PyObject_CallFunction(inccodec, "s", errors);

else

ret = _PyObject_CallNoArg(inccodec);

Py_DECREF(inccodec);

return ret;

}

PyObject *codec_getincrementalcodec(const char *encoding,

const char *errors,

const char *attrname)

{

PyObject *codec_info, *ret;

codec_info = _PyCodec_Lookup(encoding);

if (codec_info == NULL)

return NULL;

ret = codec_makeincrementalcodec(codec_info, errors, attrname);

Py_DECREF(codec_info);

return ret;

}

PyObject *codec_getstreamcodec(const char *encoding,

PyObject *stream,

const char *errors,

const int index)

{

PyObject *codecs, *streamcodec, *codeccls;

codecs = _PyCodec_Lookup(encoding);

if (codecs == NULL)

return NULL;

codeccls = PyTuple_GET_ITEM(codecs, index);

if (errors != NULL)

streamcodec = PyObject_CallFunction(codeccls, "Os", stream, errors);

else

streamcodec = PyObject_CallFunctionObjArgs(codeccls, stream, NULL);

Py_DECREF(codecs);

return streamcodec;

}

PyObject *_PyCodecInfo_GetIncrementalDecoder(PyObject *codec_info,

const char *errors)

{

return codec_makeincrementalcodec(codec_info, errors,

"incrementaldecoder");

}

PyObject *_PyCodecInfo_GetIncrementalEncoder(PyObject *codec_info,

const char *errors)

{

return codec_makeincrementalcodec(codec_info, errors,

"incrementalencoder");

}

PyObject *PyCodec_Encoder(const char *encoding)

{

return codec_getitem(encoding, 0);

}

PyObject *PyCodec_Decoder(const char *encoding)

{

return codec_getitem(encoding, 1);

}

PyObject *PyCodec_IncrementalEncoder(const char *encoding,

const char *errors)

{

return codec_getincrementalcodec(encoding, errors, "incrementalencoder");

}

PyObject *PyCodec_IncrementalDecoder(const char *encoding,

const char *errors)

{

return codec_getincrementalcodec(encoding, errors, "incrementaldecoder");

}

PyObject *PyCodec_StreamReader(const char *encoding,

PyObject *stream,

const char *errors)

{

return codec_getstreamcodec(encoding, stream, errors, 2);

}

PyObject *PyCodec_StreamWriter(const char *encoding,

PyObject *stream,

const char *errors)

{

return codec_getstreamcodec(encoding, stream, errors, 3);

}

static void

wrap_codec_error(const char *operation,

const char *encoding)

{

/* TrySetFromCause will replace the active exception with a suitably

_PyErr_TrySetFromCause("%s with '%s' codec failed",

operation, encoding);

}

static PyObject *

_PyCodec_EncodeInternal(PyObject *object,

PyObject *encoder,

const char *encoding,

const char *errors)

{

PyObject *args = NULL, *result = NULL;

PyObject *v = NULL;

args = args_tuple(object, errors);

if (args == NULL)

goto onError;

result = PyEval_CallObject(encoder, args);

if (result == NULL) {

wrap_codec_error("encoding", encoding);

goto onError;

}

if (!PyTuple_Check(result) ||

PyTuple_GET_SIZE(result) != 2) {

PyErr_SetString(PyExc_TypeError,

"encoder must return a tuple (object, integer)");

goto onError;

}

v = PyTuple_GET_ITEM(result,0);

Py_INCREF(v);

/* We don't check or use the second (integer) entry. */

Py_DECREF(args);

Py_DECREF(encoder);

Py_DECREF(result);

return v;

onError:

Py_XDECREF(result);

Py_XDECREF(args);

Py_XDECREF(encoder);

return NULL;

}

static PyObject *

_PyCodec_DecodeInternal(PyObject *object,

PyObject *decoder,

const char *encoding,

const char *errors)

{

PyObject *args = NULL, *result = NULL;

PyObject *v;

args = args_tuple(object, errors);

if (args == NULL)

goto onError;

result = PyEval_CallObject(decoder,args);

if (result == NULL) {

wrap_codec_error("decoding", encoding);

goto onError;

}

if (!PyTuple_Check(result) ||

PyTuple_GET_SIZE(result) != 2) {

PyErr_SetString(PyExc_TypeError,

"decoder must return a tuple (object,integer)");

goto onError;

}

v = PyTuple_GET_ITEM(result,0);

Py_INCREF(v);

/* We don't check or use the second (integer) entry. */

Py_DECREF(args);

Py_DECREF(decoder);

Py_DECREF(result);

return v;

onError:

Py_XDECREF(args);

Py_XDECREF(decoder);

Py_XDECREF(result);

return NULL;

}

PyObject *PyCodec_Encode(PyObject *object,

const char *encoding,

const char *errors)

{

PyObject *encoder;

encoder = PyCodec_Encoder(encoding);

if (encoder == NULL)

return NULL;

return _PyCodec_EncodeInternal(object, encoder, encoding, errors);

}

PyObject *PyCodec_Decode(PyObject *object,

const char *encoding,

const char *errors)

{

PyObject *decoder;

decoder = PyCodec_Decoder(encoding);

if (decoder == NULL)

return NULL;

return _PyCodec_DecodeInternal(object, decoder, encoding, errors);

}

PyObject * _PyCodec_LookupTextEncoding(const char *encoding,

const char *alternate_command)

{

_Py_IDENTIFIER(_is_text_encoding);

PyObject *codec;

PyObject *attr;

int is_text_codec;

codec = _PyCodec_Lookup(encoding);

if (codec == NULL)

return NULL;

/* Backwards compatibility: assume any raw tuple describes a text

if (!PyTuple_CheckExact(codec)) {

if (_PyObject_LookupAttrId(codec, &PyId__is_text_encoding, &attr) < 0) {

Py_DECREF(codec);

return NULL;

}

if (attr != NULL) {

is_text_codec = PyObject_IsTrue(attr);

Py_DECREF(attr);

if (is_text_codec <= 0) {

Py_DECREF(codec);

if (!is_text_codec)

PyErr_Format(PyExc_LookupError,

"'%.400s' is not a text encoding; "

"use %s to handle arbitrary codecs",

encoding, alternate_command);

return NULL;

}

}

}

/* This appears to be a valid text encoding */

return codec;

}

PyObject *codec_getitem_checked(const char *encoding,

const char *alternate_command,

int index)

{

PyObject *codec;

PyObject *v;

codec = _PyCodec_LookupTextEncoding(encoding, alternate_command);

if (codec == NULL)

return NULL;

v = PyTuple_GET_ITEM(codec, index);

Py_INCREF(v);

Py_DECREF(codec);

return v;

}

static PyObject * _PyCodec_TextEncoder(const char *encoding)

{

return codec_getitem_checked(encoding, "codecs.encode()", 0);

}

static PyObject * _PyCodec_TextDecoder(const char *encoding)

{

return codec_getitem_checked(encoding, "codecs.decode()", 1);

}

PyObject *_PyCodec_EncodeText(PyObject *object,

const char *encoding,

const char *errors)

{

PyObject *encoder;

encoder = _PyCodec_TextEncoder(encoding);

if (encoder == NULL)

return NULL;

return _PyCodec_EncodeInternal(object, encoder, encoding, errors);

}

PyObject *_PyCodec_DecodeText(PyObject *object,

const char *encoding,

const char *errors)

{

PyObject *decoder;

decoder = _PyCodec_TextDecoder(encoding);

if (decoder == NULL)

return NULL;

return _PyCodec_DecodeInternal(object, decoder, encoding, errors);

}

int PyCodec_RegisterError(const char *name, PyObject *error)

{

PyInterpreterState *interp = _PyInterpreterState_Get();

if (interp->codec_search_path == NULL && _PyCodecRegistry_Init())

return -1;

if (!PyCallable_Check(error)) {

PyErr_SetString(PyExc_TypeError, "handler must be callable");

return -1;

}

return PyDict_SetItemString(interp->codec_error_registry,

name, error);

}

PyObject *PyCodec_LookupError(const char *name)

{

PyObject *handler = NULL;

PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE();

if (interp->codec_search_path == NULL && _PyCodecRegistry_Init())

return NULL;

if (name==NULL)

name = "strict";

handler = _PyDict_GetItemStringWithError(interp->codec_error_registry, name);

if (handler) {

Py_INCREF(handler);

}

else if (!PyErr_Occurred()) {

PyErr_Format(PyExc_LookupError, "unknown error handler name '%.400s'", name);

}

return handler;

}

static void wrong_exception_type(PyObject *exc)

{

PyErr_Format(PyExc_TypeError,

"don't know how to handle %.200s in error callback",

exc->ob_type->tp_name);

}

PyObject *PyCodec_StrictErrors(PyObject *exc)

{

if (PyExceptionInstance_Check(exc))

PyErr_SetObject(PyExceptionInstance_Class(exc), exc);

else

PyErr_SetString(PyExc_TypeError, "codec must pass exception instance");

return NULL;

}

PyObject *PyCodec_IgnoreErrors(PyObject *exc)

{

Py_ssize_t end;

if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) {

if (PyUnicodeEncodeError_GetEnd(exc, &end))

return NULL;

}

else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeDecodeError)) {

if (PyUnicodeDecodeError_GetEnd(exc, &end))

return NULL;

}

else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeTranslateError)) {

if (PyUnicodeTranslateError_GetEnd(exc, &end))

return NULL;

}

else {

wrong_exception_type(exc);

return NULL;

}

return Py_BuildValue("(Nn)", PyUnicode_New(0, 0), end);

}

PyObject *PyCodec_ReplaceErrors(PyObject *exc)

{

Py_ssize_t start, end, i, len;

if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) {

PyObject *res;

int kind;

void *data;

if (PyUnicodeEncodeError_GetStart(exc, &start))

return NULL;

if (PyUnicodeEncodeError_GetEnd(exc, &end))

return NULL;

len = end - start;

res = PyUnicode_New(len, '?');

if (res == NULL)

return NULL;

kind = PyUnicode_KIND(res);

data = PyUnicode_DATA(res);

for (i = 0; i < len; ++i)

PyUnicode_WRITE(kind, data, i, '?');

assert(_PyUnicode_CheckConsistency(res, 1));

return Py_BuildValue("(Nn)", res, end);

}

else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeDecodeError)) {

if (PyUnicodeDecodeError_GetEnd(exc, &end))

return NULL;

return Py_BuildValue("(Cn)",

(int)Py_UNICODE_REPLACEMENT_CHARACTER,

end);

}

else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeTranslateError)) {

PyObject *res;

int kind;

void *data;

if (PyUnicodeTranslateError_GetStart(exc, &start))

return NULL;

if (PyUnicodeTranslateError_GetEnd(exc, &end))

return NULL;

len = end - start;

res = PyUnicode_New(len, Py_UNICODE_REPLACEMENT_CHARACTER);

if (res == NULL)

return NULL;

kind = PyUnicode_KIND(res);

data = PyUnicode_DATA(res);

for (i=0; i < len; i++)

PyUnicode_WRITE(kind, data, i, Py_UNICODE_REPLACEMENT_CHARACTER);

assert(_PyUnicode_CheckConsistency(res, 1));

return Py_BuildValue("(Nn)", res, end);

}

else {

wrong_exception_type(exc);

return NULL;

}

}

PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc)

{

if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) {

PyObject *restuple;

PyObject *object;

Py_ssize_t i;

Py_ssize_t start;

Py_ssize_t end;

PyObject *res;

unsigned char *outp;

Py_ssize_t ressize;

Py_UCS4 ch;

if (PyUnicodeEncodeError_GetStart(exc, &start))

return NULL;

if (PyUnicodeEncodeError_GetEnd(exc, &end))

return NULL;

if (!(object = PyUnicodeEncodeError_GetObject(exc)))

return NULL;

if (end - start > PY_SSIZE_T_MAX / (2+7+1))

end = start + PY_SSIZE_T_MAX / (2+7+1);

for (i = start, ressize = 0; i < end; ++i) {

/* object is guaranteed to be "ready" */

ch = PyUnicode_READ_CHAR(object, i);

if (ch<10)

ressize += 2+1+1;

else if (ch<100)

ressize += 2+2+1;

else if (ch<1000)

ressize += 2+3+1;

else if (ch<10000)

ressize += 2+4+1;

else if (ch<100000)

ressize += 2+5+1;

else if (ch<1000000)

ressize += 2+6+1;

else

ressize += 2+7+1;

}

/* allocate replacement */

res = PyUnicode_New(ressize, 127);

if (res == NULL) {

Py_DECREF(object);

return NULL;

}

outp = PyUnicode_1BYTE_DATA(res);

/* generate replacement */

for (i = start; i < end; ++i) {

int digits;

int base;

ch = PyUnicode_READ_CHAR(object, i);

*outp++ = '&';

*outp++ = '#';

if (ch<10) {

digits = 1;

base = 1;

}

else if (ch<100) {

digits = 2;

base = 10;

}

else if (ch<1000) {

digits = 3;

base = 100;

}

else if (ch<10000) {

digits = 4;

base = 1000;

}

else if (ch<100000) {

digits = 5;

base = 10000;

}

else if (ch<1000000) {

digits = 6;

base = 100000;

}

else {

digits = 7;

base = 1000000;

}

while (digits-->0) {

*outp++ = '0' + ch/base;

ch %= base;

base /= 10;

}

*outp++ = ';';

}

assert(_PyUnicode_CheckConsistency(res, 1));

restuple = Py_BuildValue("(Nn)", res, end);

Py_DECREF(object);

return restuple;

}

else {

wrong_exception_type(exc);

return NULL;

}

}

PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc)

{

PyObject *object;

Py_ssize_t i;

Py_ssize_t start;

Py_ssize_t end;

PyObject *res;

unsigned char *outp;

int ressize;

Py_UCS4 c;

if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeDecodeError)) {

const unsigned char *p;

if (PyUnicodeDecodeError_GetStart(exc, &start))

return NULL;

if (PyUnicodeDecodeError_GetEnd(exc, &end))

return NULL;

if (!(object = PyUnicodeDecodeError_GetObject(exc)))

return NULL;

p = (const unsigned char*)PyBytes_AS_STRING(object);

res = PyUnicode_New(4 * (end - start), 127);

if (res == NULL) {

Py_DECREF(object);

return NULL;

}

outp = PyUnicode_1BYTE_DATA(res);

for (i = start; i < end; i++, outp += 4) {

unsigned char c = p[i];

outp[0] = '\\';

outp[1] = 'x';

outp[2] = Py_hexdigits[(c>>4)&0xf];

outp[3] = Py_hexdigits[c&0xf];

}

assert(_PyUnicode_CheckConsistency(res, 1));

Py_DECREF(object);

return Py_BuildValue("(Nn)", res, end);

}

if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) {

if (PyUnicodeEncodeError_GetStart(exc, &start))

return NULL;

if (PyUnicodeEncodeError_GetEnd(exc, &end))

return NULL;

if (!(object = PyUnicodeEncodeError_GetObject(exc)))

return NULL;

}

else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeTranslateError)) {

if (PyUnicodeTranslateError_GetStart(exc, &start))

return NULL;

if (PyUnicodeTranslateError_GetEnd(exc, &end))

return NULL;

if (!(object = PyUnicodeTranslateError_GetObject(exc)))

return NULL;

}

else {

wrong_exception_type(exc);

return NULL;

}

if (end - start > PY_SSIZE_T_MAX / (1+1+8))

end = start + PY_SSIZE_T_MAX / (1+1+8);

for (i = start, ressize = 0; i < end; ++i) {

/* object is guaranteed to be "ready" */

c = PyUnicode_READ_CHAR(object, i);

if (c >= 0x10000) {

ressize += 1+1+8;

}

else if (c >= 0x100) {

ressize += 1+1+4;

}

else

ressize += 1+1+2;

}

res = PyUnicode_New(ressize, 127);

if (res == NULL) {

Py_DECREF(object);

return NULL;

}

outp = PyUnicode_1BYTE_DATA(res);

for (i = start; i < end; ++i) {

c = PyUnicode_READ_CHAR(object, i);

*outp++ = '\\';

if (c >= 0x00010000) {

*outp++ = 'U';

*outp++ = Py_hexdigits[(c>>28)&0xf];

*outp++ = Py_hexdigits[(c>>24)&0xf];

*outp++ = Py_hexdigits[(c>>20)&0xf];

*outp++ = Py_hexdigits[(c>>16)&0xf];

*outp++ = Py_hexdigits[(c>>12)&0xf];

*outp++ = Py_hexdigits[(c>>8)&0xf];

}

else if (c >= 0x100) {

*outp++ = 'u';

*outp++ = Py_hexdigits[(c>>12)&0xf];

*outp++ = Py_hexdigits[(c>>8)&0xf];

}

else

*outp++ = 'x';

*outp++ = Py_hexdigits[(c>>4)&0xf];

*outp++ = Py_hexdigits[c&0xf];

}

assert(_PyUnicode_CheckConsistency(res, 1));

Py_DECREF(object);

return Py_BuildValue("(Nn)", res, end);

}

static _PyUnicode_Name_CAPI *ucnhash_CAPI = NULL;

PyObject *PyCodec_NameReplaceErrors(PyObject *exc)

{

if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) {

PyObject *restuple;

PyObject *object;

Py_ssize_t i;

Py_ssize_t start;

Py_ssize_t end;

PyObject *res;

unsigned char *outp;

Py_ssize_t ressize;

int replsize;

Py_UCS4 c;

char buffer[256]; /* NAME_MAXLEN */

if (PyUnicodeEncodeError_GetStart(exc, &start))

return NULL;

if (PyUnicodeEncodeError_GetEnd(exc, &end))

return NULL;

if (!(object = PyUnicodeEncodeError_GetObject(exc)))

return NULL;

if (!ucnhash_CAPI) {

/* load the unicode data module */

ucnhash_CAPI = (_PyUnicode_Name_CAPI *)PyCapsule_Import(

PyUnicodeData_CAPSULE_NAME, 1);

if (!ucnhash_CAPI)

return NULL;

}

for (i = start, ressize = 0; i < end; ++i) {

/* object is guaranteed to be "ready" */

c = PyUnicode_READ_CHAR(object, i);

if (ucnhash_CAPI->getname(NULL, c, buffer, sizeof(buffer), 1)) {

replsize = 1+1+1+(int)strlen(buffer)+1;

}