How to use

UTF8

in

com.wizzardo.tools.misc

public static int decode(byte[] bytes, int offset, int length, char[] chars) {
  return decode(bytes, offset, length, chars, 0);
}

public static byte[] encode(char[] chars) {
  return encode(chars, 0, chars.length);
}

/**
 * @return byte array with UTF-8 encoded string
 **/
public byte[] toBytes() {
  if (utf8Buffer == null)
    utf8Buffer = new UTF8();
  return utf8Buffer.toBytes(buffer, 0, length);
}

public static byte[] encode(char[] chars, int off, int length) {
  byte[] bytes = new byte[count(chars, off, length)];
  encode(chars, off, length, bytes);
  return bytes;
}

    b1 = bytes[offset++];
    b2 = bytes[offset++];
    if (isMalformed3(b, b1, b2)) {
      chars[i++] = MALFORMED;
    } else {
      char ch = (char) (b << 12 ^ b1 << 6 ^ b2 ^ -123008);
      if (isSurrogate(ch)) {
        chars[i++] = MALFORMED;
      } else {
    if (offset >= to || !isMalformed3_2(b, bytes[offset])) {
      chars[i++] = MALFORMED;
      return i;
  int b3 = bytes[offset++];
  int value = b << 18 ^ b1 << 12 ^ b2 << 6 ^ b3 ^ 3678080;
  if (!isMalformed4(b1, b2, b3) && isSupplementaryCodePoint(value)) {
    chars[i++] = highSurrogate(value);
    chars[i++] = lowSurrogate(value);
  } else {
    chars[i++] = MALFORMED;
  if (i1 <= 244 && (offset >= to || !isMalformed4_2(i1, bytes[offset] & 0xff))) {
    ++offset;
    if (offset >= to || !isMalformed4_3(bytes[offset])) {
      chars[i++] = MALFORMED;
      return i;
if (isNotContinuation(b1)) {

public byte[] toBytes(char[] chars, int offset, int length) {
  if (buffer == null || buffer.length < length * 4)
    buffer = new byte[length * 4];
  return encodeAndTrim(chars, offset, length, buffer);
}

    b1 = bytes[offset++];
    b2 = bytes[offset++];
    if (isMalformed3(b, b1, b2)) {
      chars[i++] = MALFORMED;
    } else {
      char ch = (char) (b << 12 ^ b1 << 6 ^ b2 ^ -123008);
      if (isSurrogate(ch)) {
        chars[i++] = MALFORMED;
      } else {
    if (offset >= to || !isMalformed3_2(b, bytes[offset])) {
      chars[i++] = MALFORMED;
      return i;
  int b3 = bytes[offset++];
  int value = b << 18 ^ b1 << 12 ^ b2 << 6 ^ b3 ^ 3678080;
  if (!isMalformed4(b1, b2, b3) && isSupplementaryCodePoint(value)) {
    chars[i++] = highSurrogate(value);
    chars[i++] = lowSurrogate(value);
  } else {
    chars[i++] = MALFORMED;
  if (i1 <= 244 && (offset >= to || !isMalformed4_2(i1, bytes[offset] & 0xff))) {
    ++offset;
    if (offset >= to || !isMalformed4_3(bytes[offset])) {
      chars[i++] = MALFORMED;
      return i;
if (isNotContinuation(b1)) {

public static byte[] encode(char[] chars, int off, int length) {
  byte[] bytes = new byte[count(chars, off, length)];
  encode(chars, off, length, bytes);
  return bytes;
}

public byte[] toBytes(char[] chars, int offset, int length) {
  if (buffer == null || buffer.length < length * 4)
    buffer = new byte[length * 4];
  return encodeAndTrim(chars, offset, length, buffer);
}

context.buffer[cl++] = bytes[offset++];
context.length = 0;
if (decode(context.buffer, 0, cl, chars, context).charsOffset == i) { // 2 bytes
  if (offset < to) {
    context.buffer[cl++] = bytes[offset++];
    context.length = 0;
    if (decode(context.buffer, 0, cl, chars, context).charsOffset == i) {// 3 bytes
      if (offset < to) {
        context.buffer[cl++] = bytes[offset++];
        context.length = 0;
        i = decode(context.buffer, 0, cl, chars, context).charsOffset; // 4 bytes
      } else {
        context.length = cl;
        b1 = bytes[offset++];
        b2 = bytes[offset++];
        if (isMalformed3(b, b1, b2)) {
          chars[i++] = MALFORMED;
        } else {
          char ch = (char) (b << 12 ^ b1 << 6 ^ b2 ^ -123008);
          if (isSurrogate(ch)) {
            chars[i++] = MALFORMED;
          } else {
        if (offset >= to || !isMalformed3_2(b, bytes[offset])) {
          return context.update(i, bytes, offset - 1, to);
      int b3 = bytes[offset++];
      int value = b << 18 ^ b1 << 12 ^ b2 << 6 ^ b3 ^ 3678080;

public void toBytes(Supplier<byte[]> bytesSupplier, UTF8.BytesConsumer bytesConsumer) {
  UTF8.encode(buffer, 0, length, bytesSupplier, bytesConsumer);
}

public static int decode(byte[] bytes, int offset, int length, char[] chars) {
  return decode(bytes, offset, length, chars, 0);
}

/**
 * @return byte array with UTF-8 encoded string
 **/
public byte[] toBytes() {
  if (utf8Buffer == null)
    utf8Buffer = new UTF8();
  return utf8Buffer.toBytes(buffer, 0, length);
}

context.buffer[cl++] = bytes[offset++];
context.length = 0;
if (decode(context.buffer, 0, cl, chars, context).charsOffset == i) { // 2 bytes
  if (offset < to) {
    context.buffer[cl++] = bytes[offset++];
    context.length = 0;
    if (decode(context.buffer, 0, cl, chars, context).charsOffset == i) {// 3 bytes
      if (offset < to) {
        context.buffer[cl++] = bytes[offset++];
        context.length = 0;
        i = decode(context.buffer, 0, cl, chars, context).charsOffset; // 4 bytes
      } else {
        context.length = cl;
        b1 = bytes[offset++];
        b2 = bytes[offset++];
        if (isMalformed3(b, b1, b2)) {
          chars[i++] = MALFORMED;
        } else {
          char ch = (char) (b << 12 ^ b1 << 6 ^ b2 ^ -123008);
          if (isSurrogate(ch)) {
            chars[i++] = MALFORMED;
          } else {
        if (offset >= to || !isMalformed3_2(b, bytes[offset])) {
          return context.update(i, bytes, offset - 1, to);
      int b3 = bytes[offset++];
      int value = b << 18 ^ b1 << 12 ^ b2 << 6 ^ b3 ^ 3678080;

public void toBytes(Supplier<byte[]> bytesSupplier, UTF8.BytesConsumer bytesConsumer) {
  UTF8.encode(buffer, 0, length, bytesSupplier, bytesConsumer);
}

for (int i = 0; i <= 0x7F; i++) {
  bytes[0] = (byte) i;
  int r = UTF8.decode(bytes, 0, bytes.length, chars);
  Assert.assertEquals(new String(bytes, charset), new String(chars, 0, r));
  for (int j = 0; j < 256; j++) {
    bytes[1] = (byte) (j & 0x7f | 0x80);
    int r = UTF8.decode(bytes, 0, bytes.length, chars);
    Assert.assertEquals(i + " " + j, new String(bytes, charset), new String(chars, 0, r));
    for (int k = 0; k < 64; k++) {
      bytes[2] = (byte) (k & 0x7f | 0x80);
      int r = UTF8.decode(bytes, 0, bytes.length, chars);
      String expected = new String(bytes, charset);
      if (expected.charAt(0) >= '\uD800' && expected.charAt(0) <= '\uDFFF')
      for (int m = 0; m < 64; m++) {
        bytes[3] = (byte) (m & 0x7f | 0x80);
        int r = UTF8.decode(bytes, 0, bytes.length, chars);
        if (r == 1 && chars[0] == '�') {
          char[] c = new String(bytes, charset).toCharArray();

    b1 = bytes[offset++];
    b2 = bytes[offset++];
    if (isMalformed3(b, b1, b2)) {
      chars[i++] = MALFORMED;
    } else {
      char ch = (char) (b << 12 ^ b1 << 6 ^ b2 ^ -123008);
      if (isSurrogate(ch)) {
        chars[i++] = MALFORMED;
      } else {
    if (offset >= to || !isMalformed3_2(b, bytes[offset])) {
      return offsets.update(i, offset - 1);
  int b3 = bytes[offset++];
  int value = b << 18 ^ b1 << 12 ^ b2 << 6 ^ b3 ^ 3678080;
  if (!isMalformed4(b1, b2, b3) && isSupplementaryCodePoint(value)) {
    chars[i++] = highSurrogate(value);
    chars[i++] = lowSurrogate(value);
  } else {
    chars[i++] = MALFORMED;
  if (i1 <= 244 && (offset >= to || !isMalformed4_2(i1, bytes[offset] & 0xff))) {
    offset++;
    if (offset >= to || !isMalformed4_3(bytes[offset]))
      return offsets.update(i, offset - 2);
if (isNotContinuation(b1)) {
  chars[i++] = MALFORMED;
  --offset;

public static byte[] encode(char[] chars) {
  return encode(chars, 0, chars.length);
}

@Test
public void decode_continuous_1() throws UnsupportedEncodingException {
  String s = "€";
  char[] chars = new char[1];
  byte[] bytes = s.getBytes("utf-8");
  Assert.assertEquals(3, bytes.length);
  int read;
  UTF8.DecodeOffsets offsets = new UTF8.DecodeOffsets();
  read = 1;
  offsets = UTF8.decode(bytes, read, chars, offsets);
  Assert.assertEquals(0, offsets.bytesOffset);
  Assert.assertEquals(0, offsets.charsOffset);
  read++;
  offsets = UTF8.decode(bytes, read, chars, offsets);
  Assert.assertEquals(0, offsets.bytesOffset);
  Assert.assertEquals(0, offsets.charsOffset);
  read++;
  offsets = UTF8.decode(bytes, read, chars, offsets);
  Assert.assertEquals(3, offsets.bytesOffset);
  Assert.assertEquals(1, offsets.charsOffset);
  Assert.assertEquals(s.charAt(0), chars[0]);
}

    b1 = bytes[offset++];
    b2 = bytes[offset++];
    if (isMalformed3(b, b1, b2)) {
      chars[i++] = MALFORMED;
    } else {
      char ch = (char) (b << 12 ^ b1 << 6 ^ b2 ^ -123008);
      if (isSurrogate(ch)) {
        chars[i++] = MALFORMED;
      } else {
    if (offset >= to || !isMalformed3_2(b, bytes[offset])) {
      return offsets.update(i, offset - 1);
  int b3 = bytes[offset++];
  int value = b << 18 ^ b1 << 12 ^ b2 << 6 ^ b3 ^ 3678080;
  if (!isMalformed4(b1, b2, b3) && isSupplementaryCodePoint(value)) {
    chars[i++] = highSurrogate(value);
    chars[i++] = lowSurrogate(value);
  } else {
    chars[i++] = MALFORMED;
  if (i1 <= 244 && (offset >= to || !isMalformed4_2(i1, bytes[offset] & 0xff))) {
    offset++;
    if (offset >= to || !isMalformed4_3(bytes[offset]))
      return offsets.update(i, offset - 2);
if (isNotContinuation(b1)) {
  chars[i++] = MALFORMED;
  --offset;