Skip to content

Commit

Permalink
performance optimize , 1. cache string names 2. cache Charset
Browse files Browse the repository at this point in the history
  • Loading branch information
@agapple
agapple committed Nov 16, 2022
1 parent afef455 commit f7a4979
Show file tree
Hide file tree
Showing 21 changed files with 226 additions and 141 deletions.
19 changes: 19 additions & 0 deletions dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,15 @@ static final class Entry {
protected final String mysqlCharset;
protected final String mysqlCollation;
protected final String javaCharset;
protected final Charset charset;

Entry(final int id, String mysqlCharset, // NL
String mysqlCollation, String javaCharset){
this.charsetId = id;
this.mysqlCharset = mysqlCharset;
this.mysqlCollation = mysqlCollation;
this.javaCharset = javaCharset;
this.charset = Charset.isSupported(javaCharset) ? Charset.forName(javaCharset) : null;
}
}

Expand Down Expand Up @@ -390,6 +392,23 @@ public static String getJavaCharset(final int id) {
}
}

public static Charset getNioCharset(final int id) {
Entry entry = getEntry(id);

if (entry != null) {
if (entry.charset != null) {
return entry.charset;
} else {
logger.warn("Unknown java charset for: id = " + id + ", name = " + entry.mysqlCharset + ", coll = "
+ entry.mysqlCollation);
return null;
}
} else {
logger.warn("Unexpect mysql charset: " + id);
return null;
}
}

public static void main(String[] args) {
for (int i = 0; i < entries.length; i++) {
Entry entry = entries[i];
Expand Down
42 changes: 24 additions & 18 deletions dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.taobao.tddl.dbsync.binlog;

import java.nio.charset.Charset;

import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.appendNumber2;
import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.appendNumber4;
import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.usecondsToStr;
Expand Down Expand Up @@ -57,23 +59,27 @@ public class JsonConversion {
public static final int VALUE_ENTRY_SIZE_LARGE = (1 + LARGE_OFFSET_SIZE);

public static Json_Value parse_value(int type, LogBuffer buffer, long len, String charsetName) {
return parse_value(type, buffer, len, Charset.forName(charsetName));
}

public static Json_Value parse_value(int type, LogBuffer buffer, long len, Charset charset) {
buffer = buffer.duplicate(buffer.position(), (int) len);
switch (type) {
case JSONB_TYPE_SMALL_OBJECT:
return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, false, charsetName);
return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, false, charset);
case JSONB_TYPE_LARGE_OBJECT:
return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, true, charsetName);
return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, true, charset);
case JSONB_TYPE_SMALL_ARRAY:
return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, false, charsetName);
return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, false, charset);
case JSONB_TYPE_LARGE_ARRAY:
return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, true, charsetName);
return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, true, charset);
default:
return parse_scalar(type, buffer, len, charsetName);
return parse_scalar(type, buffer, len, charset);
}
}

private static Json_Value parse_array_or_object(Json_enum_type type, LogBuffer buffer, long len, boolean large,
String charsetName) {
Charset charset) {
long offset_size = large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
if (len < 2 * offset_size) {
throw new IllegalArgumentException("illegal json data");
Expand All @@ -100,7 +106,7 @@ private static long read_offset_or_size(LogBuffer buffer, boolean large) {
return large ? buffer.getUint32() : buffer.getUint16();
}

private static Json_Value parse_scalar(int type, LogBuffer buffer, long len, String charsetName) {
private static Json_Value parse_scalar(int type, LogBuffer buffer, long len, Charset charset) {
switch (type) {
case JSONB_TYPE_LITERAL:
/* purecov: inspected */
Expand Down Expand Up @@ -156,7 +162,7 @@ private static Json_Value parse_scalar(int type, LogBuffer buffer, long len, Str
}
return new Json_Value(Json_enum_type.STRING, buffer.rewind()
.forward((int) n)
.getFixString((int) str_len, charsetName));
.getFixString((int) str_len, charset));
case JSONB_TYPE_OPAQUE:
/*
* There should always be at least one byte, which tells the
Expand Down Expand Up @@ -242,7 +248,7 @@ public Json_Value(Json_enum_type t, LogBuffer data, long element_count, long byt
this.m_large = large;
}

public String key(int i, String charsetName) {
public String key(int i, Charset charset) {
m_data.rewind();
int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL;
Expand All @@ -255,10 +261,10 @@ public String key(int i, String charsetName) {
// entry, always two
// bytes.
long key_length = m_data.getUint16();
return m_data.rewind().forward((int) key_offset).getFixString((int) key_length, charsetName);
return m_data.rewind().forward((int) key_offset).getFixString((int) key_length, charset);
}

public Json_Value element(int i, String charsetName) {
public Json_Value element(int i, Charset charset) {
m_data.rewind();
int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL;
Expand All @@ -271,23 +277,23 @@ public Json_Value element(int i, String charsetName) {
int type = m_data.forward(entry_offset).getUint8();
if (type == JSONB_TYPE_INT16 || type == JSONB_TYPE_UINT16 || type == JSONB_TYPE_LITERAL
|| (m_large && (type == JSONB_TYPE_INT32 || type == JSONB_TYPE_UINT32))) {
return parse_scalar(type, m_data, value_entry_size - 1, charsetName);
return parse_scalar(type, m_data, value_entry_size - 1, charset);
}
int value_offset = (int) read_offset_or_size(m_data, m_large);
return parse_value(type, m_data.rewind().forward(value_offset), (int) m_length - value_offset, charsetName);
return parse_value(type, m_data.rewind().forward(value_offset), (int) m_length - value_offset, charset);
}

public StringBuilder toJsonString(StringBuilder buf, String charsetName) {
public StringBuilder toJsonString(StringBuilder buf, Charset charset) {
switch (m_type) {
case OBJECT:
buf.append("{");
for (int i = 0; i < m_element_count; ++i) {
if (i > 0) {
buf.append(", ");
}
buf.append('"').append(key(i, charsetName)).append('"');
buf.append('"').append(key(i, charset)).append('"');
buf.append(": ");
element(i, charsetName).toJsonString(buf, charsetName);
element(i, charset).toJsonString(buf, charset);
}
buf.append("}");
break;
Expand All @@ -297,7 +303,7 @@ public StringBuilder toJsonString(StringBuilder buf, String charsetName) {
if (i > 0) {
buf.append(", ");
}
element(i, charsetName).toJsonString(buf, charsetName);
element(i, charset).toJsonString(buf, charset);
}
buf.append("]");
break;
Expand Down Expand Up @@ -398,7 +404,7 @@ public StringBuilder toJsonString(StringBuilder buf, String charsetName) {
}
buf.append('"').append(text).append('"');
} else {
text = m_data.getFixString((int) m_length, charsetName);
text = m_data.getFixString((int) m_length, charset);
buf.append('"').append(escapse(text)).append('"');
}

Expand Down
10 changes: 8 additions & 2 deletions dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonDiffConversion.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.taobao.tddl.dbsync.binlog;

import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;

Expand Down Expand Up @@ -36,6 +37,11 @@ public class JsonDiffConversion {

public static StringBuilder print_json_diff(LogBuffer buffer, long len, String columnName, int columnIndex,
String charsetName) {
return print_json_diff(buffer, len, columnName, columnIndex, Charset.forName(charsetName));
}

public static StringBuilder print_json_diff(LogBuffer buffer, long len, String columnName, int columnIndex,
Charset charset) {
int position = buffer.position();
List<String> operation_names = new ArrayList<>();
while (buffer.hasRemaining()) {
Expand Down Expand Up @@ -111,14 +117,14 @@ public static StringBuilder print_json_diff(LogBuffer buffer, long len, String c
Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(),
buffer,
value_length - 1,
charsetName);
charset);
buffer.forward((int) value_length - 1);
// Read value
if (jsonValue.m_type == Json_enum_type.ERROR) {
throw new IllegalArgumentException("parsing json value");
}
StringBuilder jsonBuilder = new StringBuilder();
jsonValue.toJsonString(jsonBuilder, charsetName);
jsonValue.toJsonString(jsonBuilder, charset);
builder.append(jsonBuilder);
}

Expand Down
Loading

0 comments on commit f7a4979

Please sign in to comment.