Skip to content

feat(java): Support codegen for xlang serialization in java for strict mode #2312

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 15 commits into
base: main
Choose a base branch
from
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,10 @@ public String codecQualifiedClassName(Class<?> beanClass) {

protected abstract String codecSuffix();

protected abstract Expression buildXlangDecodeExpression();

protected abstract Expression buildXlangEncodeExpression();

protected <T> T fory(Function<Fory, T> function) {
return fory.getJITContext().asyncVisitFory(function);
}
Expand All @@ -238,8 +242,7 @@ public String genCode() {
ctx.extendsClasses(ctx.type(parentSerializerClass));
ctx.reserveName(POJO_CLASS_TYPE_NAME);
ctx.addField(ctx.type(Fory.class), FORY_NAME);
Expression encodeExpr = buildEncodeExpression();
Expression decodeExpr = buildDecodeExpression();
boolean xlang = fory.isCrossLanguage();
String constructorCode =
StringUtils.format(
""
Expand All @@ -250,22 +253,43 @@ public String genCode() {
FORY_NAME,
"cls",
POJO_CLASS_TYPE_NAME);

ctx.clearExprState();
String encodeCode = encodeExpr.genCode(ctx).code();
encodeCode = ctx.optimizeMethodCode(encodeCode);
ctx.clearExprState();
String decodeCode = decodeExpr.genCode(ctx).code();
decodeCode = ctx.optimizeMethodCode(decodeCode);
ctx.overrideMethod(
"write",
encodeCode,
void.class,
MemoryBuffer.class,
BUFFER_NAME,
Object.class,
ROOT_OBJECT_NAME);
ctx.overrideMethod("read", decodeCode, Object.class, MemoryBuffer.class, BUFFER_NAME);
if (!xlang) {
Expression encodeExpr = buildEncodeExpression();
Expression decodeExpr = buildDecodeExpression();
ctx.clearExprState();
String encodeCode = encodeExpr.genCode(ctx).code();
encodeCode = ctx.optimizeMethodCode(encodeCode);
ctx.clearExprState();
String decodeCode = decodeExpr.genCode(ctx).code();
decodeCode = ctx.optimizeMethodCode(decodeCode);
ctx.overrideMethod(
"write",
encodeCode,
void.class,
MemoryBuffer.class,
BUFFER_NAME,
Object.class,
ROOT_OBJECT_NAME);
ctx.overrideMethod("read", decodeCode, Object.class, MemoryBuffer.class, BUFFER_NAME);
} else {
Expression xlangEncodeExpr = buildXlangEncodeExpression();
Expression xlangDecodeExpr = buildXlangDecodeExpression();
ctx.clearExprState();
String xlangEncodeCode = xlangEncodeExpr.genCode(ctx).code();
xlangEncodeCode = ctx.optimizeMethodCode(xlangEncodeCode);
ctx.clearExprState();
String xlangDecodeCode = xlangDecodeExpr.genCode(ctx).code();
xlangDecodeCode = ctx.optimizeMethodCode(xlangDecodeCode);
ctx.overrideMethod(
"xwrite",
xlangEncodeCode,
void.class,
MemoryBuffer.class,
BUFFER_NAME,
Object.class,
ROOT_OBJECT_NAME);
ctx.overrideMethod("xread", xlangDecodeCode, Object.class, MemoryBuffer.class, BUFFER_NAME);
}
registerJITNotifyCallback();
ctx.addConstructor(constructorCode, Fory.class, "fory", Class.class, POJO_CLASS_TYPE_NAME);
return ctx.genCode();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,11 @@ public Expression buildEncodeExpression() {
return expressions;
}

@Override
public Expression buildXlangEncodeExpression() {
throw new IllegalStateException("unreachable");
}

private Expression writeEmbedTypeFieldValue(
Expression bean, Expression buffer, FieldInfo fieldInfo) {
Descriptor descriptor = createDescriptor(fieldInfo);
Expand Down Expand Up @@ -381,6 +386,11 @@ public Expression buildDecodeExpression() {
return expressionBuilder;
}

@Override
public Expression buildXlangDecodeExpression() {
throw new IllegalStateException("unreachable");
}

public Expression buildRecordDecodeExpression() {
Reference buffer = new Reference(BUFFER_NAME, bufferTypeRef, false);
StaticInvoke components =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -126,11 +126,19 @@ public String genCode() {
MetaSharedCodecBuilder.class.getName(),
"serializer",
SERIALIZER_FIELD_NAME);
boolean xlang = fory.isCrossLanguage();
ctx.clearExprState();
Expression decodeExpr = buildDecodeExpression();
String decodeCode = decodeExpr.genCode(ctx).code();
decodeCode = ctx.optimizeMethodCode(decodeCode);
ctx.overrideMethod("read", decodeCode, Object.class, MemoryBuffer.class, BUFFER_NAME);
if (!xlang) {
Expression decodeExpr = buildDecodeExpression();
String decodeCode = decodeExpr.genCode(ctx).code();
decodeCode = ctx.optimizeMethodCode(decodeCode);
ctx.overrideMethod("read", decodeCode, Object.class, MemoryBuffer.class, BUFFER_NAME);
} else {
Expression xlangDecodeExpression = buildXlangDecodeExpression();
String xlangDecodeCode = xlangDecodeExpression.genCode(ctx).code();
xlangDecodeCode = ctx.optimizeMethodCode(xlangDecodeCode);
ctx.overrideMethod("xread", xlangDecodeCode, Object.class, MemoryBuffer.class, BUFFER_NAME);
}
registerJITNotifyCallback();
ctx.addConstructor(constructorCode, Fory.class, "fory", Class.class, POJO_CLASS_TYPE_NAME);
return ctx.genCode();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,11 @@ public Expression buildEncodeExpression() {
return expressions;
}

@Override
public Expression buildXlangEncodeExpression() {
return buildEncodeExpression();
}

private void addGroupExpressions(
List<List<Descriptor>> writeGroup,
int numGroups,
Expand Down Expand Up @@ -485,6 +490,11 @@ public Expression buildDecodeExpression() {
return expressions;
}

@Override
public Expression buildXlangDecodeExpression() {
return buildDecodeExpression();
}

private void deserializeReadGroup(
List<List<Descriptor>> readGroups,
int numGroups,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,22 @@ public class ClassInfo {
}
}

public static ClassInfo copy(ClassInfo classInfo) {
final ClassInfo copy =
new ClassInfo(
classInfo.cls,
classInfo.fullNameBytes,
classInfo.namespaceBytes,
classInfo.typeNameBytes,
classInfo.isDynamicGeneratedClass,
classInfo.serializer,
classInfo.classId,
(short) classInfo.xtypeId);
copy.needToWriteClassDef = classInfo.needToWriteClassDef;
copy.classDef = classInfo.classDef;
return copy;
}

public Class<?> getCls() {
return cls;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,11 @@
import static org.apache.fory.meta.Encoders.PACKAGE_DECODER;
import static org.apache.fory.meta.Encoders.PACKAGE_ENCODER;
import static org.apache.fory.meta.Encoders.TYPE_NAME_DECODER;
import static org.apache.fory.resolver.ClassResolver.NIL_CLASS_INFO;
import static org.apache.fory.resolver.ClassResolver.NO_CLASS_ID;
import static org.apache.fory.serializer.CodegenSerializer.loadCodegenSerializer;
import static org.apache.fory.serializer.CodegenSerializer.loadCompatibleCodegenSerializer;
import static org.apache.fory.serializer.CodegenSerializer.supportCodegenForJavaSerialization;
import static org.apache.fory.serializer.collection.MapSerializers.HashMapSerializer;
import static org.apache.fory.type.TypeUtils.qualifiedName;

Expand All @@ -49,6 +53,7 @@
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.fory.Fory;
import org.apache.fory.builder.JITContext;
import org.apache.fory.collection.IdentityMap;
import org.apache.fory.collection.IdentityObjectIntMap;
import org.apache.fory.collection.LongMap;
Expand All @@ -66,8 +71,11 @@
import org.apache.fory.reflect.ReflectionUtils;
import org.apache.fory.reflect.TypeRef;
import org.apache.fory.serializer.ArraySerializers;
import org.apache.fory.serializer.CodegenSerializer;
import org.apache.fory.serializer.CompatibleSerializer;
import org.apache.fory.serializer.EnumSerializer;
import org.apache.fory.serializer.LazySerializer;
import org.apache.fory.serializer.MetaSharedSerializer;
import org.apache.fory.serializer.NonexistentClass;
import org.apache.fory.serializer.NonexistentClassSerializers;
import org.apache.fory.serializer.ObjectSerializer;
Expand Down Expand Up @@ -98,7 +106,7 @@ public class XtypeResolver implements TypeResolver {
private final Config config;
private final Fory fory;
private final ClassResolver classResolver;
private final ClassInfoHolder classInfoCache = new ClassInfoHolder(ClassResolver.NIL_CLASS_INFO);
private final ClassInfoHolder classInfoCache = new ClassInfoHolder(NIL_CLASS_INFO);
private final MetaStringResolver metaStringResolver;
// IdentityMap has better lookup performance, when loadFactor is 0.05f, performance is better
private final IdentityMap<Class<?>, ClassInfo> classInfoMap = new IdentityMap<>(64, loadFactor);
Expand All @@ -109,6 +117,7 @@ public class XtypeResolver implements TypeResolver {
new ObjectMap<>(16, loadFactor);
private final Map<Class<?>, ClassDef> classDefMap = new HashMap<>();
private final boolean shareMeta;
private final Set<Class<?>> classCtx = new HashSet<>();
private int xtypeIdGenerator = 64;

// Use ClassInfo[] or LongMap?
Expand Down Expand Up @@ -242,6 +251,70 @@ private void register(
xtypeIdToClassMap.put(xtypeId, classInfo);
}

private void addSerializer(Class<?> type, ClassInfo classInfo) {
classInfoMap.put(type, classInfo);
xtypeIdToClassMap.put(classInfo.xtypeId, classInfo);
qualifiedType2ClassInfo.put(qualifiedName(classInfo.decodeTypeName(), classInfo.decodeTypeName()), classInfo);

}

private Class<? extends Serializer> getObjectSerializerClass(
Class<?> cls,
boolean shareMeta,
boolean codegen,
JITContext.SerializerJITCallback<Class<? extends Serializer>> callback) {
if (codegen) {
if (classCtx.contains(cls)) {
// avoid potential recursive call for seq codec generation.
return CodegenSerializer.LazyInitBeanSerializer.class;
} else {
try {
classCtx.add(cls);
Class<? extends Serializer> sc;
switch (fory.getCompatibleMode()) {
case SCHEMA_CONSISTENT:
sc =
fory.getJITContext()
.registerSerializerJITCallback(
() -> ObjectSerializer.class,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I prefer creating serializer lazily, since the hash compute for type meta will read names of all registered classes. And when class registration is invoked, if the serializer are created egaerly, some classes don't have a change to let the registered name/tag being used for hash computing

() -> loadCodegenSerializer(fory, cls),
callback);
return sc;
case COMPATIBLE:
sc =
fory.getJITContext()
.registerSerializerJITCallback(
() -> shareMeta ? ObjectSerializer.class : CompatibleSerializer.class,
() ->
shareMeta
? loadCodegenSerializer(fory, cls)
: loadCompatibleCodegenSerializer(fory, cls),
callback);
return sc;
default:
throw new UnsupportedOperationException(
String.format("Unsupported mode %s", fory.getCompatibleMode()));
}
} finally {
classCtx.remove(cls);
}
}
} else {
if (codegen) {
LOG.info("Object of type {} can't be serialized by jit", cls);
}
switch (fory.getCompatibleMode()) {
case SCHEMA_CONSISTENT:
return ObjectSerializer.class;
case COMPATIBLE:
return shareMeta ? ObjectSerializer.class : MetaSharedSerializer.class;
default:
throw new UnsupportedOperationException(
String.format("Unsupported mode %s", fory.getCompatibleMode()));
}
}
}

private boolean isStructType(Serializer serializer) {
if (serializer instanceof ObjectSerializer || serializer instanceof GeneratedSerializer) {
return true;
Expand Down Expand Up @@ -363,6 +436,9 @@ public ClassInfo getClassInfo(Class<?> cls, boolean createIfAbsent) {
if (createIfAbsent) {
return getClassInfo(cls);
}
if (classCtx.contains(cls)) {
return null;
}
return classInfoMap.get(cls);
}

Expand Down Expand Up @@ -580,7 +656,36 @@ private ClassDef buildClassDef(ClassInfo classInfo) {

@Override
public <T> Serializer<T> getSerializer(Class<T> cls) {
return (Serializer) getClassInfo(cls).serializer;
ClassInfo classInfo = getClassInfo(cls);
if (classInfo.serializer instanceof LazySerializer.LazyObjectSerializer) {
boolean codegen =
supportCodegenForJavaSerialization(cls) && fory.getConfig().isCodeGenEnabled();
Class<? extends Serializer> objectSerializerClass =
getObjectSerializerClass(
cls,
shareMeta,
codegen,
new JITContext.SerializerJITCallback<Class<? extends Serializer>>() {
@Override
public void onSuccess(Class<? extends Serializer> result) {
Serializer<?> objectSerializer = Serializers.newSerializer(fory, cls, result);
ClassInfo copy = ClassInfo.copy(classInfo);
copy.serializer = objectSerializer;
addSerializer(cls, copy);
if (classInfoCache.classInfo.cls == cls) {
classInfoCache.classInfo = NIL_CLASS_INFO; // clear class info cache
}
Preconditions.checkState(getSerializer(cls).getClass() == result);
}

@Override
public Object id() {
return cls;
}
});
classInfo.serializer = Serializers.newSerializer(fory, cls, objectSerializerClass);
}
return (Serializer<T>) classInfo.serializer;
}

@Override
Expand Down
Loading
Loading