GIRAPH-1195
authorDionysios Logothetis <dlogothetis@gmail.com>
Thu, 31 May 2018 18:07:33 +0000 (11:07 -0700)
committerDionysios Logothetis <dionysios@fb.com>
Thu, 31 May 2018 18:07:33 +0000 (11:07 -0700)
closes #76

giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java
giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableBiMapSerializerUtils.java [new file with mode: 0644]
giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java [deleted file]
giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperTest.java
pom.xml

index 2713316..5d51a3f 100644 (file)
@@ -27,6 +27,7 @@ import java.util.Map.Entry;
 import java.util.Random;
 
 import com.esotericsoftware.kryo.util.DefaultClassResolver;
+import de.javakaffee.kryoserializers.guava.ImmutableMapSerializer;
 import org.apache.giraph.conf.GiraphConfigurationSettable;
 import com.esotericsoftware.kryo.ClassResolver;
 import com.esotericsoftware.kryo.ReferenceResolver;
@@ -39,7 +40,7 @@ import org.apache.giraph.writable.kryo.serializers.ArraysAsListSerializer;
 import org.apache.giraph.writable.kryo.serializers.CollectionsNCopiesSerializer;
 import org.apache.giraph.writable.kryo.serializers.DirectWritableSerializer;
 import org.apache.giraph.writable.kryo.serializers.FastUtilSerializer;
-import org.apache.giraph.writable.kryo.serializers.ImmutableMapSerializer;
+import org.apache.giraph.writable.kryo.serializers.ImmutableBiMapSerializerUtils;
 import org.apache.giraph.writable.kryo.serializers.ReusableFieldSerializer;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
@@ -322,7 +323,8 @@ public class HadoopKryo extends Kryo {
     if (minor >= '8') {
       try {
         kryo.register(Class.forName("java.lang.invoke.SerializedLambda"));
-        kryo.register(Class.forName("com.esotericsoftware.kryo.Kryo$Closure"),
+        kryo.register(Class.forName(
+          "com.esotericsoftware.kryo.serializers.ClosureSerializer$Closure"),
             new ClosureSerializer());
       } catch (ClassNotFoundException e) {
         throw new IllegalStateException(
@@ -336,12 +338,8 @@ public class HadoopKryo extends Kryo {
         new CollectionsNCopiesSerializer());
 
     ImmutableListSerializer.registerSerializers(kryo);
-
-    registerSerializer(kryo, "com.google.common.collect.RegularImmutableMap",
-        new ImmutableMapSerializer());
-    registerSerializer(kryo,
-        "com.google.common.collect.SingletonImmutableBiMap",
-        new ImmutableMapSerializer());
+    ImmutableMapSerializer.registerSerializers(kryo);
+    ImmutableBiMapSerializerUtils.registerSerializers(kryo);
 
     // There are many fastutil classes, register them at the end,
     // so they don't use up small registration numbers
diff --git a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableBiMapSerializerUtils.java b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableBiMapSerializerUtils.java
new file mode 100644 (file)
index 0000000..6e66b47
--- /dev/null
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.writable.kryo.serializers;
+
+
+import com.esotericsoftware.kryo.Kryo;
+import com.google.common.collect.ImmutableBiMap;
+import de.javakaffee.kryoserializers.guava.ImmutableMapSerializer;
+
+/**
+ * Helper class used to register serializers for the different classes
+ * return by the {@link ImmutableBiMap} methods.
+ */
+public class ImmutableBiMapSerializerUtils {
+
+  /**
+   * Private default constructor
+   */
+  private ImmutableBiMapSerializerUtils() {
+  }
+
+  /**
+   * Registers different {@link ImmutableBiMap} implementations with Kryo.
+   * @param kryo {@link Kryo} instance to register class with.
+   */
+  public static void registerSerializers(Kryo kryo) {
+    ImmutableMapSerializer serializer = new ImmutableMapSerializer();
+    kryo.register(ImmutableBiMap.class, serializer);
+    kryo.register(ImmutableBiMap.of().getClass(), serializer);
+    Object o1 = new Object();
+    Object o2 = new Object();
+    Object o3 = new Object();
+    Object o4 = new Object();
+    Object o5 = new Object();
+    kryo.register(ImmutableBiMap.of(o1, o1).getClass(), serializer);
+    kryo.register(ImmutableBiMap.of(o1, o1, o2, o2).getClass(), serializer);
+    kryo.register(ImmutableBiMap.of(o1, o1, o2, o2, o3, o3)
+      .getClass(), serializer);
+    kryo.register(ImmutableBiMap.of(o1, o1, o2, o2, o3, o3, o4, o4)
+      .getClass(), serializer);
+    kryo.register(ImmutableBiMap.of(o1, o1, o2, o2, o3, o3, o4, o4, o5, o5)
+      .getClass(), serializer);
+  }
+}
diff --git a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java
deleted file mode 100644 (file)
index 8dd2d1c..0000000
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.writable.kryo.serializers;
-
-
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.io.Input;
-import com.esotericsoftware.kryo.serializers.MapSerializer;
-import com.google.common.collect.ImmutableMap;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Serializer for {@link ImmutableMap}
- */
-public class ImmutableMapSerializer extends MapSerializer {
-  @Override
-  public Map read(Kryo kryo, Input input, Class<Map> type) {
-    Map map = super.read(kryo, input,
-        (Class<Map>) ((Object) HashMap.class));
-    return ImmutableMap.copyOf(map);
-  }
-}
index 6e0eb2f..9129a4e 100644 (file)
@@ -25,8 +25,11 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
 
+import com.google.common.collect.ImmutableBiMap;
+import com.google.common.collect.ImmutableMap;
 import org.apache.giraph.conf.GiraphConfiguration;
 import org.apache.giraph.conf.GiraphConfigurationSettable;
 import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
@@ -171,6 +174,41 @@ public class KryoWritableWrapperTest {
   }
 
   @Test
+  public void testImmutableMapSerialization() throws IOException {
+    Map original = ImmutableMap.of("x", "y", "y", "z");
+    Map copy = kryoSerDeser(original);
+    assertEquals(original, copy);
+  }
+
+  @Test
+  public void testImmutableMapSinglePairSerialization() throws IOException {
+    Map original = ImmutableMap.of("x", "y");
+    Map copy = kryoSerDeser(original);
+    assertEquals(original, copy);
+  }
+
+  @Test
+  public void testImmutableBiMap() throws IOException {
+    Map original = ImmutableBiMap.of("x", "y", "z", "w");
+    Map copy = kryoSerDeser(original);
+    assertEquals(original, copy);
+  }
+
+  @Test
+  public void testSingletonImmutableBiMapSerialization() throws IOException {
+    Map original = ImmutableBiMap.of("x", "y");
+    Map copy = kryoSerDeser(original);
+    assertEquals(original, copy);
+  }
+
+  @Test
+  public void testEmptyImmutableBiMap() throws IOException {
+    Map original = ImmutableBiMap.of();
+    Map copy = kryoSerDeser(original);
+    assertEquals(original, copy);
+  }
+
+  @Test
   public void testFastutilSet() throws ClassNotFoundException, IOException {
     LongOpenHashSet set = new LongOpenHashSet();
     set.add(6);
diff --git a/pom.xml b/pom.xml
index fd752fb..1522fd4 100644 (file)
--- a/pom.xml
+++ b/pom.xml
@@ -343,15 +343,16 @@ under the License.
     <dep.json.version>20160810</dep.json.version>
     <dep.junit.version>4.11</dep.junit.version>
     <dep.jython.version>2.5.3</dep.jython.version>
-    <dep.kryo.version>3.0.1</dep.kryo.version>
-    <dep.kryo-serializers.version>0.29</dep.kryo-serializers.version>
+    <dep.kryo.version>4.0.0</dep.kryo.version>
+    <dep.reflectasm.version>1.11.3</dep.reflectasm.version>
+    <dep.kryo-serializers.version>0.42</dep.kryo-serializers.version>
     <dep.libthrift.version>0.9.0</dep.libthrift.version>
     <dep.log4j.version>1.2.17</dep.log4j.version>
     <dep.mockito.version>1.9.5</dep.mockito.version>
     <!-- note: old version of netty is required by hadoop_facebook for tests to succeed -->
     <dep.netty.version>4.0.14.Final</dep.netty.version>
     <dep.oldnetty.version>3.2.2.Final</dep.oldnetty.version>
-    <dep.objenesis.version>2.1</dep.objenesis.version>
+    <dep.objenesis.version>2.2</dep.objenesis.version>
     <dep.openhft-compiler.version>2.2.1</dep.openhft-compiler.version>
     <dep.paranamer.version>2.5.2</dep.paranamer.version>
     <dep.slf4j.version>1.7.6</dep.slf4j.version>
@@ -2050,6 +2051,11 @@ under the License.
         <version>${dep.kryo.version}</version>
       </dependency>
       <dependency>
+        <groupId>com.esotericsoftware.reflectasm</groupId>
+        <artifactId>reflectasm</artifactId>
+        <version>${dep.reflectasm.version}</version>
+      </dependency>
+      <dependency>
         <groupId>de.javakaffee</groupId>
         <artifactId>kryo-serializers</artifactId>
         <version>${dep.kryo-serializers.version}</version>