/** * Make a copy of the writable object using serialization to a buffer * @param src the object to copy from * @param dst the object to copy into, which is destroyed * @return dst param (the copy) * @throws IOException */ @SuppressWarnings("unchecked") public static <T> T copy(Configuration conf, T src, T dst) throws IOException { CopyInCopyOutBuffer buffer = CLONE_BUFFERS.get(); buffer.outBuffer.reset(); SerializationFactory factory = getFactory(conf); Class<T> cls = (Class<T>) src.getClass(); Serializer<T> serializer = factory.getSerializer(cls); serializer.open(buffer.outBuffer); serializer.serialize(src); buffer.moveData(); Deserializer<T> deserializer = factory.getDeserializer(cls); deserializer.open(buffer.inBuffer); dst = deserializer.deserialize(dst); return dst; }
/** * Make a copy of the writable object using serialization to a buffer * @param dst the object to copy from * @param src the object to copy into, which is destroyed * @throws IOException */ @SuppressWarnings("unchecked") public static <T> T copy(Configuration conf, T src, T dst) throws IOException { CopyInCopyOutBuffer buffer = cloneBuffers.get(); buffer.outBuffer.reset(); SerializationFactory factory = getFactory(conf); Class<T> cls = (Class<T>) src.getClass(); Serializer<T> serializer = factory.getSerializer(cls); serializer.open(buffer.outBuffer); serializer.serialize(src); buffer.moveData(); Deserializer<T> deserializer = factory.getDeserializer(cls); deserializer.open(buffer.inBuffer); dst = deserializer.deserialize(dst); return dst; }
private <K> K serDeser(K conf) throws Exception { SerializationFactory factory = new SerializationFactory(CONF); Serializer<K> serializer = factory.getSerializer(GenericsUtil.getClass(conf)); Deserializer<K> deserializer = factory.getDeserializer(GenericsUtil.getClass(conf)); DataOutputBuffer out = new DataOutputBuffer(); serializer.open(out); serializer.serialize(conf); serializer.close(); DataInputBuffer in = new DataInputBuffer(); in.reset(out.getData(), out.getLength()); deserializer.open(in); K after = deserializer.deserialize(null); deserializer.close(); return after; }
private <E> E makeCopyForPassByValue(Serialization<E> serialization, E obj) throws IOException { Serializer<E> ser = serialization.getSerializer(GenericsUtil.getClass(obj)); Deserializer<E> deser = serialization.getDeserializer(GenericsUtil.getClass(obj)); DataOutputBuffer dof = threadLocalDataOutputBuffer.get(); dof.reset(); ser.open(dof); ser.serialize(obj); ser.close(); obj = ReflectionUtils.newInstance(GenericsUtil.getClass(obj), getChainJobConf()); ByteArrayInputStream bais = new ByteArrayInputStream(dof.getData(), 0, dof.getLength()); deser.open(bais); deser.deserialize(obj); deser.close(); return obj; }
/** * Make a copy of the writable object using serialization to a buffer * @param src the object to copy from * @param dst the object to copy into, which is destroyed * @return dst param (the copy) * @throws IOException */ @SuppressWarnings("unchecked") public static <T> T copy(Configuration conf, T src, T dst) throws IOException { CopyInCopyOutBuffer buffer = cloneBuffers.get(); buffer.outBuffer.reset(); SerializationFactory factory = getFactory(conf); Class<T> cls = (Class<T>) src.getClass(); Serializer<T> serializer = factory.getSerializer(cls); serializer.open(buffer.outBuffer); serializer.serialize(src); buffer.moveData(); Deserializer<T> deserializer = factory.getDeserializer(cls); deserializer.open(buffer.inBuffer); dst = deserializer.deserialize(dst); return dst; }
/** Deserializes the object in the given datainput using * available Hadoop serializations. * @throws IOException */ public static<T> T deserialize(Configuration conf, DataInput in , T obj , Class<T> objClass) throws IOException { SerializationFactory serializationFactory = new SerializationFactory(getOrCreateConf(conf)); Deserializer<T> deserializer = serializationFactory.getDeserializer( objClass); int length = WritableUtils.readVInt(in); byte[] arr = new byte[length]; in.readFully(arr); List<ByteBuffer> list = new ArrayList<>(); list.add(ByteBuffer.wrap(arr)); try (ByteBufferInputStream is = new ByteBufferInputStream(list)) { deserializer.open(is); T newObj = deserializer.deserialize(obj); return newObj; }finally { if(deserializer != null) deserializer.close(); } }
private <T> T cloneObj(T t) throws IOException { Serializer<T> keySerializer; Class<T> keyClass; PipedInputStream pis = new PipedInputStream(); PipedOutputStream pos = new PipedOutputStream(pis); keyClass = (Class<T>)t.getClass(); keySerializer = serializationFactory.getSerializer(keyClass); keySerializer.open(pos); keySerializer.serialize(t); Deserializer<T> keyDesiralizer = serializationFactory.getDeserializer(keyClass); keyDesiralizer.open(pis); T clonedArg0 = keyDesiralizer.deserialize(null); pos.close(); pis.close(); keySerializer.close(); keyDesiralizer.close(); return clonedArg0; }
@Test public void testSerializer() throws Exception { WebPage originalWebPage = new WebPage(new URL("http://www.jboss.org"), "opensource", 10L); ByteArrayOutputStream baos = new ByteArrayOutputStream(); JBossMarshallerSerialization<WebPage> marshallerSerialization = new JBossMarshallerSerialization<>(); Serializer<WebPage> serializer = marshallerSerialization.getSerializer(WebPage.class); serializer.open(baos); serializer.serialize(originalWebPage); serializer.close(); Deserializer<WebPage> deserializer = marshallerSerialization.getDeserializer(WebPage.class); deserializer.open(new ByteArrayInputStream(baos.toByteArray())); WebPage deserializedWebPage = deserializer.deserialize(null); deserializer.close(); assertEquals(deserializedWebPage, originalWebPage); }
@SuppressWarnings("unchecked") private void readObject(ObjectInputStream aStream) throws IOException, ClassNotFoundException { String className = aStream.readUTF(); Class clazz; try { clazz = (Class) conf.getClassByName(className); } catch (ClassNotFoundException ce) { throw new IOException("Split class " + className + " not found", ce); } Deserializer deserializer = factory.getDeserializer(clazz); deserializer.open(aStream); split = (InputSplit) deserializer.deserialize(null); }
@SuppressWarnings("unchecked") private static <T> T getSplitDetails(FSDataInputStream inFile, long offset, Configuration configuration) throws IOException { inFile.seek(offset); String className = StringInterner.weakIntern(Text.readString(inFile)); Class<T> cls; try { cls = (Class<T>) configuration.getClassByName(className); } catch (ClassNotFoundException ce) { IOException wrap = new IOException("Split class " + className + " not found"); wrap.initCause(ce); throw wrap; } SerializationFactory factory = new SerializationFactory(configuration); Deserializer<T> deserializer = (Deserializer<T>) factory.getDeserializer(cls); deserializer.open(inFile); T split = deserializer.deserialize(null); return split; }
@SuppressWarnings({ "unchecked", "unused" }) private <T> T getSplitDetails(Path file, long offset) throws IOException { FileSystem fs = file.getFileSystem(getYarnConf()); FSDataInputStream inFile = null; T split = null; try { inFile = fs.open(file); inFile.seek(offset); String className = Text.readString(inFile); Class<T> cls; try { cls = (Class<T>) getYarnConf().getClassByName(className); } catch (ClassNotFoundException ce) { IOException wrap = new IOException(String.format("Split class %s not found", className)); wrap.initCause(ce); throw wrap; } SerializationFactory factory = new SerializationFactory(getYarnConf()); Deserializer<T> deserializer = (Deserializer<T>) factory.getDeserializer(cls); deserializer.open(inFile); split = deserializer.deserialize(null); } finally { IOUtils.closeStream(inFile); } return split; }
