/** * Struct use only. * * @return */ public TField newField() { if (createIndex < obj.length()) { Map.Entry<TFieldIdEnum, FieldMetaData> entry = null; if (addStep == 2) { String fieldName = obj.getString(createIndex << 1); entry = elementMetas.get(fieldName); createIndex++; } else { int i = createIndex; Object o; while (i < obj.length() && ((o = obj.get(i)) == null || o == JSONObject.NULL)) { currentIndex();// array index: +1 i++; } entry = elementMetaArr[i]; createIndex = i + 1; } FieldMetaData fm = entry.getValue(); prevFieldMetaData = fm; return new TField(fm.fieldName, fm.valueMetaData.type, entry.getKey().getThriftFieldId()); } return null; }
/** * 读取操作 */ @Override public void read(TProtocol iprot) throws TException { if (!"org.apache.thrift.scheme.StandardScheme".equals(iprot.getScheme().getName())) throw new TApplicationException("Service scheme must be 'org.apache.thrift.scheme.StandardScheme' !"); TField schemeField; iprot.readStructBegin(); while (Boolean.TRUE) { schemeField = iprot.readFieldBegin(); if (schemeField.type == TType.STOP) break; if (schemeField.type == TType.STRING) str = iprot.readString(); else throw new TApplicationException("field type must be 'String' !"); iprot.readFieldEnd(); } iprot.readStructEnd(); }
private void writeMessage(PMessage<?,?> message, TProtocol protocol) throws TException, SerializerException { PMessageDescriptor<?, ?> type = message.descriptor(); protocol.writeStructBegin(new TStruct(message.descriptor() .getQualifiedName())); for (PField field : type.getFields()) { if (!message.has(field.getId())) { continue; } protocol.writeFieldBegin(new TField(field.getName(), forType(field.getDescriptor().getType()), (short) field.getId())); writeTypedValue(message.get(field.getId()), field.getDescriptor(), protocol); protocol.writeFieldEnd(); } protocol.writeFieldStop(); protocol.writeStructEnd(); }
/** * Deserialize only the id of the field set in a TUnion (addressed by recursively using field id) * from a byte record. * @param bytes The serialized object to read from * @param fieldIdPathFirst First of the FieldId's that define a path to a TUnion * @param fieldIdPathRest The rest FieldId's that define a path to a TUnion * @throws TException */ public Short partialDeserializeSetFieldIdInUnion(byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum ... fieldIdPathRest) throws TException { try { TField field = locateField(bytes, fieldIdPathFirst, fieldIdPathRest); if (field != null){ protocol_.readStructBegin(); // The Union return protocol_.readFieldBegin().id; // The field set in the union } return null; } catch (Exception e) { throw new TException(e); } finally { trans_.clear(); protocol_.reset(); } }
@Override public void read(TProtocol iprot, TUnion struct) throws TException { struct.setField_ = null; struct.value_ = null; iprot.readStructBegin(); TField field = iprot.readFieldBegin(); struct.value_ = struct.standardSchemeReadValue(iprot, field); if (struct.value_ != null) { struct.setField_ = struct.enumForId(field.id); } iprot.readFieldEnd(); // this is so that we will eat the stop byte. we could put a check here to // make sure that it actually *is* the stop byte, but it's faster to do it // this way. iprot.readFieldBegin(); iprot.readStructEnd(); }
@Override public TField readFieldBegin() throws TException { if (!getCurrentContext().hasMoreChildren()) { return new TField("", UNUSED_TYPE, (short) 0); } getCurrentContext().read(); JsonNode jsonName = getCurrentContext().getCurrentChild(); if (!jsonName.isTextual()) { throw new RuntimeException("Expected String for a field name"); } String fieldName = jsonName.asText(); currentFieldClass.push(getCurrentContext().getClassByFieldName(fieldName)); return getCurrentContext().getTFieldByName(fieldName); }
@Override public void writeFieldBegin(TField field) throws TException { if (field.type == TType.STOP) { return; } try { currentType = thriftFieldIdToParquetField[field.id]; if (currentType == null) { throw new ParquetEncodingException("field " + field.id + " was not found in " + thriftType + " and " + schema.getType()); } final int index = currentType.getIndex(); recordConsumer.startField(currentType.getName(), index); currentProtocol = children[index]; } catch (ArrayIndexOutOfBoundsException e) { throw new ParquetEncodingException("field " + field.id + " was not found in " + thriftType + " and " + schema.getType()); } }
private void handleUnrecognizedField(TField field, StructType type, TProtocol in) throws TException { switch (type.getStructOrUnionType()) { case STRUCT: // this is an unrecognized field in a struct, not a union notifyIgnoredFieldsOfRecord(field); //read the value and ignore it, NullProtocol will do nothing new ProtocolReadToWrite().readOneValue(in, new NullProtocol(), field.type); break; case UNION: // this is a union with an unrecognized member -- this is fatal for this record // in the write path, because it will be unreadable in the read path. // throwing here means we will either skip this record entirely, or fail completely. throw new DecodingSchemaMismatchException("Unrecognized union member with id: " + field.id + " for struct:\n" + type); case UNKNOWN: throw unknownStructOrUnion(type); default: throw unrecognizedStructOrUnion(type.getStructOrUnionType()); } }
@Test public void TestExtraFieldWhenFieldIndexIsNotStartFromZero() throws Exception { CountingErrorHandler countingHandler = new CountingErrorHandler() { @Override public void handleFieldIgnored(TField field) { assertEquals(3, field.id); fieldIgnoredCount++; } }; BufferedProtocolReadToWrite structForRead = new BufferedProtocolReadToWrite(ThriftSchemaConverter.toStructType(StructWithIndexStartsFrom4.class), countingHandler); //Data has an extra field of type struct final ByteArrayOutputStream in = new ByteArrayOutputStream(); StructWithExtraField dataWithNewExtraField = new StructWithExtraField(new Phone("111", "222"), new Phone("333", "444")); dataWithNewExtraField.write(protocol(in)); //read using the schema that doesn't have the extra field final ByteArrayOutputStream out = new ByteArrayOutputStream(); structForRead.readOne(protocol(new ByteArrayInputStream(in.toByteArray())), protocol(out)); assertEquals(1, countingHandler.recordCountOfMissingFields); assertEquals(1, countingHandler.fieldIgnoredCount); }
@Override public void after(Object target, Object[] args, Object result, Throwable throwable) { if (isDebug) { logger.afterInterceptor(target, args, result, throwable); } if (!validate(target)) { return; } final boolean shouldTrace = ((ServerMarkerFlagFieldAccessor)target)._$PINPOINT$_getServerMarkerFlag(); if (shouldTrace) { InterceptorScopeInvocation currentTransaction = this.scope.getCurrentInvocation(); Object attachment = currentTransaction.getAttachment(); if (attachment instanceof ThriftClientCallContext) { ThriftClientCallContext clientCallContext = (ThriftClientCallContext)attachment; if (result instanceof TField) { handleClientRequest((TField)result, clientCallContext); } } } }
public void writeTraceHeader(ThriftHeader headerKey, TProtocol oprot) throws TException { Object headerValue = this.thriftHeaders.get(headerKey); if (headerValue == null) { return; } byte headerType = headerKey.getType(); TField traceField = new TField(headerKey.name(), headerKey.getType(), headerKey.getId()); oprot.writeFieldBegin(traceField); try { if (headerType == TType.STRING) { // these will be read as byte buffer although it's probably safe to just use writeString here. // see org.apache.thrift.protocol.TProtocolUtil.skip(TProtocol, byte, int) oprot.writeBinary(stringToByteBuffer((String)headerValue)); } else if (headerType == TType.I64) { oprot.writeI64((Long)headerValue); } else if (headerType == TType.I16) { oprot.writeI16((Short)headerValue); } else if (headerType == TType.BOOL) { oprot.writeBool((Boolean)headerValue); } else { throw new TProtocolException("Invalid pinpoint header type - " + headerType); } } finally { oprot.writeFieldEnd(); } }
private TField getTField(Object thriftObject, String fieldName) throws TException { try { Map<String, ThriftField> classFields = getClassFields(thriftObject); ThriftField thriftField = classFields.get(fieldName); if(thriftField==null) { // Empty field -> skip return new TField(); } byte type = thriftField.fieldMetaData.valueMetaData.type; short id = thriftField.tfieldIdEnum.getThriftFieldId(); // An enum type is deserialized as an I32 if (TType.ENUM == type) { type = TType.I32; } return new TField("", type, id); } catch (Exception exp) { throw new TException("Unexpected getTField fieldName=" + fieldName, exp); } }
public void write(TProtocol oprot) throws TException { TStruct struct = new TStruct("TApplicationException"); TField field = new TField(); oprot.writeStructBegin(struct); if (getMessage() != null) { field.name = "message"; field.type = TType.STRING; field.id = 1; oprot.writeFieldBegin(field); oprot.writeString(getMessage()); oprot.writeFieldEnd(); } field.name = "type"; field.type = TType.I32; field.id = 2; oprot.writeFieldBegin(field); oprot.writeI32(type_); oprot.writeFieldEnd(); oprot.writeFieldStop(); oprot.writeStructEnd(); }
public void read(TProtocol iprot, TUnion struct) throws TException { struct.setField_ = null; struct.value_ = null; iprot.readStructBegin(); TField field = iprot.readFieldBegin(); struct.value_ = struct.standardSchemeReadValue(iprot, field); if (struct.value_ != null) { struct.setField_ = struct.enumForId(field.id); } iprot.readFieldEnd(); // this is so that we will eat the stop byte. we could put a check here to // make sure that it actually *is* the stop byte, but it's faster to do it // this way. iprot.readFieldBegin(); iprot.readStructEnd(); }
/** * 写入操作 */ @Override public void write(TProtocol oprot) throws TException { if (!"org.apache.thrift.scheme.StandardScheme".equals(oprot.getScheme().getName())) throw new TApplicationException("Service scheme must be 'org.apache.thrift.scheme.StandardScheme' !"); oprot.writeStructBegin(getTStruct()); if (str != null) { oprot.writeFieldBegin(new TField("value", TType.STRING, (short) 0)); oprot.writeString(str); oprot.writeFieldEnd(); } oprot.writeFieldStop(); oprot.writeStructEnd(); }
private TField locateField(byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum ... fieldIdPathRest) throws TException { trans_.reset(bytes); TFieldIdEnum[] fieldIdPath= new TFieldIdEnum[fieldIdPathRest.length + 1]; fieldIdPath[0] = fieldIdPathFirst; for (int i = 0; i < fieldIdPathRest.length; i++){ fieldIdPath[i + 1] = fieldIdPathRest[i]; } // index into field ID path being currently searched for int curPathIndex = 0; // this will be the located field, or null if it is not located TField field = null; protocol_.readStructBegin(); while (curPathIndex < fieldIdPath.length) { field = protocol_.readFieldBegin(); // we can stop searching if we either see a stop or we go past the field // id we're looking for (since fields should now be serialized in asc // order). if (field.type == TType.STOP || field.id > fieldIdPath[curPathIndex].getThriftFieldId()) { return null; } if (field.id != fieldIdPath[curPathIndex].getThriftFieldId()) { // Not the field we're looking for. Skip field. TProtocolUtil.skip(protocol_, field.type); protocol_.readFieldEnd(); } else { // This field is the next step in the path. Step into field. curPathIndex++; if (curPathIndex < fieldIdPath.length) { protocol_.readStructBegin(); } } } return field; }
public static TApplicationException read(TProtocol iprot) throws TException { TField field; iprot.readStructBegin(); String message = null; int type = UNKNOWN; while (true) { field = iprot.readFieldBegin(); if (field.type == TType.STOP) { break; } switch (field.id) { case 1: if (field.type == TType.STRING) { message = iprot.readString(); } else { TProtocolUtil.skip(iprot, field.type); } break; case 2: if (field.type == TType.I32) { type = iprot.readI32(); } else { TProtocolUtil.skip(iprot, field.type); } break; default: TProtocolUtil.skip(iprot, field.type); break; } iprot.readFieldEnd(); } iprot.readStructEnd(); return new TApplicationException(type, message); }
/** * Reads a {@link TApplicationException} from the specified {@link TProtocol}. * * <p>Note: This has been copied from {@link TApplicationException#read(TProtocol)} due to API differences * between libthrift 0.9.x and 0.10.x. */ public static TApplicationException read(TProtocol iprot) throws TException { TField field; iprot.readStructBegin(); String message = null; int type = TApplicationException.UNKNOWN; while (true) { field = iprot.readFieldBegin(); if (field.type == TType.STOP) { break; } switch (field.id) { case 1: if (field.type == TType.STRING) { message = iprot.readString(); } else { TProtocolUtil.skip(iprot, field.type); } break; case 2: if (field.type == TType.I32) { type = iprot.readI32(); } else { TProtocolUtil.skip(iprot, field.type); } break; default: TProtocolUtil.skip(iprot, field.type); break; } iprot.readFieldEnd(); } iprot.readStructEnd(); return new TApplicationException(type, message); }
@Override protected TField getTFieldByName(String name) throws TException { if (!fieldNameMap.containsKey(name)) { throw new TException("Unknown field: " + name); } return fieldNameMap.get(name); }
@Override public void writeFieldBegin(TField field) throws TException { try { getCurrentWriter().writeFieldName(field.name); } catch (IOException ex) { throw new TException(ex); } }
@Override public void writeFieldBegin(TField field) throws TException { if (!writeFieldBegin) { protocol.writeFieldBegin(field); if (replaceFields.containsKey(field.name)) { writeFieldBegin = true; currentField = field; } } }
public static void main(String[] args) throws TException { TTransport trans = new TSimpleFileTransport("data.comp", false, true); TProtocol proto = new TCompactProtocol(trans); Trade trade = new Trade(); trade.symbol = "F"; trade.price = 13.10; trade.size = 2500; proto.writeStructBegin(new TStruct()); proto.writeFieldBegin(new TField("symbol", TType.STRING, (short) 1)); proto.writeString(trade.symbol); proto.writeFieldEnd(); proto.writeFieldBegin(new TField("price", TType.DOUBLE, (short) 2)); proto.writeDouble(trade.price); proto.writeFieldEnd(); proto.writeFieldBegin(new TField("size", TType.I32, (short) 3)); proto.writeI32(trade.size); proto.writeFieldEnd(); proto.writeFieldStop(); proto.writeStructEnd(); System.out.println("Wrote trade to file"); }
public static void main(String[] args) throws TException { TTransport trans = new TSimpleFileTransport("data", true, false); TProtocol proto = new TBinaryProtocol(trans); Trade trade_read = new Trade(); TField field = new TField(); TStruct struct_obj = proto.readStructBegin(); while(true) { field = proto.readFieldBegin(); if (field.id == TType.STOP) { break; } switch(field.id) { case 1: trade_read.symbol = proto.readString(); break; case 2: trade_read.price = proto.readDouble(); break; case 3: trade_read.size = proto.readI32(); break; default: TProtocolUtil.skip(proto,field.type); break; } proto.readFieldEnd(); } proto.readStructEnd(); System.out.println("Trade: " + trade_read.symbol + " " + trade_read.size + " @ " + trade_read.price); }
/** * reads the content of a struct (fields) from the underlying protocol and passes the events to c * @param c the field consumer * @throws TException */ public void readStructContent(FieldConsumer c) throws TException { TField field; while (true) { field = protocol.readFieldBegin(); if (field.type == TType.STOP) { break; } c.consumeField(protocol, this, field.id, field.type); } }
public PrimitiveFieldHandler(PrimitiveConverter delegate, final ThriftField field, List<TProtocol> events) { this.delegate = delegate; this.events = events; final byte thriftType = field.getType().getType() == ThriftTypeID.ENUM ? ThriftTypeID.I32.getThriftType() : // enums are serialized as I32 field.getType().getType().getThriftType(); this.readFieldBegin = new ParquetProtocol("readFieldBegin()") { @Override public TField readFieldBegin() throws TException { return new TField(field.getName(), thriftType, field.getFieldId()); } }; }
public GroupFieldhandler(GroupConverter delegate, final ThriftField field, List<TProtocol> events) { this.delegate = delegate; this.events = events; this.readFieldBegin = new ParquetProtocol("readFieldBegin()") { @Override public TField readFieldBegin() throws TException { return new TField(field.getName(), field.getType().getType().getThriftType(), field.getFieldId()); } }; }
private void readOneStruct(TProtocol in, TProtocol out) throws TException { final TStruct struct = in.readStructBegin(); out.writeStructBegin(struct); TField field; while ((field = in.readFieldBegin()).type != TType.STOP) { out.writeFieldBegin(field); readOneValue(in, out, field.type); in.readFieldEnd(); out.writeFieldEnd(); } out.writeFieldStop(); in.readStructEnd(); out.writeStructEnd(); }
/** * When data contains extra field, it should notify the handler and read the data with extra field dropped * @throws Exception */ @Test public void testMissingFieldHandling() throws Exception { CountingErrorHandler countingHandler = new CountingErrorHandler() { @Override public void handleFieldIgnored(TField field) { assertEquals(field.id, 4); fieldIgnoredCount++; } }; BufferedProtocolReadToWrite structForRead = new BufferedProtocolReadToWrite(ThriftSchemaConverter.toStructType(StructV3.class), countingHandler); //Data has an extra field of type struct final ByteArrayOutputStream in = new ByteArrayOutputStream(); StructV4WithExtracStructField dataWithNewSchema = new StructV4WithExtracStructField("name"); dataWithNewSchema.setAge("10"); dataWithNewSchema.setGender("male"); StructV3 structV3 = new StructV3("name"); structV3.setAge("10"); dataWithNewSchema.setAddedStruct(structV3); dataWithNewSchema.write(protocol(in)); //read using the schema that doesn't have the extra field final ByteArrayOutputStream out = new ByteArrayOutputStream(); structForRead.readOne(protocol(new ByteArrayInputStream(in.toByteArray())), protocol(out)); //record will be read without extra field assertEquals(1, countingHandler.recordCountOfMissingFields); assertEquals(1, countingHandler.fieldIgnoredCount); StructV4WithExtracStructField b = StructV4WithExtracStructField.class.newInstance(); b.read(protocol(new ByteArrayInputStream(out.toByteArray()))); assertEquals(dataWithNewSchema.getName(), b.getName()); assertEquals(dataWithNewSchema.getAge(), b.getAge()); assertEquals(dataWithNewSchema.getGender(), b.getGender()); assertEquals(null, b.getAddedStruct()); }
private void handleClientRequest(TField field, ThriftClientCallContext clientCallContext) { ThriftHeader traceHeaderKey = ThriftHeader.findThriftHeaderKeyById(field.id); // check if field is pinpoint header field if (traceHeaderKey == null || field.type != traceHeaderKey.getType()) { clientCallContext.setTraceHeaderToBeRead(NONE); } else { clientCallContext.setTraceHeaderToBeRead(traceHeaderKey); } }
public void writeFieldBegin(TField field) throws TException { // Note that extra type information is omitted in BSON! Context ctx = peekContext(); //TBSONSecuredWrapper.ThriftSecuredField securedField=tbsonSecuredWrapper.getField(ctx.name, field.id); TBSONSecuredWrapper.ThriftSecuredField securedField=tbsonSecuredWrapper.getField(null, field.id); if( securedField.isSecured() && ctx.securedDbObject==null){ ctx.securedDbObject = new BasicDBObject(); } pushContext(new FieldContext(field.name, field.id, securedField)); }
public TField readFieldBegin() throws TException { StructContext context = (StructContext) peekContext(); if(context.fieldsStack.isEmpty()) { // Empty stack -> returns a TType.STOP return new TField(); } String fieldName = context.fieldsStack.peek(); TField currentField = getTField(context.thriftObject, fieldName); //currentField.id // IF the field is skiped change the type to void Map<Class<?>, List<Short>> filter = threadSafeFieldIds.get(); if(filter!=null) { List<Short> fieldsFiltered = filter.get(context.thriftObject.getClass()); if(fieldsFiltered != null && fieldsFiltered.contains(currentField.id)) { return new TField(currentField.name, TType.VOID, currentField.id); } } // If the field is a struct push a struct context in the stack if (currentField.type == TType.STRUCT) { StructContext structContext = new StructContext(fieldName); structContext.setDbObject((DBObject) context.dbObject.get(fieldName)); structContext.thriftObject = getThriftObject(context.thriftObject, fieldName); pushContext(structContext); } return currentField; }