public static List<XAttrProtos.XAttrProto> convertXAttrProto( List<XAttr> xAttrSpec) { if (xAttrSpec == null) { return Lists.newArrayListWithCapacity(0); } ArrayList<XAttrProtos.XAttrProto> xAttrs = Lists.newArrayListWithCapacity( xAttrSpec.size()); for (XAttr a : xAttrSpec) { XAttrProtos.XAttrProto.Builder builder = XAttrProtos.XAttrProto.newBuilder(); builder.setNamespace(convert(a.getNameSpace())); if (a.getName() != null) { builder.setName(a.getName()); } if (a.getValue() != null) { builder.setValue(getByteString(a.getValue())); } xAttrs.add(builder.build()); } return xAttrs; }
@Override public void setXAttrs(Map<String, List<XAttr>> xAttrsToSet, EnumSet<XAttrSetFlag> flag) throws IOException { SetXAttrsRequestProto.Builder builder = SetXAttrsRequestProto.newBuilder(); for (String src : xAttrsToSet.keySet()) { List<XAttr> xAttrs = xAttrsToSet.get(src); SetXAttrSingleRequestProto singleReq = SetXAttrSingleRequestProto.newBuilder() .setSrc(src) .addAllXAttr(NuCypherExtPBHelperClient.convertXAttrProto(xAttrs)) .build(); builder.addXAttrsToSet(singleReq); } SetXAttrsRequestProto req = builder.setFlag(NuCypherExtPBHelperClient.convert(flag)).build(); try { rpcProxy.setXAttrs(null, req); } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } }
/** * Build xattr map from <code>XAttr</code> list, the key is * xattr name with prefix, and value is xattr value. */ public static Map<String, byte[]> buildXAttrMap(List<XAttr> xAttrs) { if (xAttrs == null) { return null; } Map<String, byte[]> xAttrMap = Maps.newHashMap(); for (XAttr xAttr : xAttrs) { String name = getPrefixName(xAttr); byte[] value = xAttr.getValue(); if (value == null) { value = new byte[0]; } xAttrMap.put(name, value); } return xAttrMap; }
private static void appendXAttrsToXml(ContentHandler contentHandler, List<XAttr> xAttrs) throws SAXException { for (XAttr xAttr: xAttrs) { contentHandler.startElement("", "", "XATTR", new AttributesImpl()); XMLUtils.addSaxString(contentHandler, "NAMESPACE", xAttr.getNameSpace().toString()); XMLUtils.addSaxString(contentHandler, "NAME", xAttr.getName()); if (xAttr.getValue() != null) { try { XMLUtils.addSaxString(contentHandler, "VALUE", XAttrCodec.encodeValue(xAttr.getValue(), XAttrCodec.HEX)); } catch (IOException e) { throw new SAXException(e); } } contentHandler.endElement("", "", "XATTR"); } }
private static List<XAttr> readXAttrsFromXml(Stanza st) throws InvalidXmlException { if (!st.hasChildren("XATTR")) { return null; } List<Stanza> stanzas = st.getChildren("XATTR"); List<XAttr> xattrs = Lists.newArrayListWithCapacity(stanzas.size()); for (Stanza a: stanzas) { XAttr.Builder builder = new XAttr.Builder(); builder.setNameSpace(XAttr.NameSpace.valueOf(a.getValue("NAMESPACE"))). setName(a.getValue("NAME")); String v = a.getValueOrNull("VALUE"); if (v != null) { try { builder.setValue(XAttrCodec.decodeValue(v)); } catch (IOException e) { throw new InvalidXmlException(e.toString()); } } xattrs.add(builder.build()); } return xattrs; }
/** * Set the FileEncryptionInfo for an INode. */ void setFileEncryptionInfo(String src, FileEncryptionInfo info) throws IOException { // Make the PB for the xattr final HdfsProtos.PerFileEncryptionInfoProto proto = PBHelper.convertPerFileEncInfo(info); final byte[] protoBytes = proto.toByteArray(); final XAttr fileEncryptionAttr = XAttrHelper.buildXAttr(CRYPTO_XATTR_FILE_ENCRYPTION_INFO, protoBytes); final List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1); xAttrs.add(fileEncryptionAttr); writeLock(); try { FSDirXAttrOp.unprotectedSetXAttrs(this, src, xAttrs, EnumSet.of(XAttrSetFlag.CREATE)); } finally { writeUnlock(); } }
void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag, boolean logRetryCache) throws IOException { checkOperation(OperationCategory.WRITE); HdfsFileStatus auditStat = null; writeLock(); try { checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot set XAttr on " + src); auditStat = FSDirXAttrOp.setXAttr(dir, src, xAttr, flag, logRetryCache); } catch (AccessControlException e) { logAuditEvent(false, "setXAttr", src); throw e; } finally { writeUnlock(); } getEditLog().logSync(); logAuditEvent(true, "setXAttr", src, null, auditStat); }
void removeXAttr(String src, XAttr xAttr, boolean logRetryCache) throws IOException { checkOperation(OperationCategory.WRITE); HdfsFileStatus auditStat = null; writeLock(); try { checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot remove XAttr entry on " + src); auditStat = FSDirXAttrOp.removeXAttr(dir, src, xAttr, logRetryCache); } catch (AccessControlException e) { logAuditEvent(false, "removeXAttr", src); throw e; } finally { writeUnlock(); } getEditLog().logSync(); logAuditEvent(true, "removeXAttr", src, null, auditStat); }
public static ImmutableList<XAttr> loadXAttrs( XAttrFeatureProto proto, final String[] stringTable) { ImmutableList.Builder<XAttr> b = ImmutableList.builder(); for (XAttrCompactProto xAttrCompactProto : proto.getXAttrsList()) { int v = xAttrCompactProto.getName(); int nid = (v >> XATTR_NAME_OFFSET) & XATTR_NAME_MASK; int ns = (v >> XATTR_NAMESPACE_OFFSET) & XATTR_NAMESPACE_MASK; ns |= ((v >> XATTR_NAMESPACE_EXT_OFFSET) & XATTR_NAMESPACE_EXT_MASK) << 2; String name = stringTable[nid]; byte[] value = null; if (xAttrCompactProto.getValue() != null) { value = xAttrCompactProto.getValue().toByteArray(); } b.add(new XAttr.Builder().setNameSpace(XATTR_NAMESPACE_VALUES[ns]) .setName(name).setValue(value).build()); } return b.build(); }
private static XAttrFeatureProto.Builder buildXAttrs(XAttrFeature f, final SaverContext.DeduplicationMap<String> stringMap) { XAttrFeatureProto.Builder b = XAttrFeatureProto.newBuilder(); for (XAttr a : f.getXAttrs()) { XAttrCompactProto.Builder xAttrCompactBuilder = XAttrCompactProto. newBuilder(); int nsOrd = a.getNameSpace().ordinal(); Preconditions.checkArgument(nsOrd < 8, "Too many namespaces."); int v = ((nsOrd & XATTR_NAMESPACE_MASK) << XATTR_NAMESPACE_OFFSET) | ((stringMap.getId(a.getName()) & XATTR_NAME_MASK) << XATTR_NAME_OFFSET); v |= (((nsOrd >> 2) & XATTR_NAMESPACE_EXT_MASK) << XATTR_NAMESPACE_EXT_OFFSET); xAttrCompactBuilder.setName(v); if (a.getValue() != null) { xAttrCompactBuilder.setValue(PBHelper.getByteString(a.getValue())); } b.addXAttrs(xAttrCompactBuilder.build()); } return b; }
@Override // ClientProtocol public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag) throws IOException { checkNNStartup(); CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); if (cacheEntry != null && cacheEntry.isSuccess()) { return; // Return previous response } boolean success = false; try { namesystem.setXAttr(src, xAttr, flag, cacheEntry != null); success = true; } finally { RetryCache.setState(cacheEntry, success); } }
static void checkPermissionForApi(FSPermissionChecker pc, XAttr xAttr, boolean isRawPath) throws AccessControlException { final boolean isSuperUser = pc.isSuperUser(); if (xAttr.getNameSpace() == XAttr.NameSpace.USER || (xAttr.getNameSpace() == XAttr.NameSpace.TRUSTED && isSuperUser)) { return; } if (xAttr.getNameSpace() == XAttr.NameSpace.RAW && isRawPath && isSuperUser) { return; } if (XAttrHelper.getPrefixName(xAttr). equals(SECURITY_XATTR_UNREADABLE_BY_SUPERUSER)) { if (xAttr.getValue() != null) { throw new AccessControlException("Attempt to set a value for '" + SECURITY_XATTR_UNREADABLE_BY_SUPERUSER + "'. Values are not allowed for this xattr."); } return; } throw new AccessControlException("User doesn't have permission for xattr: " + XAttrHelper.getPrefixName(xAttr)); }
static List<XAttr> filterXAttrsForApi(FSPermissionChecker pc, List<XAttr> xAttrs, boolean isRawPath) { assert xAttrs != null : "xAttrs can not be null"; if (xAttrs.isEmpty()) { return xAttrs; } List<XAttr> filteredXAttrs = Lists.newArrayListWithCapacity(xAttrs.size()); final boolean isSuperUser = pc.isSuperUser(); for (XAttr xAttr : xAttrs) { if (xAttr.getNameSpace() == XAttr.NameSpace.USER) { filteredXAttrs.add(xAttr); } else if (xAttr.getNameSpace() == XAttr.NameSpace.TRUSTED && isSuperUser) { filteredXAttrs.add(xAttr); } else if (xAttr.getNameSpace() == XAttr.NameSpace.RAW && isSuperUser && isRawPath) { filteredXAttrs.add(xAttr); } else if (XAttrHelper.getPrefixName(xAttr). equals(SECURITY_XATTR_UNREADABLE_BY_SUPERUSER)) { filteredXAttrs.add(xAttr); } } return filteredXAttrs; }
/** * Set xattr for a file or directory. * * @param src * - path on which it sets the xattr * @param xAttr * - xAttr details to set * @param flag * - xAttrs flags * @throws IOException */ static HdfsFileStatus setXAttr( FSDirectory fsd, String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag, boolean logRetryCache) throws IOException { checkXAttrsConfigFlag(fsd); checkXAttrSize(fsd, xAttr); FSPermissionChecker pc = fsd.getPermissionChecker(); XAttrPermissionFilter.checkPermissionForApi( pc, xAttr, FSDirectory.isReservedRawName(src)); byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src); src = fsd.resolvePath(pc, src, pathComponents); List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1); xAttrs.add(xAttr); INodesInPath iip; fsd.writeLock(); try { iip = fsd.getINodesInPath4Write(src); checkXAttrChangeAccess(fsd, iip, xAttr, pc); unprotectedSetXAttrs(fsd, src, xAttrs, flag); } finally { fsd.writeUnlock(); } fsd.getEditLog().logSetXAttrs(src, xAttrs, logRetryCache); return fsd.getAuditFileInfo(iip); }
static List<XAttr> listXAttrs( FSDirectory fsd, String src) throws IOException { FSDirXAttrOp.checkXAttrsConfigFlag(fsd); final FSPermissionChecker pc = fsd.getPermissionChecker(); final boolean isRawPath = FSDirectory.isReservedRawName(src); byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src); src = fsd.resolvePath(pc, src, pathComponents); final INodesInPath iip = fsd.getINodesInPath(src, true); if (fsd.isPermissionEnabled()) { /* To access xattr names, you need EXECUTE in the owning directory. */ fsd.checkParentAccess(pc, iip, FsAction.EXECUTE); } final List<XAttr> all = FSDirXAttrOp.getXAttrs(fsd, src); return XAttrPermissionFilter. filterXAttrsForApi(pc, all, isRawPath); }
static List<XAttr> unprotectedRemoveXAttrs( FSDirectory fsd, final String src, final List<XAttr> toRemove) throws IOException { assert fsd.hasWriteLock(); INodesInPath iip = fsd.getINodesInPath4Write( FSDirectory.normalizePath(src), true); INode inode = FSDirectory.resolveLastINode(iip); int snapshotId = iip.getLatestSnapshotId(); List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode); List<XAttr> removedXAttrs = Lists.newArrayListWithCapacity(toRemove.size()); List<XAttr> newXAttrs = filterINodeXAttrs(existingXAttrs, toRemove, removedXAttrs); if (existingXAttrs.size() != newXAttrs.size()) { XAttrStorage.updateINodeXAttrs(inode, newXAttrs, snapshotId); return removedXAttrs; } return null; }
private static void checkXAttrChangeAccess( FSDirectory fsd, INodesInPath iip, XAttr xAttr, FSPermissionChecker pc) throws AccessControlException { if (fsd.isPermissionEnabled() && xAttr.getNameSpace() == XAttr.NameSpace .USER) { final INode inode = iip.getLastINode(); if (inode != null && inode.isDirectory() && inode.getFsPermission().getStickyBit()) { if (!pc.isSuperUser()) { fsd.checkOwner(pc, iip); } } else { fsd.checkPathAccess(pc, iip, FsAction.WRITE); } } }
/** * Verifies that the combined size of the name and value of an xattr is within * the configured limit. Setting a limit of zero disables this check. */ private static void checkXAttrSize(FSDirectory fsd, XAttr xAttr) { if (fsd.getXattrMaxSize() == 0) { return; } int size = xAttr.getName().getBytes(Charsets.UTF_8).length; if (xAttr.getValue() != null) { size += xAttr.getValue().length; } if (size > fsd.getXattrMaxSize()) { throw new HadoopIllegalArgumentException( "The XAttr is too big. The maximum combined size of the" + " name and value is " + fsd.getXattrMaxSize() + ", but the total size is " + size); } }
public static List<XAttrProto> convertXAttrProto( List<XAttr> xAttrSpec) { if (xAttrSpec == null) { return Lists.newArrayListWithCapacity(0); } ArrayList<XAttrProto> xAttrs = Lists.newArrayListWithCapacity( xAttrSpec.size()); for (XAttr a : xAttrSpec) { XAttrProto.Builder builder = XAttrProto.newBuilder(); builder.setNamespace(convert(a.getNameSpace())); if (a.getName() != null) { builder.setName(a.getName()); } if (a.getValue() != null) { builder.setValue(getByteString(a.getValue())); } xAttrs.add(builder.build()); } return xAttrs; }
@Test public void testToJsonFromXAttrs() throws IOException { String jsonString = "{\"XAttrs\":[{\"name\":\"user.a1\",\"value\":\"0x313233\"}," + "{\"name\":\"user.a2\",\"value\":\"0x313131\"}]}"; XAttr xAttr1 = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.USER). setName("a1").setValue(XAttrCodec.decodeValue("0x313233")).build(); XAttr xAttr2 = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.USER). setName("a2").setValue(XAttrCodec.decodeValue("0x313131")).build(); List<XAttr> xAttrs = Lists.newArrayList(); xAttrs.add(xAttr1); xAttrs.add(xAttr2); Assert.assertEquals(jsonString, JsonUtil.toJsonString(xAttrs, XAttrCodec.HEX)); }
@Test public void testToXAttrMap() throws IOException { String jsonString = "{\"XAttrs\":[{\"name\":\"user.a1\",\"value\":\"0x313233\"}," + "{\"name\":\"user.a2\",\"value\":\"0x313131\"}]}"; ObjectReader reader = new ObjectMapper().reader(Map.class); Map<?, ?> json = reader.readValue(jsonString); XAttr xAttr1 = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.USER). setName("a1").setValue(XAttrCodec.decodeValue("0x313233")).build(); XAttr xAttr2 = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.USER). setName("a2").setValue(XAttrCodec.decodeValue("0x313131")).build(); List<XAttr> xAttrs = Lists.newArrayList(); xAttrs.add(xAttr1); xAttrs.add(xAttr2); Map<String, byte[]> xAttrMap = XAttrHelper.buildXAttrMap(xAttrs); Map<String, byte[]> parsedXAttrMap = JsonUtil.toXAttrs(json); Assert.assertEquals(xAttrMap.size(), parsedXAttrMap.size()); Iterator<Entry<String, byte[]>> iter = xAttrMap.entrySet().iterator(); while(iter.hasNext()) { Entry<String, byte[]> entry = iter.next(); Assert.assertArrayEquals(entry.getValue(), parsedXAttrMap.get(entry.getKey())); } }
/** * Test that xattrs are properly tracked by the standby */ @Test(timeout = 60000) public void testXAttrsTrackedOnStandby() throws Exception { fs.create(path).close(); fs.setXAttr(path, name1, value1, EnumSet.of(XAttrSetFlag.CREATE)); fs.setXAttr(path, name2, value2, EnumSet.of(XAttrSetFlag.CREATE)); HATestUtil.waitForStandbyToCatchUp(nn0, nn1); List<XAttr> xAttrs = nn1.getRpcServer().getXAttrs("/file", null); assertEquals(2, xAttrs.size()); cluster.shutdownNameNode(0); // Failover the current standby to active. cluster.shutdownNameNode(0); cluster.transitionToActive(1); Map<String, byte[]> xattrs = fs.getXAttrs(path); Assert.assertEquals(xattrs.size(), 2); Assert.assertArrayEquals(value1, xattrs.get(name1)); Assert.assertArrayEquals(value2, xattrs.get(name2)); fs.delete(path, true); }
@Test public void testXAttrFeature() { replication = 3; preferredBlockSize = 128*1024*1024; INodeFile inf = createINodeFile(replication, preferredBlockSize); ImmutableList.Builder<XAttr> builder = new ImmutableList.Builder<XAttr>(); XAttr xAttr = new XAttr.Builder().setNameSpace(XAttr.NameSpace.USER). setName("a1").setValue(new byte[]{0x31, 0x32, 0x33}).build(); builder.add(xAttr); XAttrFeature f = new XAttrFeature(builder.build()); inf.addXAttrFeature(f); XAttrFeature f1 = inf.getXAttrFeature(); assertEquals(xAttr, f1.getXAttrs().get(0)); inf.removeXAttrFeature(); f1 = inf.getXAttrFeature(); assertEquals(f1, null); }
/** * Build xattr map from <code>XAttr</code> list, the key is * xattr name with prefix, and value is xattr value. */ public static Map<String, byte[]> buildXAttrMap(List<XAttr> xAttrs) { if (xAttrs == null) { return null; } Map<String, byte[]> xAttrMap = Maps.newHashMap(); for (XAttr xAttr : xAttrs) { String name = getPrefixedName(xAttr); byte[] value = xAttr.getValue(); if (value == null) { value = new byte[0]; } xAttrMap.put(name, value); } return xAttrMap; }
public XAttrFeature(List<XAttr> xAttrs) { if (xAttrs != null && !xAttrs.isEmpty()) { List<XAttr> toPack = new ArrayList<XAttr>(); ImmutableList.Builder<XAttr> b = null; for (XAttr attr : xAttrs) { if (attr.getValue() == null || attr.getValue().length <= PACK_THRESHOLD) { toPack.add(attr); } else { if (b == null) { b = ImmutableList.builder(); } b.add(attr); } } this.attrs = XAttrFormat.toBytes(toPack); if (b != null) { this.xAttrs = b.build(); } } }
private static XAttrFeatureProto.Builder buildXAttrs(XAttrFeature f, final SaverContext.DeduplicationMap<String> stringMap) { XAttrFeatureProto.Builder b = XAttrFeatureProto.newBuilder(); for (XAttr a : f.getXAttrs()) { XAttrCompactProto.Builder xAttrCompactBuilder = XAttrCompactProto. newBuilder(); int nsOrd = a.getNameSpace().ordinal(); Preconditions.checkArgument(nsOrd < 8, "Too many namespaces."); int v = ((nsOrd & XATTR_NAMESPACE_MASK) << XATTR_NAMESPACE_OFFSET) | ((stringMap.getId(a.getName()) & XATTR_NAME_MASK) << XATTR_NAME_OFFSET); v |= (((nsOrd >> 2) & XATTR_NAMESPACE_EXT_MASK) << XATTR_NAMESPACE_EXT_OFFSET); xAttrCompactBuilder.setName(v); if (a.getValue() != null) { xAttrCompactBuilder.setValue(PBHelperClient.getByteString(a.getValue())); } b.addXAttrs(xAttrCompactBuilder.build()); } return b; }
@Override // ClientProtocol public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag) throws IOException { checkNNStartup(); namesystem.checkOperation(OperationCategory.WRITE); CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); if (cacheEntry != null && cacheEntry.isSuccess()) { return; // Return previous response } boolean success = false; try { namesystem.setXAttr(src, xAttr, flag, cacheEntry != null); success = true; } finally { RetryCache.setState(cacheEntry, success); } }
static void checkPermissionForApi(FSPermissionChecker pc, XAttr xAttr, boolean isRawPath) throws AccessControlException { final boolean isSuperUser = pc.isSuperUser(); if (xAttr.getNameSpace() == XAttr.NameSpace.USER || (xAttr.getNameSpace() == XAttr.NameSpace.TRUSTED && isSuperUser)) { return; } if (xAttr.getNameSpace() == XAttr.NameSpace.RAW && isRawPath && isSuperUser) { return; } if (XAttrHelper.getPrefixedName(xAttr). equals(SECURITY_XATTR_UNREADABLE_BY_SUPERUSER)) { if (xAttr.getValue() != null) { throw new AccessControlException("Attempt to set a value for '" + SECURITY_XATTR_UNREADABLE_BY_SUPERUSER + "'. Values are not allowed for this xattr."); } return; } throw new AccessControlException("User doesn't have permission for xattr: " + XAttrHelper.getPrefixedName(xAttr)); }
static List<XAttr> filterXAttrsForApi(FSPermissionChecker pc, List<XAttr> xAttrs, boolean isRawPath) { assert xAttrs != null : "xAttrs can not be null"; if (xAttrs.isEmpty()) { return xAttrs; } List<XAttr> filteredXAttrs = Lists.newArrayListWithCapacity(xAttrs.size()); final boolean isSuperUser = pc.isSuperUser(); for (XAttr xAttr : xAttrs) { if (xAttr.getNameSpace() == XAttr.NameSpace.USER) { filteredXAttrs.add(xAttr); } else if (xAttr.getNameSpace() == XAttr.NameSpace.TRUSTED && isSuperUser) { filteredXAttrs.add(xAttr); } else if (xAttr.getNameSpace() == XAttr.NameSpace.RAW && isSuperUser && isRawPath) { filteredXAttrs.add(xAttr); } else if (XAttrHelper.getPrefixedName(xAttr). equals(SECURITY_XATTR_UNREADABLE_BY_SUPERUSER)) { filteredXAttrs.add(xAttr); } } return filteredXAttrs; }