/** * Returns a ConnectionId object. * @param addr Remote address for the connection. * @param protocol Protocol for RPC. * @param ticket UGI * @param rpcTimeout timeout * @param conf Configuration object * @return A ConnectionId instance * @throws IOException */ static ConnectionId getConnectionId(InetSocketAddress addr, Class<?> protocol, UserGroupInformation ticket, int rpcTimeout, RetryPolicy connectionRetryPolicy, Configuration conf) throws IOException { if (connectionRetryPolicy == null) { final int max = conf.getInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_DEFAULT); final int retryInterval = conf.getInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY, CommonConfigurationKeysPublic .IPC_CLIENT_CONNECT_RETRY_INTERVAL_DEFAULT); connectionRetryPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( max, retryInterval, TimeUnit.MILLISECONDS); } return new ConnectionId(addr, protocol, ticket, rpcTimeout, connectionRetryPolicy, conf); }
private static NamenodeProtocol createNNProxyWithNamenodeProtocol( InetSocketAddress address, Configuration conf, UserGroupInformation ugi, boolean withRetries) throws IOException { NamenodeProtocolPB proxy = (NamenodeProtocolPB) createNameNodeProxy( address, conf, ugi, NamenodeProtocolPB.class); if (withRetries) { // create the proxy with retries RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(5, 200, TimeUnit.MILLISECONDS); Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); methodNameToPolicyMap.put("getBlocks", timeoutPolicy); methodNameToPolicyMap.put("getAccessKeys", timeoutPolicy); NamenodeProtocol translatorProxy = new NamenodeProtocolTranslatorPB(proxy); return (NamenodeProtocol) RetryProxy.create( NamenodeProtocol.class, translatorProxy, methodNameToPolicyMap); } else { return new NamenodeProtocolTranslatorPB(proxy); } }
private DFSClient genClientWithDummyHandler() throws IOException { URI nnUri = dfs.getUri(); FailoverProxyProvider<ClientProtocol> failoverProxyProvider = NameNodeProxies.createFailoverProxyProvider(conf, nnUri, ClientProtocol.class, true, null); InvocationHandler dummyHandler = new DummyRetryInvocationHandler( failoverProxyProvider, RetryPolicies .failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL, Integer.MAX_VALUE, DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT, DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT)); ClientProtocol proxy = (ClientProtocol) Proxy.newProxyInstance( failoverProxyProvider.getInterface().getClassLoader(), new Class[] { ClientProtocol.class }, dummyHandler); DFSClient client = new DFSClient(null, proxy, conf, null); return client; }
private static NativeFileSystemStore createDefaultStore(Configuration conf) { NativeFileSystemStore store = new Jets3tNativeFileSystemStore(); RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( conf.getInt("fs.s3.maxRetries", 4), conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS); Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); exceptionToPolicyMap.put(IOException.class, basePolicy); exceptionToPolicyMap.put(S3Exception.class, basePolicy); RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); methodNameToPolicyMap.put("storeFile", methodPolicy); methodNameToPolicyMap.put("rename", methodPolicy); return (NativeFileSystemStore) RetryProxy.create(NativeFileSystemStore.class, store, methodNameToPolicyMap); }
private static FileSystemStore createDefaultStore(Configuration conf) { FileSystemStore store = new Jets3tFileSystemStore(); RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( conf.getInt("fs.s3.maxRetries", 4), conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS); Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); exceptionToPolicyMap.put(IOException.class, basePolicy); exceptionToPolicyMap.put(S3Exception.class, basePolicy); RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>(); methodNameToPolicyMap.put("storeBlock", methodPolicy); methodNameToPolicyMap.put("retrieveBlock", methodPolicy); return (FileSystemStore) RetryProxy.create(FileSystemStore.class, store, methodNameToPolicyMap); }
/** * Creates an explicitly HA-enabled proxy object. * * @param conf the configuration object * @param nameNodeUri the URI pointing either to a specific NameNode or to a * logical nameservice. * @param xface the IPC interface which should be created * @param failoverProxyProvider Failover proxy provider * @return an object containing both the proxy and the associated * delegation token service it corresponds to */ @SuppressWarnings("unchecked") public static <T> ProxyAndInfo<T> createHAProxy( Configuration conf, URI nameNodeUri, Class<T> xface, AbstractNNFailoverProxyProvider<T> failoverProxyProvider) { Preconditions.checkNotNull(failoverProxyProvider); // HA case DfsClientConf config = new DfsClientConf(conf); T proxy = (T) RetryProxy.create(xface, failoverProxyProvider, RetryPolicies.failoverOnNetworkException( RetryPolicies.TRY_ONCE_THEN_FAIL, config.getMaxFailoverAttempts(), config.getMaxRetryAttempts(), config.getFailoverSleepBaseMillis(), config.getFailoverSleepMaxMillis())); Text dtService; if (failoverProxyProvider.useLogicalURI()) { dtService = HAUtilClient.buildTokenServiceForLogicalUri(nameNodeUri, HdfsConstants.HDFS_URI_SCHEME); } else { dtService = SecurityUtil.buildTokenService( DFSUtilClient.getNNAddress(nameNodeUri)); } return new ProxyAndInfo<>(proxy, dtService, DFSUtilClient.getNNAddressCheckLogical(conf, nameNodeUri)); }
private static NamenodeProtocol createNNProxyWithNamenodeProtocol( InetSocketAddress address, Configuration conf, UserGroupInformation ugi, boolean withRetries) throws IOException { NamenodeProtocolPB proxy = (NamenodeProtocolPB) createNameNodeProxy( address, conf, ugi, NamenodeProtocolPB.class, 0); if (withRetries) { // create the proxy with retries RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(5, 200, TimeUnit.MILLISECONDS); Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); methodNameToPolicyMap.put("getBlocks", timeoutPolicy); methodNameToPolicyMap.put("getAccessKeys", timeoutPolicy); NamenodeProtocol translatorProxy = new NamenodeProtocolTranslatorPB(proxy); return (NamenodeProtocol) RetryProxy.create( NamenodeProtocol.class, translatorProxy, methodNameToPolicyMap); } else { return new NamenodeProtocolTranslatorPB(proxy); } }
private DFSClient genClientWithDummyHandler() throws IOException { URI nnUri = dfs.getUri(); FailoverProxyProvider<ClientProtocol> failoverProxyProvider = NameNodeProxiesClient.createFailoverProxyProvider(conf, nnUri, ClientProtocol.class, true, null); InvocationHandler dummyHandler = new DummyRetryInvocationHandler( failoverProxyProvider, RetryPolicies .failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL, Integer.MAX_VALUE, HdfsClientConfigKeys.Failover.SLEEPTIME_BASE_DEFAULT, HdfsClientConfigKeys.Failover.SLEEPTIME_MAX_DEFAULT)); ClientProtocol proxy = (ClientProtocol) Proxy.newProxyInstance( failoverProxyProvider.getInterface().getClassLoader(), new Class[] { ClientProtocol.class }, dummyHandler); DFSClient client = new DFSClient(null, proxy, conf, null); return client; }
public static RaidProtocol createRaidnode(RaidProtocol rpcRaidnode) throws IOException { Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); exceptionToPolicyMap.put(RemoteException.class, RetryPolicies.retryByRemoteException( RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap)); RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>(); methodNameToPolicyMap.put("create", methodPolicy); return (RaidProtocol) RetryProxy.create(RaidProtocol.class, rpcRaidnode, methodNameToPolicyMap); }
private static SnapshotProtocol createSnapshotNode(SnapshotProtocol rpcSnapshotNode) throws IOException { RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( 5, 5000, TimeUnit.MILLISECONDS); Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); exceptionToPolicyMap.put(RemoteException.class, RetryPolicies.retryByRemoteException( RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap)); RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>(); methodNameToPolicyMap.put("create", methodPolicy); return (SnapshotProtocol) RetryProxy.create(SnapshotProtocol.class, rpcSnapshotNode, methodNameToPolicyMap); }
private static AvatarProtocol createAvatarnode(AvatarProtocol rpcAvatarnode) throws IOException { Map<Class<? extends Exception>, RetryPolicy> remoteExceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); exceptionToPolicyMap.put(RemoteException.class, RetryPolicies .retryByRemoteException(RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap)); RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); methodNameToPolicyMap.put("create", methodPolicy); return (AvatarProtocol) RetryProxy.create(AvatarProtocol.class, rpcAvatarnode, methodNameToPolicyMap); }
private static NativeFileSystemStore createDefaultStore(Configuration conf) { NativeFileSystemStore store = new Jets3tNativeFileSystemStore(); RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( conf.getInt("fs.s3.maxRetries", 4), conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS); Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); exceptionToPolicyMap.put(IOException.class, basePolicy); exceptionToPolicyMap.put(S3Exception.class, basePolicy); RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); methodNameToPolicyMap.put("storeFile", methodPolicy); return (NativeFileSystemStore) RetryProxy.create(NativeFileSystemStore.class, store, methodNameToPolicyMap); }
static ClientProtocol createNamenode(ClientProtocol rpcNamenode, Configuration conf) throws IOException { long sleepTime = conf.getLong("dfs.client.rpc.retry.sleep", LEASE_SOFTLIMIT_PERIOD); RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( 5, sleepTime, TimeUnit.MILLISECONDS); Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class, createPolicy); Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); exceptionToPolicyMap.put(RemoteException.class, RetryPolicies.retryByRemoteException( RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap)); RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>(); methodNameToPolicyMap.put("create", methodPolicy); return (ClientProtocol) RetryProxy.create(ClientProtocol.class, rpcNamenode, methodNameToPolicyMap); }
private static HighTideProtocol createHighTidenode(HighTideProtocol rpcHighTidenode) throws IOException { RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( 5, 5000, TimeUnit.MILLISECONDS); Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); exceptionToPolicyMap.put(RemoteException.class, RetryPolicies.retryByRemoteException( RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap)); RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>(); methodNameToPolicyMap.put("create", methodPolicy); return (HighTideProtocol) RetryProxy.create(HighTideProtocol.class, rpcHighTidenode, methodNameToPolicyMap); }
/** * Creates the namenode proxy with the passed protocol. This will handle * creation of either HA- or non-HA-enabled proxy objects, depending upon * if the provided URI is a configured logical URI. * * @param conf the configuration containing the required IPC * properties, client failover configurations, etc. * @param nameNodeUri the URI pointing either to a specific NameNode * or to a logical nameservice. * @param xface the IPC interface which should be created * @return an object containing both the proxy and the associated * delegation token service it corresponds to * @throws IOException if there is an error creating the proxy **/ @SuppressWarnings("unchecked") public static <T> ProxyAndInfo<T> createProxy(Configuration conf, URI nameNodeUri, Class<T> xface) throws IOException { Class<FailoverProxyProvider<T>> failoverProxyProviderClass = getFailoverProxyProviderClass(conf, nameNodeUri, xface); if (failoverProxyProviderClass == null) { // Non-HA case return createNonHAProxy(conf, NameNode.getAddress(nameNodeUri), xface, UserGroupInformation.getCurrentUser(), true); } else { // HA case FailoverProxyProvider<T> failoverProxyProvider = NameNodeProxies .createFailoverProxyProvider(conf, failoverProxyProviderClass, xface, nameNodeUri); Conf config = new Conf(conf); T proxy = (T) RetryProxy.create(xface, failoverProxyProvider, RetryPolicies .failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL, config.maxFailoverAttempts, config.failoverSleepBaseMillis, config.failoverSleepMaxMillis)); Text dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri); return new ProxyAndInfo<T>(proxy, dtService); } }
/** Create a {@link NameNode} proxy */ static DatanodeProtocolPB createNamenodeWithRetry( DatanodeProtocolPB rpcNamenode) { RetryPolicy createPolicy = RetryPolicies .retryUpToMaximumCountWithFixedSleep(5, HdfsConstants.LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS); Map<Class<? extends Exception>, RetryPolicy> remoteExceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class, createPolicy); Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); exceptionToPolicyMap.put(RemoteException.class, RetryPolicies .retryByRemoteException(RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap)); RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); methodNameToPolicyMap.put("create", methodPolicy); return (DatanodeProtocolPB) RetryProxy.create(DatanodeProtocolPB.class, rpcNamenode, methodNameToPolicyMap); }
private DFSClient genClientWithDummyHandler() throws IOException { URI nnUri = dfs.getUri(); Class<FailoverProxyProvider<ClientProtocol>> failoverProxyProviderClass = NameNodeProxies.getFailoverProxyProviderClass(conf, nnUri, ClientProtocol.class); FailoverProxyProvider<ClientProtocol> failoverProxyProvider = NameNodeProxies.createFailoverProxyProvider(conf, failoverProxyProviderClass, ClientProtocol.class, nnUri); InvocationHandler dummyHandler = new DummyRetryInvocationHandler( failoverProxyProvider, RetryPolicies .failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL, Integer.MAX_VALUE, DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT, DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT)); ClientProtocol proxy = (ClientProtocol) Proxy.newProxyInstance( failoverProxyProvider.getInterface().getClassLoader(), new Class[] { ClientProtocol.class }, dummyHandler); DFSClient client = new DFSClient(null, proxy, conf, null); return client; }