Java 类org.apache.http.client.protocol.HttpClientContext 实例源码

public DatarouterHttpResponse(HttpResponse response, HttpClientContext context,
        Consumer<HttpEntity> httpEntityConsumer){
    this.response = response;
    this.cookies = context.getCookieStore().getCookies();
    if(response != null){
        this.statusCode = response.getStatusLine().getStatusCode();
        this.entity = "";

        HttpEntity httpEntity = response.getEntity();
        if(httpEntity == null){
            return;
        }
        if(httpEntityConsumer != null){
            httpEntityConsumer.accept(httpEntity);
            return;
        }
        try{
            this.entity = EntityUtils.toString(httpEntity);
        }catch(IOException e){
            logger.error("Exception occurred while reading HTTP response entity", e);
        }finally{
            EntityUtils.consumeQuietly(httpEntity);
        }
    }
}

public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {

            AuthState authState = (AuthState) context.getAttribute(HttpClientContext.TARGET_AUTH_STATE);

            if (authState.getAuthScheme() != null || authState.hasAuthOptions()) {
                return;
            }

            // If no authState has been established and this is a PUT or POST request, add preemptive authorisation
            String requestMethod = request.getRequestLine().getMethod();
            if (alwaysSendAuth || requestMethod.equals(HttpPut.METHOD_NAME) || requestMethod.equals(HttpPost.METHOD_NAME)) {
                CredentialsProvider credentialsProvider = (CredentialsProvider) context.getAttribute(HttpClientContext.CREDS_PROVIDER);
                HttpHost targetHost = (HttpHost) context.getAttribute(HttpCoreContext.HTTP_TARGET_HOST);
                Credentials credentials = credentialsProvider.getCredentials(new AuthScope(targetHost.getHostName(), targetHost.getPort()));
                if (credentials == null) {
                    throw new HttpException("No credentials for preemptive authentication");
                }
                authState.update(authScheme, credentials);
            }
        }

public Map<String, String> connect(String url, Map<String, Object> parameters) throws ManagerResponseException {

        Map<String, String> response = new HashMap<String, String>();
        CloseableHttpClient httpclient = HttpClients.createDefault();
        List<NameValuePair> nvps = new ArrayList<>();
        nvps.add(new BasicNameValuePair("j_username", (String) parameters.get("login")));
        nvps.add(new BasicNameValuePair("j_password", (String) parameters.get("password")));
        localContext = HttpClientContext.create();
        localContext.setCookieStore(new BasicCookieStore());
        HttpPost httpPost = new HttpPost(url);
        try {
            httpPost.setEntity(new UrlEncodedFormEntity(nvps));
            CloseableHttpResponse httpResponse = httpclient.execute(httpPost, localContext);
            ResponseHandler<String> handler = new CustomResponseErrorHandler();
            String body = handler.handleResponse(httpResponse);
            response.put(BODY, body);
            httpResponse.close();
        } catch (Exception e) {
            authentificationUtils.getMap().clear();
            throw new ManagerResponseException(e.getMessage(), e);
        }

        return response;
    }

@Override
public AsyncClientHttpRequest createAsyncRequest(URI uri, HttpMethod httpMethod) throws IOException {
    HttpAsyncClient asyncClient = getHttpAsyncClient();
    startAsyncClient();
    HttpUriRequest httpRequest = createHttpUriRequest(httpMethod, uri);
    postProcessHttpRequest(httpRequest);
       HttpContext context = createHttpContext(httpMethod, uri);
       if (context == null) {
           context = HttpClientContext.create();
       }
       // Request configuration not set in the context
       if (context.getAttribute(HttpClientContext.REQUEST_CONFIG) == null) {
           // Use request configuration given by the user, when available
           RequestConfig config = null;
           if (httpRequest instanceof Configurable) {
               config = ((Configurable) httpRequest).getConfig();
           }
           if (config == null) {
               config = RequestConfig.DEFAULT;
           }
           context.setAttribute(HttpClientContext.REQUEST_CONFIG, config);
       }
    return new HttpComponentsAsyncClientHttpRequest(asyncClient, httpRequest, context);
}

public ResponseWrap(CloseableHttpClient httpClient, HttpRequestBase request, CloseableHttpResponse response, HttpClientContext context,
        ObjectMapper _mapper) {
    this.response = response;
    this.httpClient = httpClient;
    this.request = request;
    this.context = context;
    mapper = _mapper;

    try {
        HttpEntity entity = response.getEntity();
        if (entity != null) {
            this.entity = new BufferedHttpEntity(entity);
        } else {
            this.entity = new BasicHttpEntity();
        }

        EntityUtils.consumeQuietly(entity);
        this.response.close();
    } catch (IOException e) {
        logger.warn(e.getMessage());
    }
}

@Override
public Proxy determineProxy(HttpHost host, HttpRequest request, HttpContext context, IPPool ipPool,
        CrawlerSession crawlerSession) {
    HttpClientContext httpClientContext = HttpClientContext.adapt(context);

    Proxy proxy = (Proxy) crawlerSession.getExtInfo(VSCRAWLER_AVPROXY_KEY);
    if (proxy == null) {
        String accessUrl = null;
        if (request instanceof HttpRequestWrapper || request instanceof HttpGet) {
            accessUrl = HttpUriRequest.class.cast(request).getURI().toString();
        }
        if (!PoolUtil.isDungProxyEnabled(httpClientContext)) {
            log.info("{}不会被代理", accessUrl);
            return null;
        }
        proxy = ipPool.getIP(host.getHostName(), accessUrl);
        if (proxy == null) {
            return null;
        }
        crawlerSession.setExtInfo(VSCRAWLER_AVPROXY_KEY, proxy);
    }

    return proxy;
}

@Test
public void testPreemptiveAuthentication() throws Exception {
    final CountingAuthHandler requestHandler = new CountingAuthHandler();
    this.serverBootstrap.registerHandler("*", requestHandler);

    final HttpHost target = start();

    final HttpClientContext context = HttpClientContext.create();
    final AuthCache authCache = new BasicAuthCache();
    authCache.put(target, new BasicScheme());
    context.setAuthCache(authCache);
    final BasicCredentialsProvider credsProvider = new BasicCredentialsProvider();
    credsProvider.setCredentials(AuthScope.ANY,
            new UsernamePasswordCredentials("test", "test"));
    context.setCredentialsProvider(credsProvider);

    final HttpGet httpget = new HttpGet("/");
    final HttpResponse response1 = this.httpclient.execute(target, httpget, context);
    final HttpEntity entity1 = response1.getEntity();
    Assert.assertEquals(HttpStatus.SC_OK, response1.getStatusLine().getStatusCode());
    Assert.assertNotNull(entity1);
    EntityUtils.consume(entity1);

    Assert.assertEquals(1, requestHandler.getCount());
}

protected HttpContext createHttpContext(HttpProxy httpProxy, CookieStore cookieStore) {
    HttpContext httpContext = new HttpClientContext();

    if (cookieStore != null) {
        httpContext.setAttribute(HttpClientContext.COOKIE_STORE, cookieStore);
    }

    if (httpProxy != null && StringUtils.isNotBlank(httpProxy.getUsername())) {
        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(new AuthScope(httpProxy.getHost(), httpProxy.getPort()),
                new UsernamePasswordCredentials(httpProxy.getUsername(), httpProxy.getPassword()));
        httpContext.setAttribute(HttpClientContext.CREDS_PROVIDER, credentialsProvider);
    }

    return httpContext;
}

@Override
public boolean retryRequest(
    HttpResponse httpResponse, int executionCount, HttpContext httpContext) {
  int status = httpResponse.getStatusLine().getStatusCode();
  if (executionCount < 3 && status >= 500 && status < 600) {
    HttpClientContext context = HttpClientContext.adapt(httpContext);
    logger.info(
        "Retry " + executionCount + " for request for: "
            + context.getRequest().getRequestLine().getUri());
    return true;
  }
  return false;
}

private void retrieveAccessToken() throws IOException, WeiboClientException {
    String state = "__MY_STATE__";
    String authorizationCallback = "https://api.weibo.com/oauth2/authorize";
    String url = this.client.getAuthorizationUrl(ResponseType.Code, DisplayType.Default, state, authorizationCallback);
    //httpget
    CloseableHttpResponse response = null;
    HttpClientContext context = HttpClientContext.create();
    HttpGet httpGet = new HttpGet(url);
    response = httpClient.execute(httpGet, context);
    // 获取所有的重定向位置
    List<URI> redirectLocations = context.getRedirectLocations();
    //end
    System.out.println("Please visit: " + url);
    System.out.print("Input code: ");
    in = new BufferedReader(new InputStreamReader(System.in));
    String code = in.readLine();
    String accessTokenCallback = "https://api.weibo.com/oauth2/authorize";
    SinaWeibo2AccessToken accessToken = this.client.getAccessTokenByCode(code, accessTokenCallback);
    System.out.println();
    System.out.println("Access token: " + accessToken.getToken());
    System.out.println("Uid: " + accessToken.getUid());
    System.out.println("Expires in: " + accessToken.getExpiresIn());
    System.out.println("Remind in: " + accessToken.getRemindIn());
    accessToken = new SinaWeibo2AccessToken(accessToken.getToken());
    this.client.setAccessToken(accessToken);
}

@SuppressWarnings({ "unchecked", "rawtypes" })
public void testTooLargeResponse() throws Exception {
    ContentTooLongException tooLong = new ContentTooLongException("too long!");
    CloseableHttpAsyncClient httpClient = mock(CloseableHttpAsyncClient.class);
    when(httpClient.<HttpResponse>execute(any(HttpAsyncRequestProducer.class), any(HttpAsyncResponseConsumer.class),
            any(HttpClientContext.class), any(FutureCallback.class))).then(new Answer<Future<HttpResponse>>() {
        @Override
        public Future<HttpResponse> answer(InvocationOnMock invocationOnMock) throws Throwable {
            HeapBufferedAsyncResponseConsumer consumer = (HeapBufferedAsyncResponseConsumer) invocationOnMock.getArguments()[1];
            FutureCallback callback = (FutureCallback) invocationOnMock.getArguments()[3];
            assertEquals(new ByteSizeValue(100, ByteSizeUnit.MB).bytesAsInt(), consumer.getBufferLimit());
            callback.failed(tooLong);
            return null;
        }
    });
    RemoteScrollableHitSource source = sourceWithMockedClient(true, httpClient);

    AtomicBoolean called = new AtomicBoolean();
    Consumer<Response> checkResponse = r -> called.set(true);
    Throwable e = expectThrows(RuntimeException.class,
            () -> source.doStartNextScroll(FAKE_SCROLL_ID, timeValueMillis(0), checkResponse));
    // Unwrap the some artifacts from the test
    while (e.getMessage().equals("failed")) {
        e = e.getCause();
    }
    // This next exception is what the user sees
    assertEquals("Remote responded with a chunk that was too large. Use a smaller batch size.", e.getMessage());
    // And that exception is reported as being caused by the underlying exception returned by the client
    assertSame(tooLong, e.getCause());
    assertFalse(called.get());
}

/**
 * Verifies the content of the {@link HttpRequest} that's internally created and passed through to the http client
 */
@SuppressWarnings("unchecked")
public void testInternalHttpRequest() throws Exception {
    ArgumentCaptor<HttpAsyncRequestProducer> requestArgumentCaptor = ArgumentCaptor.forClass(HttpAsyncRequestProducer.class);
    int times = 0;
    for (String httpMethod : getHttpMethods()) {
        HttpUriRequest expectedRequest = performRandomRequest(httpMethod);
        verify(httpClient, times(++times)).<HttpResponse>execute(requestArgumentCaptor.capture(),
                any(HttpAsyncResponseConsumer.class), any(HttpClientContext.class), any(FutureCallback.class));
        HttpUriRequest actualRequest = (HttpUriRequest)requestArgumentCaptor.getValue().generateRequest();
        assertEquals(expectedRequest.getURI(), actualRequest.getURI());
        assertEquals(expectedRequest.getClass(), actualRequest.getClass());
        assertArrayEquals(expectedRequest.getAllHeaders(), actualRequest.getAllHeaders());
        if (expectedRequest instanceof HttpEntityEnclosingRequest) {
            HttpEntity expectedEntity = ((HttpEntityEnclosingRequest) expectedRequest).getEntity();
            if (expectedEntity != null) {
                HttpEntity actualEntity = ((HttpEntityEnclosingRequest) actualRequest).getEntity();
                assertEquals(EntityUtils.toString(expectedEntity), EntityUtils.toString(actualEntity));
            }
        }
    }
}

@Before
@SuppressWarnings("unchecked")
public void createRestClient() throws IOException {
    CloseableHttpAsyncClient httpClient = mock(CloseableHttpAsyncClient.class);
    when(httpClient.<HttpResponse>execute(any(HttpAsyncRequestProducer.class), any(HttpAsyncResponseConsumer.class),
           any(HttpClientContext.class), any(FutureCallback.class))).thenAnswer(new Answer<Future<HttpResponse>>() {
        @Override
        public Future<HttpResponse> answer(InvocationOnMock invocationOnMock) throws Throwable {
            HttpAsyncRequestProducer requestProducer = (HttpAsyncRequestProducer) invocationOnMock.getArguments()[0];
            HttpUriRequest request = (HttpUriRequest)requestProducer.generateRequest();
            HttpHost httpHost = requestProducer.getTarget();
            HttpClientContext context = (HttpClientContext) invocationOnMock.getArguments()[2];
            assertThat(context.getAuthCache().get(httpHost), instanceOf(BasicScheme.class));
            FutureCallback<HttpResponse> futureCallback = (FutureCallback<HttpResponse>) invocationOnMock.getArguments()[3];
            //return the desired status code or exception depending on the path
            if (request.getURI().getPath().equals("/soe")) {
                futureCallback.failed(new SocketTimeoutException(httpHost.toString()));
            } else if (request.getURI().getPath().equals("/coe")) {
                futureCallback.failed(new ConnectTimeoutException(httpHost.toString()));
            } else if (request.getURI().getPath().equals("/ioe")) {
                futureCallback.failed(new IOException(httpHost.toString()));
            } else {
                int statusCode = Integer.parseInt(request.getURI().getPath().substring(1));
                StatusLine statusLine = new BasicStatusLine(new ProtocolVersion("http", 1, 1), statusCode, "");
                futureCallback.completed(new BasicHttpResponse(statusLine));
            }
            return null;
        }
    });
    int numHosts = RandomNumbers.randomIntBetween(getRandom(), 2, 5);
    httpHosts = new HttpHost[numHosts];
    for (int i = 0; i < numHosts; i++) {
        httpHosts[i] = new HttpHost("localhost", 9200 + i);
    }
    failureListener = new HostsTrackingFailureListener();
    restClient = new RestClient(httpClient, 10000, new Header[0], httpHosts, null, failureListener);
}

private HttpClientContext prepareContext()
{
    HttpHost targetHost = new HttpHost(serverConfig.getServerName(), serverConfig.getPort(), serverConfig.isUseHTTPS() ? "https" : "http");
    AuthCache authCache = new BasicAuthCache();
    authCache.put(targetHost, new BasicScheme());

    CredentialsProvider credsProvider = new BasicCredentialsProvider();
    UsernamePasswordCredentials credentials
     = new UsernamePasswordCredentials(serverConfig.getUserName(), serverConfig.getPassword());
    credsProvider.setCredentials(AuthScope.ANY, credentials);

    // Add AuthCache to the execution context
    HttpClientContext context = HttpClientContext.create();
    context.setCredentialsProvider(credsProvider);
    context.setAuthCache(authCache);
    return context;
}

/**
 * Remove a Cookie by name and path
 *
 * @param name cookie name
 * @param path cookie path
 */
@PublicAtsApi
public void removeCookie( String name, String path ) {

    BasicCookieStore cookieStore = (BasicCookieStore) httpContext.getAttribute(HttpClientContext.COOKIE_STORE);
    if (cookieStore != null) {

        List<Cookie> cookies = cookieStore.getCookies();
        cookieStore.clear();
        for (Cookie cookie : cookies) {
            if (!cookie.getName().equals(name) || !cookie.getPath().equals(path)) {
                cookieStore.addCookie(cookie);
            }
        }
    }
}

private HttpContext httpContext() {
    BasicCredentialsProvider basicCredentialsProvider = new BasicCredentialsProvider();
    if (isNotBlank(this.username) && this.password != null) {
        UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(this.username, this.password);
        HttpHost httpHost = HttpHost.create(this.rootConfluenceUrl);
        AuthScope authScope = new AuthScope(httpHost);
        basicCredentialsProvider.setCredentials(authScope, credentials);

        BasicAuthCache basicAuthCache = new BasicAuthCache();
        basicAuthCache.put(httpHost, new BasicScheme());

        HttpClientContext httpClientContext = HttpClientContext.create();
        httpClientContext.setCredentialsProvider(basicCredentialsProvider);
        httpClientContext.setAuthCache(basicAuthCache);

        return httpClientContext;
    } else {
        return null;
    }
}

static SonarHttpRequester getSonarHttpRequester(GlobalConfigDataForSonarInstance globalConfigDataForSonarInstance) {

        try {
            HttpGet request = new HttpGet(getSonarApiServerVersion(globalConfigDataForSonarInstance));

            HttpClientContext context = HttpClientContext.create();
            CloseableHttpClient client = HttpClientBuilder.create().build();
            CloseableHttpResponse response = client.execute(request, context);
            String sonarVersion = EntityUtils.toString(response.getEntity());

            if (majorSonarVersion(sonarVersion) <= 5) {
                return new SonarHttpRequester5x();
            } else if (majorSonarVersion(sonarVersion) >= 6 && minorSonarVersion(sonarVersion) == 0) {
                return new SonarHttpRequester60();
            } else if (majorSonarVersion(sonarVersion) >= 6 && minorSonarVersion(sonarVersion) >= 1) {
                return new SonarHttpRequester61();
            } else {
                throw new UnsuportedVersionException("Plugin doesn't suport this version of sonar api! Please contact the developer.");
            }
        } catch (IOException e) {
            throw new ApiConnectionException(e.getLocalizedMessage());
        }
    }

private void loginApi(GlobalConfigDataForSonarInstance globalConfigDataForSonarInstance) {

        context = HttpClientContext.create();
        client = HttpClientBuilder.create().build();

        if (StringUtils.isNotEmpty(globalConfigDataForSonarInstance.getToken())) {
            token = globalConfigDataForSonarInstance.getToken();
        } else {
            HttpPost loginHttpPost = new HttpPost(globalConfigDataForSonarInstance.getSonarUrl() + getSonarApiLogin());

            List<NameValuePair> nvps = new ArrayList<>();
            nvps.add(new BasicNameValuePair("login", globalConfigDataForSonarInstance.getUsername()));
            nvps.add(new BasicNameValuePair("password", globalConfigDataForSonarInstance.getPass()));
            nvps.add(new BasicNameValuePair("remember_me", "1"));
            loginHttpPost.setEntity(createEntity(nvps));
            loginHttpPost.addHeader(HttpHeaders.CONTENT_TYPE, "application/x-www-form-urlencoded");

            executePostRequest(client, loginHttpPost);
        }

        logged = true;
    }

@Override
  public boolean retryRequest(
          IOException exception,
          int executionCount,
          HttpContext context) {
if (executionCount > retryExecutionCount) {
          return false;
      }
      if (exception instanceof InterruptedIOException) {
          return false;
      }
      if (exception instanceof UnknownHostException) {
          return false;
      }
      if (exception instanceof ConnectTimeoutException) {
          return true;
      }
      if (exception instanceof SSLException) {
          return false;
      }
      HttpClientContext clientContext = HttpClientContext.adapt(context);
      HttpRequest request = clientContext.getRequest();
      boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
      if (idempotent) {
          // Retry if the request is considered idempotent
          return true;
      }
      return false;
  }

public static <T> T execute(HttpUriRequest request,ResponseHandler<T> responseHandler){
    String uriId = loggerRequest(request);
    if(responseHandler instanceof LocalResponseHandler){
        LocalResponseHandler lrh = (LocalResponseHandler) responseHandler;
        lrh.setUriId(uriId);
    }
    try {
        T t = httpClient.execute(request, responseHandler,HttpClientContext.create());
        if(resultErrorHandler != null){
            resultErrorHandler.doHandle(uriId, request, t);
        }
        return t;
    } catch (Exception e) {
        e.printStackTrace();
        logger.error(e.getMessage());
    }
    return null;
}

/**
 * 
 * @param mch_id mch_id
 * @param request request
 * @param clazz clazz
 * @param sign_type 数据返回验证签名类型
 * @param key 数据返回验证签名key
 * @since 2.8.5
 * @return result
 */
public static <T> T keyStoreExecuteXmlResult(String mch_id,HttpUriRequest request,Class<T> clazz,String sign_type,String key){
    String uriId = loggerRequest(request);
    ResponseHandler<T> responseHandler = XmlResponseHandler.createResponseHandler(clazz,sign_type,key);
    if(responseHandler instanceof LocalResponseHandler){
        LocalResponseHandler lrh = (LocalResponseHandler) responseHandler;
        lrh.setUriId(uriId);
    }
    try {
        T t = httpClient_mchKeyStore.get(mch_id).execute(request,responseHandler,HttpClientContext.create());
        if(resultErrorHandler != null){
            resultErrorHandler.doHandle(uriId, request, t);
        }
        return t;
    } catch (Exception e) {
        e.printStackTrace();
        logger.error(e.getMessage());
    }
    return null;
}

@Override
public boolean retryRequest(IOException exception, int executionCount, HttpContext context){
    if(logOnRetry){
        HttpClientContext clientContext = HttpClientContext.adapt(context);
        logger.warn("Request {} failure Nº {}", clientContext.getRequest().getRequestLine(), executionCount,
                exception);
    }
    Object retrySafe = context.getAttribute(RETRY_SAFE_ATTRIBUTE);
    if(retrySafe == null || !(retrySafe instanceof Boolean) || !(Boolean)retrySafe || executionCount > retryCount){
        return false;
    }
    return true;
}

private HttpClientContext createHttpClientContextFromConfig() {
    // create context from config
    HttpClientContext context = HttpClientContext.create();

    if (config.getCookieStore() != null) {
        context.setCookieStore(config.getCookieStore());
    }

    if (config.getCredsProvider() != null) {
        context.setCredentialsProvider(config.getCredsProvider());
    }

    if (config.getAuthCache() != null) {
        context.setAuthCache(config.getAuthCache());
    }

    return context;
}

public void process(HttpRequest httpRequest, HttpContext httpContext) throws HttpException, IOException {
    final HttpClientContext clientContext = HttpClientContext.adapt(httpContext);
    List<Cookie> cookies = clientContext.getCookieStore().getCookies();
    boolean set = (null != StickyCookieHolder.getTestStickySessionCookie());
    boolean found = false;
    ListIterator<Cookie> it = cookies.listIterator();
    while (it.hasNext()) {
        Cookie cookie = it.next();
        if (cookie.getName().equals(StickyCookieHolder.COOKIE_NAME)) {
            found = true;
            if (set) {
                // set the cookie with the value saved for each thread using the rule
                it.set(StickyCookieHolder.getTestStickySessionCookie());
            } else {
                // if the cookie is not set in TestStickySessionRule, remove it from here
                it.remove();
            }
        }
    }
    // if the cookie needs to be set from TestStickySessionRule but did not exist in the client cookie list, add it here.
    if (!found && set) {
        cookies.add(StickyCookieHolder.getTestStickySessionCookie());
    }
    BasicCookieStore cs = new BasicCookieStore();
    cs.addCookies(cookies.toArray(new Cookie[cookies.size()]));
    httpContext.setAttribute(HttpClientContext.COOKIE_STORE, cs);
}

@Override
public ClientHttpRequest createRequest(URI uri, HttpMethod httpMethod) throws IOException {
    CloseableHttpClient client = (CloseableHttpClient) getHttpClient();
    Assert.state(client != null, "Synchronous execution requires an HttpClient to be set");
    HttpUriRequest httpRequest = createHttpUriRequest(httpMethod, uri);
    postProcessHttpRequest(httpRequest);
       HttpContext context = createHttpContext(httpMethod, uri);
       if (context == null) {
           context = HttpClientContext.create();
       }
       // Request configuration not set in the context
       if (context.getAttribute(HttpClientContext.REQUEST_CONFIG) == null) {
           // Use request configuration given by the user, when available
           RequestConfig config = null;
           if (httpRequest instanceof Configurable) {
               config = ((Configurable) httpRequest).getConfig();
           }
           if (config == null) {
               if (this.socketTimeout > 0 || this.connectTimeout > 0) {
                   config = RequestConfig.custom()
                           .setConnectTimeout(this.connectTimeout)
                           .setSocketTimeout(this.socketTimeout)
                           .build();
               }
            else {
                   config = RequestConfig.DEFAULT;
               }
           }
           context.setAttribute(HttpClientContext.REQUEST_CONFIG, config);
       }
    if (this.bufferRequestBody) {
        return new HttpComponentsClientHttpRequest(client, httpRequest, context);
    }
    else {
        return new HttpComponentsStreamingClientHttpRequest(client, httpRequest, context);
    }
}

@Override
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {

    if (executionCount >= 3) {// 如果已经重试了3次，就放弃
        return false;
    }

    if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接，那么就重试
        return true;
    }

    if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
        return false;
    }

    if (exception instanceof InterruptedIOException) {// 超时
        return true;
    }

    if (exception instanceof UnknownHostException) {// 目标服务器不可达
        return false;
    }

    if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
        return false;
    }

    if (exception instanceof SSLException) {// ssl握手异常
        return false;
    }

    HttpClientContext clientContext = HttpClientContext.adapt(context);
    HttpRequest request = clientContext.getRequest();

    // 如果请求是幂等的，就再次尝试
    if (!(request instanceof HttpEntityEnclosingRequest)) {
        return true;
    }
    return false;
}

@Override
public void authSucceeded(final HttpHost authhost, final AuthScheme authScheme, final HttpContext context) {
    final HttpClientContext clientContext = HttpClientContext.adapt(context);
    final Credentials credentials = clientContext.getAttribute(PROXY_CREDENTIALS_INPUT_ID, Credentials.class);
    if(null != credentials) {
        clientContext.removeAttribute(PROXY_CREDENTIALS_INPUT_ID);
        if(log.isInfoEnabled()) {
            log.info(String.format("Save passphrase for proxy %s", authhost));
        }
        keychain.addCredentials(authhost.getHostName(), credentials.getUsername(), credentials.getPassword());
    }
    super.authSucceeded(authhost, authScheme, context);
}

@Override
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {

    if (executionCount >= 3) {// 如果已经重试了3次，就放弃
        return false;
    }

    if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接，那么就重试
        return true;
    }

    if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
        return false;
    }

    if (exception instanceof InterruptedIOException) {// 超时
        return true;
    }

    if (exception instanceof UnknownHostException) {// 目标服务器不可达
        return false;
    }

    if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
        return false;
    }

    if (exception instanceof SSLException) {// ssl握手异常
        return false;
    }

    HttpClientContext clientContext = HttpClientContext.adapt(context);
    HttpRequest request = clientContext.getRequest();

    // 如果请求是幂等的，就再次尝试
    if (!(request instanceof HttpEntityEnclosingRequest)) {
        return true;
    }
    return false;
}

/**
 * 执行请求
 * 
 * @date 2015年7月17日
 * @return
 */
public ResponseWrap execute() {
    settingRequest();
    if (httpClient == null) {
        httpClient = clientBuilder.build();
    }

    try {
        HttpClientContext context = HttpClientContext.create();
        CloseableHttpResponse response = httpClient.execute(request, context);
        return new ResponseWrap(httpClient, request, response, context, mapper);
    } catch (IOException e) {
        logger.error(e.getMessage(), e);
        throw new RuntimeException(e.getMessage(), e);
    }
}

@Override
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {

    if (executionCount >= 3) {// 如果已经重试了3次，就放弃
        return false;
    }

    if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接，那么就重试
        return true;
    }

    if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
        return false;
    }

    if (exception instanceof InterruptedIOException) {// 超时
        return true;
    }

    if (exception instanceof UnknownHostException) {// 目标服务器不可达
        return false;
    }

    if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
        return false;
    }

    if (exception instanceof SSLException) {// ssl握手异常
        return false;
    }

    HttpClientContext clientContext = HttpClientContext.adapt(context);
    HttpRequest request = clientContext.getRequest();

    // 如果请求是幂等的，就再次尝试
    if (!(request instanceof HttpEntityEnclosingRequest)) {
        return true;
    }
    return false;
}

private static void addPreemptiveAuthenticationProxy(HttpClientContext clientContext,
                                                     ProxyConfiguration proxyConfiguration) {

    if (proxyConfiguration.preemptiveBasicAuthenticationEnabled()) {
        HttpHost targetHost = new HttpHost(proxyConfiguration.endpoint().getHost(), proxyConfiguration.endpoint().getPort());
        final CredentialsProvider credsProvider = newProxyCredentialsProvider(proxyConfiguration);
        // Create AuthCache instance
        AuthCache authCache = new BasicAuthCache();
        // Generate BASIC scheme object and add it to the local auth cache
        BasicScheme basicAuth = new BasicScheme();
        authCache.put(targetHost, basicAuth);

        clientContext.setCredentialsProvider(credsProvider);
        clientContext.setAuthCache(authCache);
    }
}

/**
 * Returns a new HttpClientContext used for request execution.
 */
public static HttpClientContext newClientContext(HttpClientSettings settings,
                                                 Map<String, ? extends Object>
                                                         attributes) {
    final HttpClientContext clientContext = new HttpClientContext();

    if (attributes != null && !attributes.isEmpty()) {
        for (Map.Entry<String, ?> entry : attributes.entrySet()) {
            clientContext.setAttribute(entry.getKey(), entry.getValue());
        }
    }

    addPreemptiveAuthenticationProxy(clientContext, settings);
    return clientContext;

}

private static void addPreemptiveAuthenticationProxy(HttpClientContext clientContext,
                                                     HttpClientSettings settings) {

    if (settings.isPreemptiveBasicProxyAuth()) {
        HttpHost targetHost = new HttpHost(settings.getProxyHost(), settings
                .getProxyPort());
        final CredentialsProvider credsProvider = newProxyCredentialsProvider(settings);
        // Create AuthCache instance
        AuthCache authCache = new BasicAuthCache();
        // Generate BASIC scheme object and add it to the local auth cache
        BasicScheme basicAuth = new BasicScheme();
        authCache.put(targetHost, basicAuth);

        clientContext.setCredentialsProvider(credsProvider);
        clientContext.setAuthCache(authCache);
    }
}

@Override
protected HttpHost determineProxy(HttpHost host, HttpRequest request, HttpContext context) throws HttpException {
    HttpClientContext httpClientContext = HttpClientContext.adapt(context);
    Proxy proxy = proxyPlanner.determineProxy(host, request, context, ipPool, crawlerSession);

    if (proxy == null) {
        return null;
    }
    if (log.isDebugEnabled()) {
        log.debug("{} 当前使用IP为:{}:{}", host.getHostName(), proxy.getIp(), proxy.getPort());
    }
    context.setAttribute(VSCRAWLER_AVPROXY_KEY, proxy);

    if (proxy.getAuthenticationHeaders() != null) {
        for (Header header : proxy.getAuthenticationHeaders()) {
            request.addHeader(header);
        }
    }

    if (StringUtils.isNotEmpty(proxy.getUsername()) && StringUtils.isNotEmpty(proxy.getPassword())) {
        BasicCredentialsProvider credsProvider1 = new BasicCredentialsProvider();
        httpClientContext.setCredentialsProvider(credsProvider1);
        credsProvider1.setCredentials(AuthScope.ANY,
                new UsernamePasswordCredentials(proxy.getUsername(), proxy.getPassword()));
    }
    return new HttpHost(proxy.getIp(), proxy.getPort());
}

@Override
public CloseableHttpResponse execute(HttpRoute route, HttpRequestWrapper request, HttpClientContext clientContext,
        HttpExecutionAware execAware) throws IOException, HttpException {
    Proxy proxy = (Proxy) clientContext.getAttribute(VSCrawlerConstant.VSCRAWLER_AVPROXY_KEY);
    if (proxy != null) {
        proxy.recordUsage();
    }
    try {
        return delegate.execute(route, request, clientContext, execAware);
    } catch (IOException ioe) {
        if (proxy != null) {
            proxy.recordFailed();
        }
        throw ioe;
    }
}

/**
 * Open the entry page.
 *
 * @param retryTimes retry times of qr scan
 */
void open(int retryTimes) {
    final String url = WECHAT_URL_ENTRY;
    HttpHeaders customHeader = new HttpHeaders();
    customHeader.setPragma("no-cache");
    customHeader.setCacheControl("no-cache");
    customHeader.set("Upgrade-Insecure-Requests", "1");
    customHeader.set(HttpHeaders.ACCEPT, "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
    HeaderUtils.assign(customHeader, getHeader);
    restTemplate.exchange(url, HttpMethod.GET, new HttpEntity<>(customHeader), String.class);
    //manually insert two cookies into cookiestore, as they're supposed to be stored in browsers by javascript.
    CookieStore store = (CookieStore) ((StatefullRestTemplate) restTemplate).getHttpContext().getAttribute(HttpClientContext.COOKIE_STORE);
    Date maxDate = new Date(Long.MAX_VALUE);
    String domain = WECHAT_URL_ENTRY.replaceAll("https://", "").replaceAll("/", "");
    Map<String, String> cookies = new HashMap<>(3);
    cookies.put("MM_WX_NOTIFY_STATE", "1");
    cookies.put("MM_WX_SOUND_STATE", "1");
    if (retryTimes > 0) {
        cookies.put("refreshTimes", String.valueOf(retryTimes));
    }
    appendAdditionalCookies(store, cookies, domain, "/", maxDate);
    //It's now at entry page.
    this.originValue = WECHAT_URL_ENTRY;
    this.refererValue = WECHAT_URL_ENTRY.replaceAll("/$", "");
}

/**
 * Initialization
 *
 * @param hostUrl     hostUrl
 * @param baseRequest baseRequest
 * @return current user's information and contact information
 * @throws IOException if the http response body can't be convert to {@link InitResponse}
 */
InitResponse init(String hostUrl, BaseRequest baseRequest) throws IOException {
    String url = String.format(WECHAT_URL_INIT, hostUrl, RandomUtils.generateDateWithBitwiseNot());

    CookieStore store = (CookieStore) ((StatefullRestTemplate) restTemplate).getHttpContext().getAttribute(HttpClientContext.COOKIE_STORE);
    Date maxDate = new Date(Long.MAX_VALUE);
    String domain = hostUrl.replaceAll("https://", "").replaceAll("/", "");
    Map<String, String> cookies = new HashMap<>(3);
    cookies.put("MM_WX_NOTIFY_STATE", "1");
    cookies.put("MM_WX_SOUND_STATE", "1");
    appendAdditionalCookies(store, cookies, domain, "/", maxDate);
    InitRequest request = new InitRequest();
    request.setBaseRequest(baseRequest);
    HttpHeaders customHeader = new HttpHeaders();
    customHeader.set(HttpHeaders.REFERER, hostUrl + "/");
    customHeader.setOrigin(hostUrl);
    HeaderUtils.assign(customHeader, postHeader);
    ResponseEntity<String> responseEntity
            = restTemplate.exchange(url, HttpMethod.POST, new HttpEntity<>(request, customHeader), String.class);
    return jsonMapper.readValue(WechatUtils.textDecode(responseEntity.getBody()), InitResponse.class);
}

/**
 * Reads content.
 * @param httpClient HTTP client
 * @param since since date
 * @return content reference
 * @throws IOException if reading content fails
 * @throws URISyntaxException if file url is an invalid URI
 */
public SimpleDataReference readContent(CloseableHttpClient httpClient, Date since) throws IOException, URISyntaxException {
  HttpGet method = new HttpGet(fileUrl.toExternalForm());
  method.setConfig(DEFAULT_REQUEST_CONFIG);
  method.setHeader("User-Agent", HttpConstants.getUserAgent());
  HttpClientContext context = creds!=null && !creds.isEmpty()? createHttpClientContext(fileUrl, creds): null;

  try (CloseableHttpResponse httpResponse = httpClient.execute(method,context); InputStream input = httpResponse.getEntity().getContent();) {
    if (httpResponse.getStatusLine().getStatusCode()>=400) {
      throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase());
    }
    Date lastModifiedDate = readLastModifiedDate(httpResponse);
    MimeType contentType = readContentType(httpResponse);
    boolean readBody = since==null || lastModifiedDate==null || lastModifiedDate.getTime()>=since.getTime();
    SimpleDataReference ref = new SimpleDataReference(broker.getBrokerUri(), broker.getEntityDefinition().getLabel(), fileUrl.toExternalForm(), lastModifiedDate, fileUrl.toURI());
    ref.addContext(contentType, readBody? IOUtils.toByteArray(input): null);
    return ref;
  }
}

@Test
public void testMustNotAddMultipartByteRangeContentTypeTo416Response() throws Exception {
    originResponse = Proxies.enhanceResponse(
            new BasicHttpResponse(HttpVersion.HTTP_1_1, 416, "Requested Range Not Satisfiable"));

    EasyMock.expect(
            mockBackend.execute(
                    EasyMock.isA(HttpRoute.class),
                    EasyMock.isA(HttpRequestWrapper.class),
                    EasyMock.isA(HttpClientContext.class),
                    EasyMock.<HttpExecutionAware>isNull())).andReturn(originResponse);

    replayMocks();
    final HttpResponse result = impl.execute(route, request, context, null);
    verifyMocks();

    if (result.getStatusLine().getStatusCode() == 416) {
        for (final Header h : result.getHeaders("Content-Type")) {
            for (final HeaderElement elt : h.getElements()) {
                Assert.assertFalse("multipart/byteranges".equalsIgnoreCase(elt.getName()));
            }
        }
    }
}

@Test
public void testSelectNoCredentialsProvider() throws Exception {
    final TargetAuthenticationStrategy authStrategy = new TargetAuthenticationStrategy();
    final HttpResponse response = new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_UNAUTHORIZED, "UNAUTHORIZED");
    final HttpHost authhost = new HttpHost("locahost", 80);
    final HttpClientContext context = HttpClientContext.create();

    final Map<String, Header> challenges = new HashMap<String, Header>();
    challenges.put("basic", new BasicHeader(AUTH.WWW_AUTH, "Basic realm=\"test\""));
    challenges.put("digest", new BasicHeader(AUTH.WWW_AUTH, "Digest realm=\"realm1\", nonce=\"1234\""));

    final Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
        .register("basic", new BasicSchemeFactory())
        .register("digest", new DigestSchemeFactory()).build();
    context.setAuthSchemeRegistry(authSchemeRegistry);

    final Queue<AuthOption> options = authStrategy.select(challenges, authhost, response, context);
    Assert.assertNotNull(options);
    Assert.assertEquals(0, options.size());
}