/** * Tests multipart POST and verifies it via GET operations. * @throws Exception */ @Test public void multipartPostGetHeadTest() throws Exception { Account refAccount = ACCOUNT_SERVICE.createAndAddRandomAccount(); Container refContainer = refAccount.getContainerById(Container.DEFAULT_PUBLIC_CONTAINER_ID); doPostGetHeadDeleteTest(0, refAccount, refContainer, refAccount.getName(), !refContainer.isCacheable(), refAccount.getName(), refContainer.getName(), true); doPostGetHeadDeleteTest(FRONTEND_CONFIG.frontendChunkedGetResponseThresholdInBytes * 3, refAccount, refContainer, refAccount.getName(), !refContainer.isCacheable(), refAccount.getName(), refContainer.getName(), true); // failure case // size of content being POSTed is higher than what is allowed via multipart/form-data long maxAllowedSizeBytes = new NettyConfig(FRONTEND_VERIFIABLE_PROPS).nettyMultipartPostMaxSizeBytes; ByteBuffer content = ByteBuffer.wrap(TestUtils.getRandomBytes((int) maxAllowedSizeBytes + 1)); HttpHeaders headers = new DefaultHttpHeaders(); setAmbryHeadersForPut(headers, 7200, !refContainer.isCacheable(), refAccount.getName(), "application/octet-stream", null, refAccount.getName(), refContainer.getName()); HttpRequest httpRequest = RestTestUtils.createRequest(HttpMethod.POST, "/", headers); HttpPostRequestEncoder encoder = createEncoder(httpRequest, content, ByteBuffer.allocate(0)); ResponseParts responseParts = nettyClient.sendRequest(encoder.finalizeRequest(), encoder, null).get(); HttpResponse response = getHttpResponse(responseParts); assertEquals("Unexpected response status", HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, response.status()); assertTrue("No Date header", response.headers().getTimeMillis(HttpHeaderNames.DATE, -1) != -1); assertFalse("Channel should not be active", HttpUtil.isKeepAlive(response)); }
/** * Creates a {@link HttpPostRequestEncoder} that encodes the given {@code request} and {@code blobContent}. * @param request the {@link HttpRequest} containing headers and other metadata about the request. * @param blobContent the {@link ByteBuffer} that represents the content of the blob. * @param usermetadata the {@link ByteBuffer} that represents user metadata * @return a {@link HttpPostRequestEncoder} that can encode the {@code request} and {@code blobContent}. * @throws HttpPostRequestEncoder.ErrorDataEncoderException * @throws IOException */ private HttpPostRequestEncoder createEncoder(HttpRequest request, ByteBuffer blobContent, ByteBuffer usermetadata) throws HttpPostRequestEncoder.ErrorDataEncoderException, IOException { HttpDataFactory httpDataFactory = new DefaultHttpDataFactory(false); HttpPostRequestEncoder encoder = new HttpPostRequestEncoder(httpDataFactory, request, true); FileUpload fileUpload = new MemoryFileUpload(RestUtils.MultipartPost.BLOB_PART, RestUtils.MultipartPost.BLOB_PART, "application/octet-stream", "", Charset.forName("UTF-8"), blobContent.remaining()); fileUpload.setContent(Unpooled.wrappedBuffer(blobContent)); encoder.addBodyHttpData(fileUpload); fileUpload = new MemoryFileUpload(RestUtils.MultipartPost.USER_METADATA_PART, RestUtils.MultipartPost.USER_METADATA_PART, "application/octet-stream", "", Charset.forName("UTF-8"), usermetadata.remaining()); fileUpload.setContent(Unpooled.wrappedBuffer(usermetadata)); encoder.addBodyHttpData(fileUpload); return encoder; }
/** * Creates a {@link NettyMultipartRequest} with the given {@code headers} and {@code parts}. * @param headers the {@link HttpHeaders} that need to be added to the request. * @param parts the files that will form the parts of the request. * @return a {@link NettyMultipartRequest} containing all the {@code headers} and {@code parts}. * @throws Exception */ private NettyMultipartRequest createRequest(HttpHeaders headers, InMemoryFile[] parts) throws Exception { HttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/"); if (headers != null) { httpRequest.headers().set(headers); } HttpPostRequestEncoder encoder = createEncoder(httpRequest, parts); NettyMultipartRequest request = new NettyMultipartRequest(encoder.finalizeRequest(), new MockChannel(), NETTY_METRICS, Collections.emptySet(), Long.MAX_VALUE); assertTrue("Request channel is not open", request.isOpen()); while (!encoder.isEndOfInput()) { // Sending null for ctx because the encoder is OK with that. request.addContent(encoder.readChunk(PooledByteBufAllocator.DEFAULT)); } return request; }
/** * Creates a {@link HttpPostRequestEncoder} that encodes the given {@code request} and {@code parts}. * @param request the {@link HttpRequest} containing headers and other metadata about the request. * @param parts the {@link InMemoryFile}s that will form the parts of the request. * @return a {@link HttpPostRequestEncoder} that can encode the {@code request} and {@code parts}. * @throws HttpPostRequestEncoder.ErrorDataEncoderException * @throws IOException */ private HttpPostRequestEncoder createEncoder(HttpRequest request, InMemoryFile[] parts) throws HttpPostRequestEncoder.ErrorDataEncoderException, IOException { HttpDataFactory httpDataFactory = new DefaultHttpDataFactory(false); HttpPostRequestEncoder encoder = new HttpPostRequestEncoder(httpDataFactory, request, true); if (parts != null) { for (InMemoryFile part : parts) { FileUpload fileUpload = new MemoryFileUpload(part.name, part.name, "application/octet-stream", "", Charset.forName("UTF-8"), part.content.remaining()); fileUpload.setContent(Unpooled.wrappedBuffer(part.content)); encoder.addBodyHttpData(fileUpload); } } return encoder; }
/** * Tests the case where multipart upload is used. * @throws Exception */ @Test public void multipartPostTest() throws Exception { Random random = new Random(); ByteBuffer content = ByteBuffer.wrap(TestUtils.getRandomBytes(random.nextInt(128) + 128)); HttpRequest httpRequest = RestTestUtils.createRequest(HttpMethod.POST, "/", null); httpRequest.headers().set(RestUtils.Headers.SERVICE_ID, "rawBytesPostTest"); HttpPostRequestEncoder encoder = createEncoder(httpRequest, content); HttpRequest postRequest = encoder.finalizeRequest(); List<ByteBuffer> contents = new ArrayList<ByteBuffer>(); while (!encoder.isEndOfInput()) { // Sending null for ctx because the encoder is OK with that. contents.add(encoder.readChunk(PooledByteBufAllocator.DEFAULT).content().nioBuffer()); } ByteBuffer receivedContent = doPostTest(postRequest, contents); compareContent(receivedContent, Collections.singletonList(content)); }
/** * Gets the encoded size for a set of bytes * @param bytes the bytes to encode. * @return the encoded size * @throws Exception */ private int getEncodedSize(byte[] bytes) throws Exception { int encodedSize = 0; InMemoryFile[] files = {new InMemoryFile(RestUtils.MultipartPost.BLOB_PART, ByteBuffer.wrap(bytes))}; HttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/"); HttpPostRequestEncoder encoder = createEncoder(httpRequest, files); encoder.finalizeRequest(); while (!encoder.isEndOfInput()) { HttpContent httpContent = encoder.readChunk(PooledByteBufAllocator.DEFAULT); encodedSize += httpContent.content().readableBytes(); } return encodedSize; }
/** * Creates a {@link HttpPostRequestEncoder} that encodes the given {@code request} and {@code blobContent}. * @param request the {@link HttpRequest} containing headers and other metadata about the request. * @param blobContent the {@link ByteBuffer} that represents the content of the blob. * @return a {@link HttpPostRequestEncoder} that can encode the {@code request} and {@code blobContent}. * @throws HttpPostRequestEncoder.ErrorDataEncoderException * @throws IOException */ private HttpPostRequestEncoder createEncoder(HttpRequest request, ByteBuffer blobContent) throws HttpPostRequestEncoder.ErrorDataEncoderException, IOException { HttpDataFactory httpDataFactory = new DefaultHttpDataFactory(false); HttpPostRequestEncoder encoder = new HttpPostRequestEncoder(httpDataFactory, request, true); FileUpload fileUpload = new MemoryFileUpload(RestUtils.MultipartPost.BLOB_PART, RestUtils.MultipartPost.BLOB_PART, "application/octet-stream", "", Charset.forName("UTF-8"), blobContent.remaining()); fileUpload.setContent(Unpooled.wrappedBuffer(blobContent)); encoder.addBodyHttpData(fileUpload); return encoder; }
void _subscribe(CoreSubscriber<? super Long> s) { if (!parent.markSentHeaders()) { Operators.error(s, new IllegalStateException("headers have already " + "been sent")); return; } HttpDataFactory df = DEFAULT_FACTORY; try { HttpClientFormEncoder encoder = new HttpClientFormEncoder(df, parent.nettyRequest, false, HttpConstants.DEFAULT_CHARSET, HttpPostRequestEncoder.EncoderMode.RFC1738); formCallback.accept(encoder); encoder = encoder.applyChanges(parent.nettyRequest); df = encoder.newFactory; if (!encoder.isMultipart()) { parent.chunkedTransfer(false); } parent.addHandlerFirst(NettyPipeline.ChunkedWriter, new ChunkedWriteHandler()); boolean chunked = HttpUtil.isTransferEncodingChunked(parent.nettyRequest); HttpRequest r = encoder.finalizeRequest(); if (!chunked) { HttpUtil.setTransferEncodingChunked(r, false); HttpUtil.setContentLength(r, encoder.length()); } ChannelFuture f = parent.channel() .writeAndFlush(r); Flux<Long> tail = encoder.progressFlux.onBackpressureLatest(); if (encoder.cleanOnTerminate) { tail = tail.doOnCancel(encoder) .doAfterTerminate(encoder); } if (encoder.isChunked()) { tail.subscribe(s); parent.channel() .writeAndFlush(encoder); } else { FutureMono.from(f) .cast(Long.class) .switchIfEmpty(Mono.just(encoder.length())) .flux() .subscribe(s); } } catch (Throwable e) { Exceptions.throwIfFatal(e); df.cleanRequestHttpData(parent.nettyRequest); Operators.error(s, Exceptions.unwrap(e)); } }
/** * Standard post without multipart but already support on Factory (memory management) * * @return the list of HttpData object (attribute and file) to be reused on next post */ private static List<InterfaceHttpData> formpost( Bootstrap bootstrap, String host, int port, URI uriSimple, File file, HttpDataFactory factory, List<Entry<String, String>> headers) throws Exception { // XXX /formpost // Start the connection attempt. ChannelFuture future = bootstrap.connect(new InetSocketAddress(host, port)); // Wait until the connection attempt succeeds or fails. Channel channel = future.sync().channel(); // Prepare the HTTP request. HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uriSimple.toASCIIString()); // Use the PostBody encoder HttpPostRequestEncoder bodyRequestEncoder = new HttpPostRequestEncoder(factory, request, false); // false => not multipart // it is legal to add directly header or cookie into the request until finalize for (Entry<String, String> entry : headers) { request.headers().set(entry.getKey(), entry.getValue()); } // add Form attribute bodyRequestEncoder.addBodyAttribute("getform", "POST"); bodyRequestEncoder.addBodyAttribute("info", "first value"); bodyRequestEncoder.addBodyAttribute("secondinfo", "secondvalue ���&"); bodyRequestEncoder.addBodyAttribute("thirdinfo", textArea); bodyRequestEncoder.addBodyAttribute("fourthinfo", textAreaLong); bodyRequestEncoder.addBodyFileUpload("myfile", file, "application/x-zip-compressed", false); // finalize request request = bodyRequestEncoder.finalizeRequest(); // Create the bodylist to be reused on the last version with Multipart support List<InterfaceHttpData> bodylist = bodyRequestEncoder.getBodyListAttributes(); // send request channel.write(request); // test if request was chunked and if so, finish the write if (bodyRequestEncoder.isChunked()) { // could do either request.isChunked() // either do it through ChunkedWriteHandler channel.write(bodyRequestEncoder); } channel.flush(); // Do not clear here since we will reuse the InterfaceHttpData on the next request // for the example (limit action on client side). Take this as a broadcast of the same // request on both Post actions. // // On standard program, it is clearly recommended to clean all files after each request // bodyRequestEncoder.cleanFiles(); // Wait for the server to close the connection. channel.closeFuture().sync(); return bodylist; }
/** * Multipart example */ private static void formpostmultipart( Bootstrap bootstrap, String host, int port, URI uriFile, HttpDataFactory factory, Iterable<Entry<String, String>> headers, List<InterfaceHttpData> bodylist) throws Exception { // XXX /formpostmultipart // Start the connection attempt. ChannelFuture future = bootstrap.connect(new InetSocketAddress(host, port)); // Wait until the connection attempt succeeds or fails. Channel channel = future.sync().channel(); // Prepare the HTTP request. HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uriFile.toASCIIString()); // Use the PostBody encoder HttpPostRequestEncoder bodyRequestEncoder = new HttpPostRequestEncoder(factory, request, true); // true => multipart // it is legal to add directly header or cookie into the request until finalize for (Entry<String, String> entry : headers) { request.headers().set(entry.getKey(), entry.getValue()); } // add Form attribute from previous request in formpost() bodyRequestEncoder.setBodyHttpDatas(bodylist); // finalize request bodyRequestEncoder.finalizeRequest(); // send request channel.write(request); // test if request was chunked and if so, finish the write if (bodyRequestEncoder.isChunked()) { channel.write(bodyRequestEncoder); } channel.flush(); // Now no more use of file representation (and list of HttpData) bodyRequestEncoder.cleanFiles(); // Wait for the server to close the connection. channel.closeFuture().sync(); }
/** * Multipart example */ private static void formpostmultipart( Bootstrap bootstrap, String host, int port, URI uriFile, HttpDataFactory factory, List<Entry<String, String>> headers, List<InterfaceHttpData> bodylist) throws Exception { // XXX /formpostmultipart // Start the connection attempt. ChannelFuture future = bootstrap.connect(new InetSocketAddress(host, port)); // Wait until the connection attempt succeeds or fails. Channel channel = future.sync().channel(); // Prepare the HTTP request. HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uriFile.toASCIIString()); // Use the PostBody encoder HttpPostRequestEncoder bodyRequestEncoder = new HttpPostRequestEncoder(factory, request, true); // true => multipart // it is legal to add directly header or cookie into the request until finalize for (Entry<String, String> entry : headers) { request.headers().set(entry.getKey(), entry.getValue()); } // add Form attribute from previous request in formpost() bodyRequestEncoder.setBodyHttpDatas(bodylist); // finalize request bodyRequestEncoder.finalizeRequest(); // send request channel.write(request); // test if request was chunked and if so, finish the write if (bodyRequestEncoder.isChunked()) { channel.write(bodyRequestEncoder); } channel.flush(); // Now no more use of file representation (and list of HttpData) bodyRequestEncoder.cleanFiles(); // Wait for the server to close the connection. channel.closeFuture().sync(); }
/** * Tests to make sure the max allowed size for multipart requests is enforced. * @throws Exception */ // Disabling test because the encoded size is different at different times and it is hard to predict. Need a better // test //@Test public void sizeLimitationTest() throws Exception { int blobPartSize = 1024; byte[] bytes = TestUtils.getRandomBytes(blobPartSize); int encodedSize = getEncodedSize(bytes); long[] maxSizesAllowed = {encodedSize + 1, encodedSize, encodedSize - 1, 0}; for (long maxSizeAllowed : maxSizesAllowed) { InMemoryFile[] files = {new InMemoryFile(RestUtils.MultipartPost.BLOB_PART, ByteBuffer.wrap(bytes))}; HttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/"); HttpPostRequestEncoder encoder = createEncoder(httpRequest, files); NettyMultipartRequest request = new NettyMultipartRequest(encoder.finalizeRequest(), new MockChannel(), NETTY_METRICS, Collections.emptySet(), maxSizeAllowed); assertTrue("Request channel is not open", request.isOpen()); long currentSizeAdded = 0; boolean failedToAdd = false; while (!encoder.isEndOfInput()) { HttpContent httpContent = encoder.readChunk(PooledByteBufAllocator.DEFAULT); int readableBytes = httpContent.content().readableBytes(); if (currentSizeAdded + readableBytes <= maxSizeAllowed) { request.addContent(httpContent); } else { assertTrue("Max size [" + maxSizeAllowed + "] must be lesser than content size: " + encodedSize, maxSizeAllowed < encodedSize); try { request.addContent(httpContent); fail("Should have failed to add content of size [" + encodedSize + "] because it is over the max size allowed: " + maxSizeAllowed); } catch (RestServiceException e) { failedToAdd = true; assertEquals("Unexpected RestServiceErrorCode", RestServiceErrorCode.RequestTooLarge, e.getErrorCode()); break; } } currentSizeAdded += readableBytes; } assertEquals( "Success state not as expected. maxSizeAllowed=[" + maxSizeAllowed + "], encodedSize expected=[" + encodedSize + "], actual size added=[" + currentSizeAdded + "]", maxSizeAllowed < encodedSize, failedToAdd); } }
/** * Posts a blob with the given {@code headers} and {@code content}. * @param headers the headers required. * @param content the content of the blob. * @param usermetadata the {@link ByteBuffer} that represents user metadata * @return the blob ID of the blob. * @throws Exception */ private String multipartPostBlobAndVerify(HttpHeaders headers, ByteBuffer content, ByteBuffer usermetadata) throws Exception { HttpRequest httpRequest = RestTestUtils.createRequest(HttpMethod.POST, "/", headers); HttpPostRequestEncoder encoder = createEncoder(httpRequest, content, usermetadata); ResponseParts responseParts = nettyClient.sendRequest(encoder.finalizeRequest(), encoder, null).get(); return verifyPostAndReturnBlobId(responseParts); }
/** * Set Form encoding * * @param mode the encoding mode for this form encoding * @return this builder */ Form encoding(HttpPostRequestEncoder.EncoderMode mode);
