From 3860e2466cb9083efe77504724a7b4955c1d75cf Mon Sep 17 00:00:00 2001 From: Jonathan Coates Date: Fri, 12 Mar 2021 09:14:52 +0000 Subject: [PATCH] Add User-Agent to Websockets I think, haven't actually tested this :D:. Closes #730. --- .../computercraft/core/apis/HTTPAPI.java | 251 ++++++++------- .../apis/http/request/HttpRequestHandler.java | 304 +++++++++--------- 2 files changed, 281 insertions(+), 274 deletions(-) diff --git a/src/main/java/dan200/computercraft/core/apis/HTTPAPI.java b/src/main/java/dan200/computercraft/core/apis/HTTPAPI.java index 9f1c5f951..0996e5300 100644 --- a/src/main/java/dan200/computercraft/core/apis/HTTPAPI.java +++ b/src/main/java/dan200/computercraft/core/apis/HTTPAPI.java @@ -3,185 +3,206 @@ * Copyright Daniel Ratcliffe, 2011-2021. Do not distribute without permission. * Send enquiries to dratcliffe@gmail.com */ - package dan200.computercraft.core.apis; -import static dan200.computercraft.core.apis.TableHelper.getStringField; -import static dan200.computercraft.core.apis.TableHelper.optBooleanField; -import static dan200.computercraft.core.apis.TableHelper.optStringField; -import static dan200.computercraft.core.apis.TableHelper.optTableField; - -import java.net.URI; -import java.util.Collections; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; - -import javax.annotation.Nonnull; - import dan200.computercraft.ComputerCraft; import dan200.computercraft.api.lua.IArguments; import dan200.computercraft.api.lua.ILuaAPI; import dan200.computercraft.api.lua.LuaException; import dan200.computercraft.api.lua.LuaFunction; -import dan200.computercraft.core.apis.http.CheckUrl; -import dan200.computercraft.core.apis.http.HTTPRequestException; -import dan200.computercraft.core.apis.http.Resource; -import dan200.computercraft.core.apis.http.ResourceGroup; -import dan200.computercraft.core.apis.http.ResourceQueue; +import dan200.computercraft.core.apis.http.*; import dan200.computercraft.core.apis.http.request.HttpRequest; import dan200.computercraft.core.apis.http.websocket.Websocket; import io.netty.handler.codec.http.DefaultHttpHeaders; +import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpMethod; +import javax.annotation.Nonnull; +import java.net.URI; +import java.util.Collections; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +import static dan200.computercraft.core.apis.TableHelper.*; + /** * The http library allows communicating with web servers, sending and receiving data from them. * * @cc.module http * @hidden */ -public class HTTPAPI implements ILuaAPI { - private final IAPIEnvironment m_apiEnvironment; +public class HTTPAPI implements ILuaAPI +{ + private final IAPIEnvironment apiEnvironment; private final ResourceGroup checkUrls = new ResourceGroup<>(); - private final ResourceGroup requests = new ResourceQueue<>(() -> ComputerCraft.httpMaxRequests); - private final ResourceGroup websockets = new ResourceGroup<>(() -> ComputerCraft.httpMaxWebsockets); + private final ResourceGroup requests = new ResourceQueue<>( () -> ComputerCraft.httpMaxRequests ); + private final ResourceGroup websockets = new ResourceGroup<>( () -> ComputerCraft.httpMaxWebsockets ); - public HTTPAPI(IAPIEnvironment environment) { - this.m_apiEnvironment = environment; + public HTTPAPI( IAPIEnvironment environment ) + { + apiEnvironment = environment; } @Override - public String[] getNames() { - return new String[] {"http"}; + public String[] getNames() + { + return new String[] { "http" }; } @Override - public void startup() { - this.checkUrls.startup(); - this.requests.startup(); - this.websockets.startup(); + public void startup() + { + checkUrls.startup(); + requests.startup(); + websockets.startup(); } @Override - public void update() { + public void shutdown() + { + checkUrls.shutdown(); + requests.shutdown(); + websockets.shutdown(); + } + + @Override + public void update() + { // It's rather ugly to run this here, but we need to clean up // resources as often as possible to reduce blocking. Resource.cleanup(); } - @Override - public void shutdown() { - this.checkUrls.shutdown(); - this.requests.shutdown(); - this.websockets.shutdown(); - } - @LuaFunction - public final Object[] request(IArguments args) throws LuaException { + public final Object[] request( IArguments args ) throws LuaException + { String address, postString, requestMethod; Map headerTable; boolean binary, redirect; - if (args.get(0) instanceof Map) { - Map options = args.getTable(0); - address = getStringField(options, "url"); - postString = optStringField(options, "body", null); - headerTable = optTableField(options, "headers", Collections.emptyMap()); - binary = optBooleanField(options, "binary", false); - requestMethod = optStringField(options, "method", null); - redirect = optBooleanField(options, "redirect", true); + if( args.get( 0 ) instanceof Map ) + { + Map options = args.getTable( 0 ); + address = getStringField( options, "url" ); + postString = optStringField( options, "body", null ); + headerTable = optTableField( options, "headers", Collections.emptyMap() ); + binary = optBooleanField( options, "binary", false ); + requestMethod = optStringField( options, "method", null ); + redirect = optBooleanField( options, "redirect", true ); - } else { + } + else + { // Get URL and post information - address = args.getString(0); - postString = args.optString(1, null); - headerTable = args.optTable(2, Collections.emptyMap()); - binary = args.optBoolean(3, false); + address = args.getString( 0 ); + postString = args.optString( 1, null ); + headerTable = args.optTable( 2, Collections.emptyMap() ); + binary = args.optBoolean( 3, false ); requestMethod = null; redirect = true; } - HttpHeaders headers = getHeaders(headerTable); + HttpHeaders headers = getHeaders( headerTable ); HttpMethod httpMethod; - if (requestMethod == null) { + if( requestMethod == null ) + { httpMethod = postString == null ? HttpMethod.GET : HttpMethod.POST; - } else { - httpMethod = HttpMethod.valueOf(requestMethod.toUpperCase(Locale.ROOT)); - if (httpMethod == null || requestMethod.equalsIgnoreCase("CONNECT")) { - throw new LuaException("Unsupported HTTP method"); + } + else + { + httpMethod = HttpMethod.valueOf( requestMethod.toUpperCase( Locale.ROOT ) ); + if( httpMethod == null || requestMethod.equalsIgnoreCase( "CONNECT" ) ) + { + throw new LuaException( "Unsupported HTTP method" ); } } - try { - URI uri = HttpRequest.checkUri(address); - HttpRequest request = new HttpRequest(this.requests, this.m_apiEnvironment, address, postString, headers, binary, redirect); + try + { + URI uri = HttpRequest.checkUri( address ); + HttpRequest request = new HttpRequest( requests, apiEnvironment, address, postString, headers, binary, redirect ); // Make the request - request.queue(r -> r.request(uri, httpMethod)); + request.queue( r -> r.request( uri, httpMethod ) ); - return new Object[] {true}; - } catch (HTTPRequestException e) { - return new Object[] { - false, - e.getMessage() - }; + return new Object[] { true }; + } + catch( HTTPRequestException e ) + { + return new Object[] { false, e.getMessage() }; + } + } + + @LuaFunction + public final Object[] checkURL( String address ) + { + try + { + URI uri = HttpRequest.checkUri( address ); + new CheckUrl( checkUrls, apiEnvironment, address, uri ).queue( CheckUrl::run ); + + return new Object[] { true }; + } + catch( HTTPRequestException e ) + { + return new Object[] { false, e.getMessage() }; + } + } + + @LuaFunction + public final Object[] websocket( String address, Optional> headerTbl ) throws LuaException + { + if( !ComputerCraft.http_websocket_enable ) + { + throw new LuaException( "Websocket connections are disabled" ); + } + + HttpHeaders headers = getHeaders( headerTbl.orElse( Collections.emptyMap() ) ); + + try + { + URI uri = Websocket.checkUri( address ); + if( !new Websocket( websockets, apiEnvironment, uri, address, headers ).queue( Websocket::connect ) ) + { + throw new LuaException( "Too many websockets already open" ); + } + + return new Object[] { true }; + } + catch( HTTPRequestException e ) + { + return new Object[] { false, e.getMessage() }; } } @Nonnull - private static HttpHeaders getHeaders(@Nonnull Map headerTable) throws LuaException { + private HttpHeaders getHeaders( @Nonnull Map headerTable ) throws LuaException + { HttpHeaders headers = new DefaultHttpHeaders(); - for (Map.Entry entry : headerTable.entrySet()) { + for( Map.Entry entry : headerTable.entrySet() ) + { Object value = entry.getValue(); - if (entry.getKey() instanceof String && value instanceof String) { - try { - headers.add((String) entry.getKey(), value); - } catch (IllegalArgumentException e) { - throw new LuaException(e.getMessage()); + if( entry.getKey() instanceof String && value instanceof String ) + { + try + { + headers.add( (String) entry.getKey(), value ); + } + catch( IllegalArgumentException e ) + { + throw new LuaException( e.getMessage() ); } } } + + if( !headers.contains( HttpHeaderNames.USER_AGENT ) ) + { + headers.set( HttpHeaderNames.USER_AGENT, apiEnvironment.getComputerEnvironment().getUserAgent() ); + } return headers; } - - @LuaFunction - public final Object[] checkURL(String address) { - try { - URI uri = HttpRequest.checkUri(address); - new CheckUrl(this.checkUrls, this.m_apiEnvironment, address, uri).queue(CheckUrl::run); - - return new Object[] {true}; - } catch (HTTPRequestException e) { - return new Object[] { - false, - e.getMessage() - }; - } - } - - @LuaFunction - public final Object[] websocket(String address, Optional> headerTbl) throws LuaException { - if (!ComputerCraft.http_websocket_enable) { - throw new LuaException("Websocket connections are disabled"); - } - - HttpHeaders headers = getHeaders(headerTbl.orElse(Collections.emptyMap())); - - try { - URI uri = Websocket.checkUri(address); - if (!new Websocket(this.websockets, this.m_apiEnvironment, uri, address, headers).queue(Websocket::connect)) { - throw new LuaException("Too many websockets already open"); - } - - return new Object[] {true}; - } catch (HTTPRequestException e) { - return new Object[] { - false, - e.getMessage() - }; - } - } -} +} \ No newline at end of file diff --git a/src/main/java/dan200/computercraft/core/apis/http/request/HttpRequestHandler.java b/src/main/java/dan200/computercraft/core/apis/http/request/HttpRequestHandler.java index 23931eb18..845a185ee 100644 --- a/src/main/java/dan200/computercraft/core/apis/http/request/HttpRequestHandler.java +++ b/src/main/java/dan200/computercraft/core/apis/http/request/HttpRequestHandler.java @@ -3,19 +3,8 @@ * Copyright Daniel Ratcliffe, 2011-2021. Do not distribute without permission. * Send enquiries to dratcliffe@gmail.com */ - package dan200.computercraft.core.apis.http.request; -import static dan200.computercraft.core.apis.http.request.HttpRequest.getHeaderSize; - -import java.io.Closeable; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; - import dan200.computercraft.ComputerCraft; import dan200.computercraft.core.apis.handles.ArrayByteChannel; import dan200.computercraft.core.apis.handles.BinaryReadableHandle; @@ -29,22 +18,20 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.CompositeByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.handler.codec.http.DefaultFullHttpRequest; -import io.netty.handler.codec.http.DefaultHttpHeaders; -import io.netty.handler.codec.http.FullHttpRequest; -import io.netty.handler.codec.http.HttpContent; -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpHeaderValues; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpObject; -import io.netty.handler.codec.http.HttpResponse; -import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.codec.http.HttpUtil; -import io.netty.handler.codec.http.HttpVersion; -import io.netty.handler.codec.http.LastHttpContent; +import io.netty.handler.codec.http.*; -public final class HttpRequestHandler extends SimpleChannelInboundHandler implements Closeable { +import java.io.Closeable; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +import static dan200.computercraft.core.apis.http.request.HttpRequest.getHeaderSize; + +public final class HttpRequestHandler extends SimpleChannelInboundHandler implements Closeable +{ /** * Same as {@link io.netty.handler.codec.MessageAggregator}. */ @@ -53,16 +40,19 @@ public final class HttpRequestHandler extends SimpleChannelInboundHandler 0) { - URI redirect = this.getRedirect(response.status(), response.headers()); - if (redirect != null && !this.uri.equals(redirect) && this.request.redirects.getAndDecrement() > 0) { + if( request.redirects.get() > 0 ) + { + URI redirect = getRedirect( response.status(), response.headers() ); + if( redirect != null && !uri.equals( redirect ) && request.redirects.getAndDecrement() > 0 ) + { // If we have a redirect, and don't end up at the same place, then follow it. // We mark ourselves as disposed first though, to avoid firing events when the channel // becomes inactive or disposed. - this.closed = true; + closed = true; ctx.close(); - try { - HttpRequest.checkUri(redirect); - } catch (HTTPRequestException e) { + try + { + HttpRequest.checkUri( redirect ); + } + catch( HTTPRequestException e ) + { // If we cannot visit this uri, then fail. - this.request.failure(e.getMessage()); + request.failure( e.getMessage() ); return; } - this.request.request(redirect, - response.status() - .code() == 303 ? HttpMethod.GET : this.method); + request.request( redirect, response.status().code() == 303 ? HttpMethod.GET : method ); return; } } - this.responseCharset = HttpUtil.getCharset(response, StandardCharsets.UTF_8); - this.responseStatus = response.status(); - this.responseHeaders.add(response.headers()); + responseCharset = HttpUtil.getCharset( response, StandardCharsets.UTF_8 ); + responseStatus = response.status(); + responseHeaders.add( response.headers() ); } - if (message instanceof HttpContent) { + if( message instanceof HttpContent ) + { HttpContent content = (HttpContent) message; - if (this.responseBody == null) { - this.responseBody = ctx.alloc() - .compositeBuffer(DEFAULT_MAX_COMPOSITE_BUFFER_COMPONENTS); + if( responseBody == null ) + { + responseBody = ctx.alloc().compositeBuffer( DEFAULT_MAX_COMPOSITE_BUFFER_COMPONENTS ); } ByteBuf partial = content.content(); - if (partial.isReadable()) { + if( partial.isReadable() ) + { // If we've read more than we're allowed to handle, abort as soon as possible. - if (this.options.maxDownload != 0 && this.responseBody.readableBytes() + partial.readableBytes() > this.options.maxDownload) { - this.closed = true; + if( options.maxDownload != 0 && responseBody.readableBytes() + partial.readableBytes() > options.maxDownload ) + { + closed = true; ctx.close(); - this.request.failure("Response is too large"); + request.failure( "Response is too large" ); return; } - this.responseBody.addComponent(true, partial.retain()); + responseBody.addComponent( true, partial.retain() ); } - if (message instanceof LastHttpContent) { + if( message instanceof LastHttpContent ) + { LastHttpContent last = (LastHttpContent) message; - this.responseHeaders.add(last.trailingHeaders()); + responseHeaders.add( last.trailingHeaders() ); // Set the content length, if not already given. - if (this.responseHeaders.contains(HttpHeaderNames.CONTENT_LENGTH)) { - this.responseHeaders.set(HttpHeaderNames.CONTENT_LENGTH, this.responseBody.readableBytes()); + if( responseHeaders.contains( HttpHeaderNames.CONTENT_LENGTH ) ) + { + responseHeaders.set( HttpHeaderNames.CONTENT_LENGTH, responseBody.readableBytes() ); } ctx.close(); - this.sendResponse(); + sendResponse(); } } } + @Override + public void exceptionCaught( ChannelHandlerContext ctx, Throwable cause ) + { + if( ComputerCraft.logPeripheralErrors ) ComputerCraft.log.error( "Error handling HTTP response", cause ); + request.failure( cause ); + } + + private void sendResponse() + { + // Read the ByteBuf into a channel. + CompositeByteBuf body = responseBody; + byte[] bytes = body == null ? EMPTY_BYTES : NetworkUtils.toBytes( body ); + + // Decode the headers + HttpResponseStatus status = responseStatus; + Map headers = new HashMap<>(); + for( Map.Entry header : responseHeaders ) + { + String existing = headers.get( header.getKey() ); + headers.put( header.getKey(), existing == null ? header.getValue() : existing + "," + header.getValue() ); + } + + // Fire off a stats event + request.environment().addTrackingChange( TrackingField.HTTP_DOWNLOAD, getHeaderSize( responseHeaders ) + bytes.length ); + + // Prepare to queue an event + ArrayByteChannel contents = new ArrayByteChannel( bytes ); + HandleGeneric reader = request.isBinary() + ? BinaryReadableHandle.of( contents ) + : new EncodedReadableHandle( EncodedReadableHandle.open( contents, responseCharset ) ); + HttpResponseHandle stream = new HttpResponseHandle( reader, status.code(), status.reasonPhrase(), headers ); + + if( status.code() >= 200 && status.code() < 400 ) + { + request.success( stream ); + } + else + { + request.failure( status.reasonPhrase(), stream ); + } + } + /** * Determine the redirect from this response. * - * @param status The status of the HTTP response. + * @param status The status of the HTTP response. * @param headers The headers of the HTTP response. * @return The URI to redirect to, or {@code null} if no redirect should occur. */ - private URI getRedirect(HttpResponseStatus status, HttpHeaders headers) { + private URI getRedirect( HttpResponseStatus status, HttpHeaders headers ) + { int code = status.code(); - if (code < 300 || code > 307 || code == 304 || code == 306) { + if( code < 300 || code > 307 || code == 304 || code == 306 ) return null; + + String location = headers.get( HttpHeaderNames.LOCATION ); + if( location == null ) return null; + + try + { + return uri.resolve( new URI( location ) ); + } + catch( IllegalArgumentException | URISyntaxException e ) + { return null; } - - String location = headers.get(HttpHeaderNames.LOCATION); - if (location == null) { - return null; - } - - try { - return this.uri.resolve(new URI( location )); - } catch( IllegalArgumentException | URISyntaxException e ) { - return null; - } - } - - private void sendResponse() { - // Read the ByteBuf into a channel. - CompositeByteBuf body = this.responseBody; - byte[] bytes = body == null ? EMPTY_BYTES : NetworkUtils.toBytes(body); - - // Decode the headers - HttpResponseStatus status = this.responseStatus; - Map headers = new HashMap<>(); - for (Map.Entry header : this.responseHeaders) { - String existing = headers.get(header.getKey()); - headers.put(header.getKey(), existing == null ? header.getValue() : existing + "," + header.getValue()); - } - - // Fire off a stats event - this.request.environment() - .addTrackingChange(TrackingField.HTTP_DOWNLOAD, getHeaderSize(this.responseHeaders) + bytes.length); - - // Prepare to queue an event - ArrayByteChannel contents = new ArrayByteChannel(bytes); - HandleGeneric reader = this.request.isBinary() ? BinaryReadableHandle.of(contents) : new EncodedReadableHandle(EncodedReadableHandle.open(contents, - this.responseCharset)); - HttpResponseHandle stream = new HttpResponseHandle(reader, status.code(), status.reasonPhrase(), headers); - - if (status.code() >= 200 && status.code() < 400) { - this.request.success(stream); - } else { - this.request.failure(status.reasonPhrase(), stream); - } } @Override - public void close() { - this.closed = true; - if (this.responseBody != null) { - this.responseBody.release(); - this.responseBody = null; + public void close() + { + closed = true; + if( responseBody != null ) + { + responseBody.release(); + responseBody = null; } } -} +} \ No newline at end of file