mirrors/CC-Tweaked
1
0
Fork 0
mirror of https://github.com/SquidDev-CC/CC-Tweaked synced 2025-10-30 21:23:00 +00:00
Code Issues Releases Wiki Activity

WIP: Http rework (#98)

Browse Source 
- Move all HTTP tasks to a unified "MonitoredResource" model. This
   provides a uniform way of tracking object's lifetimes and disposing
   of them when complete.

 - Rewrite HTTP requests to use Netty instead of standard Java. This
   offers several advantages:
    - We have access to more HTTP verbs (mostly PATCH).
    - We can now do http -> https redirects.
    - We no longer need to spawn in a new thread for each HTTP request.
      While we do need to run some tasks off-thread in order to resolve
      IPs, it's generally a much shorter task, and so is less likely to
      inflate the thread pool.

 - Introduce several limits for the http API:
    - There's a limit on how many HTTP requests and websockets may exist
      at the same time. If the limit is reached, additional ones will be
      queued up until pending requests have finished.
    - HTTP requests may upload a maximum of 4Mib and download a maximum
      of 16Mib (configurable).

 - .getResponseCode now returns the status text, as well as the status
   code.
This commit is contained in:
SquidDev
2019-01-11 11:33:05 +00:00
committed by GitHub
parent 101b3500cc
commit 932f8a44fc
28 changed files with 1719 additions and 939 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.gitignore vendored
View File

@@ -1,13 +1,17 @@
# Build directories
/classes
/logs
/build /build
/out /out
# Runtime directories
/run /run
/run-* /run-*
/test-files
*.ipr *.ipr
*.iws *.iws
*.iml *.iml
.idea .idea
.gradle .gradle
/luaj-2.0.3/lib
/luaj-2.0.3/*.jar
*.DS_Store *.DS_Store
/test-files

13
src/main/java/dan200/computercraft/ComputerCraft.java
View File

@@ -23,6 +23,7 @@ import dan200.computercraft.api.turtle.ITurtleUpgrade;
import dan200.computercraft.api.turtle.event.TurtleAction; import dan200.computercraft.api.turtle.event.TurtleAction;
import dan200.computercraft.core.apis.AddressPredicate; import dan200.computercraft.core.apis.AddressPredicate;
import dan200.computercraft.core.apis.ApiFactories; import dan200.computercraft.core.apis.ApiFactories;
import dan200.computercraft.core.apis.http.websocket.Websocket;
import dan200.computercraft.core.filesystem.ComboMount; import dan200.computercraft.core.filesystem.ComboMount;
import dan200.computercraft.core.filesystem.FileMount; import dan200.computercraft.core.filesystem.FileMount;
import dan200.computercraft.core.filesystem.FileSystemMount; import dan200.computercraft.core.filesystem.FileSystemMount;
@@ -57,6 +58,7 @@ import dan200.computercraft.shared.turtle.blocks.TileTurtle;
import dan200.computercraft.shared.turtle.upgrades.*; import dan200.computercraft.shared.turtle.upgrades.*;
import dan200.computercraft.shared.util.CreativeTabMain; import dan200.computercraft.shared.util.CreativeTabMain;
import dan200.computercraft.shared.util.IDAssigner; import dan200.computercraft.shared.util.IDAssigner;
import dan200.computercraft.shared.util.IoUtil;
import dan200.computercraft.shared.wired.CapabilityWiredElement; import dan200.computercraft.shared.wired.CapabilityWiredElement;
import dan200.computercraft.shared.wired.WiredNode; import dan200.computercraft.shared.wired.WiredNode;
import net.minecraft.entity.player.EntityPlayer; import net.minecraft.entity.player.EntityPlayer;
@@ -77,7 +79,6 @@ import net.minecraftforge.fml.common.network.NetworkRegistry;
import net.minecraftforge.fml.common.network.simpleimpl.IMessage; import net.minecraftforge.fml.common.network.simpleimpl.IMessage;
import net.minecraftforge.fml.common.network.simpleimpl.SimpleNetworkWrapper; import net.minecraftforge.fml.common.network.simpleimpl.SimpleNetworkWrapper;
import net.minecraftforge.fml.relauncher.Side; import net.minecraftforge.fml.relauncher.Side;
import org.apache.commons.io.IOUtils;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import java.io.*; import java.io.*;
@@ -137,6 +138,12 @@ public class ComputerCraft
public static AddressPredicate http_whitelist = new AddressPredicate( DEFAULT_HTTP_WHITELIST ); public static AddressPredicate http_whitelist = new AddressPredicate( DEFAULT_HTTP_WHITELIST );
public static AddressPredicate http_blacklist = new AddressPredicate( DEFAULT_HTTP_BLACKLIST ); public static AddressPredicate http_blacklist = new AddressPredicate( DEFAULT_HTTP_BLACKLIST );
public static int httpMaxRequests = 16;
public static long httpMaxDownload = 16 * 1024 * 1024;
public static long httpMaxUpload = 4 * 1024 * 1024;
public static int httpMaxWebsockets = 4;
public static int httpMaxWebsocketMessage = Websocket.MAX_MESSAGE_SIZE;
public static boolean enableCommandBlock = false; public static boolean enableCommandBlock = false;
public static int modem_range = 64; public static int modem_range = 64;
public static int modem_highAltitudeRange = 384; public static int modem_highAltitudeRange = 384;
@@ -599,12 +606,12 @@ public class ComputerCraft
} }
else else
{ {
IOUtils.closeQuietly( zipFile ); IoUtil.closeQuietly( zipFile );
} }
} }
catch( IOException e ) catch( IOException e )
{ {
if( zipFile != null ) IOUtils.closeQuietly( zipFile ); if( zipFile != null ) IoUtil.closeQuietly( zipFile );
} }
} }
} }

196
src/main/java/dan200/computercraft/core/apis/HTTPAPI.java
View File

@@ -6,39 +6,40 @@
package dan200.computercraft.core.apis; package dan200.computercraft.core.apis;
import com.google.common.collect.ImmutableSet;
import dan200.computercraft.ComputerCraft; import dan200.computercraft.ComputerCraft;
import dan200.computercraft.api.lua.ILuaAPI; import dan200.computercraft.api.lua.ILuaAPI;
import dan200.computercraft.api.lua.ILuaContext; import dan200.computercraft.api.lua.ILuaContext;
import dan200.computercraft.api.lua.LuaException; import dan200.computercraft.api.lua.LuaException;
import dan200.computercraft.core.apis.http.*; import dan200.computercraft.core.apis.http.CheckUrl;
import dan200.computercraft.core.apis.http.HTTPRequestException;
import dan200.computercraft.core.apis.http.Resource;
import dan200.computercraft.core.apis.http.ResourceQueue;
import dan200.computercraft.core.apis.http.request.HttpRequest;
import dan200.computercraft.core.apis.http.websocket.Websocket;
import io.netty.handler.codec.http.DefaultHttpHeaders;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpMethod;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.Closeable;
import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.net.URL; import java.util.Collections;
import java.util.*; import java.util.Locale;
import java.util.concurrent.Future; import java.util.Map;
import static dan200.computercraft.core.apis.ArgumentHelper.*; import static dan200.computercraft.core.apis.ArgumentHelper.*;
import static dan200.computercraft.core.apis.TableHelper.*; import static dan200.computercraft.core.apis.TableHelper.*;
public class HTTPAPI implements ILuaAPI public class HTTPAPI implements ILuaAPI
{ {
private static final Set<String> HTTP_METHODS = ImmutableSet.of(
"GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE"
);
private final IAPIEnvironment m_apiEnvironment; private final IAPIEnvironment m_apiEnvironment;
private final List<Future<?>> m_httpTasks;
private final Set<Closeable> m_closeables; private final ResourceQueue<CheckUrl> checkUrls = new ResourceQueue<>();
private final ResourceQueue<HttpRequest> requests = new ResourceQueue<>( () -> ComputerCraft.httpMaxRequests );
private final ResourceQueue<Websocket> websockets = new ResourceQueue<>( () -> ComputerCraft.httpMaxWebsockets );
public HTTPAPI( IAPIEnvironment environment ) public HTTPAPI( IAPIEnvironment environment )
{ {
m_apiEnvironment = environment; m_apiEnvironment = environment;
m_httpTasks = new ArrayList<>();
m_closeables = new HashSet<>();
} }
@Override @Override
@@ -50,45 +51,27 @@ public class HTTPAPI implements ILuaAPI
} }
@Override @Override
public void update() public void startup()
{ {
// Wait for all of our http requests checkUrls.startup();
synchronized( m_httpTasks ) requests.startup();
{ websockets.startup();
Iterator<Future<?>> it = m_httpTasks.iterator();
while( it.hasNext() )
{
final Future<?> h = it.next();
if( h.isDone() ) it.remove();
}
}
} }
@Override @Override
public void shutdown() public void shutdown()
{ {
synchronized( m_httpTasks ) checkUrls.shutdown();
{ requests.shutdown();
for( Future<?> r : m_httpTasks ) websockets.shutdown();
{ }
r.cancel( false );
} @Override
m_httpTasks.clear(); public void update()
} {
synchronized( m_closeables ) // It's rather ugly to run this here, but we need to clean up
{ // resources as often as possible to reduce blocking.
for( Closeable x : m_closeables ) Resource.cleanup();
{
try
{
x.close();
}
catch( IOException ignored )
{
}
}
m_closeables.clear();
}
} }
@Nonnull @Nonnull
@@ -109,16 +92,16 @@ public class HTTPAPI implements ILuaAPI
{ {
case 0: // request case 0: // request
{ {
String urlString, postString, requestMethod; String address, postString, requestMethod;
Map<Object, Object> headerTable; Map<Object, Object> headerTable;
boolean binary, redirect; boolean binary, redirect;
if( args.length >= 1 && args[0] instanceof Map ) if( args.length >= 1 && args[0] instanceof Map )
{ {
Map<?, ?> options = (Map) args[0]; Map<?, ?> options = (Map) args[0];
urlString = getStringField( options, "url" ); address = getStringField( options, "url" );
postString = optStringField( options, "body", null ); postString = optStringField( options, "body", null );
headerTable = optTableField( options, "headers", null ); headerTable = optTableField( options, "headers", Collections.emptyMap() );
binary = optBooleanField( options, "binary", false ); binary = optBooleanField( options, "binary", false );
requestMethod = optStringField( options, "method", null ); requestMethod = optStringField( options, "method", null );
redirect = optBooleanField( options, "redirect", true ); redirect = optBooleanField( options, "redirect", true );
@@ -127,43 +110,46 @@ public class HTTPAPI implements ILuaAPI
else else
{ {
// Get URL and post information // Get URL and post information
urlString = getString( args, 0 ); address = getString( args, 0 );
postString = optString( args, 1, null ); postString = optString( args, 1, null );
headerTable = optTable( args, 2, null ); headerTable = optTable( args, 2, Collections.emptyMap() );
binary = optBoolean( args, 3, false ); binary = optBoolean( args, 3, false );
requestMethod = null; requestMethod = null;
redirect = true; redirect = true;
} }
Map<String, String> headers = null; HttpHeaders headers = getHeaders( headerTable );
if( headerTable != null )
HttpMethod httpMethod;
if( requestMethod == null )
{ {
headers = new HashMap<>( headerTable.size() ); httpMethod = postString == null ? HttpMethod.GET : HttpMethod.POST;
for( Object key : headerTable.keySet() ) }
else
{
httpMethod = HttpMethod.valueOf( requestMethod.toUpperCase( Locale.ROOT ) );
if( httpMethod == null || requestMethod.equalsIgnoreCase( "CONNECT" ) )
{ {
Object value = headerTable.get( key ); throw new LuaException( "Unsupported HTTP method" );
if( key instanceof String && value instanceof String )
{
headers.put( (String) key, (String) value );
}
} }
} }
if( requestMethod != null && !HTTP_METHODS.contains( requestMethod ) )
{
throw new LuaException( "Unsupported HTTP method" );
}
// Make the request
try try
{ {
URL url = HTTPRequest.checkURL( urlString ); URI uri = HttpRequest.checkUri( address );
HTTPRequest request = new HTTPRequest( m_apiEnvironment, urlString, url, postString, headers, binary, requestMethod, redirect );
synchronized( m_httpTasks ) HttpRequest request = new HttpRequest( requests, m_apiEnvironment, address, postString, headers, binary, redirect );
long requestBody = request.body().readableBytes() + HttpRequest.getHeaderSize( headers );
if( ComputerCraft.httpMaxUpload != 0 && requestBody > ComputerCraft.httpMaxUpload )
{ {
m_httpTasks.add( HTTPExecutor.EXECUTOR.submit( request ) ); throw new HTTPRequestException( "Request body is too large" );
} }
// Make the request
request.queue( r -> r.request( uri, httpMethod ) );
return new Object[] { true }; return new Object[] { true };
} }
catch( HTTPRequestException e ) catch( HTTPRequestException e )
@@ -171,21 +157,16 @@ public class HTTPAPI implements ILuaAPI
return new Object[] { false, e.getMessage() }; return new Object[] { false, e.getMessage() };
} }
} }
case 1: case 1: // checkURL
{ {
// checkURL String address = getString( args, 0 );
// Get URL
String urlString = getString( args, 0 );
// Check URL // Check URL
try try
{ {
URL url = HTTPRequest.checkURL( urlString ); URI uri = HttpRequest.checkUri( address );
HTTPCheck check = new HTTPCheck( m_apiEnvironment, urlString, url ); new CheckUrl( checkUrls, m_apiEnvironment, address, uri ).queue( CheckUrl::run );
synchronized( m_httpTasks )
{
m_httpTasks.add( HTTPExecutor.EXECUTOR.submit( check ) );
}
return new Object[] { true }; return new Object[] { true };
} }
catch( HTTPRequestException e ) catch( HTTPRequestException e )
@@ -198,31 +179,18 @@ public class HTTPAPI implements ILuaAPI
String address = getString( args, 0 ); String address = getString( args, 0 );
Map<Object, Object> headerTbl = optTable( args, 1, Collections.emptyMap() ); Map<Object, Object> headerTbl = optTable( args, 1, Collections.emptyMap() );
HashMap<String, String> headers = new HashMap<>( headerTbl.size() );
for( Object key : headerTbl.keySet() )
{
Object value = headerTbl.get( key );
if( key instanceof String && value instanceof String )
{
headers.put( (String) key, (String) value );
}
}
if( !ComputerCraft.http_websocket_enable ) if( !ComputerCraft.http_websocket_enable )
{ {
throw new LuaException( "Websocket connections are disabled" ); throw new LuaException( "Websocket connections are disabled" );
} }
HttpHeaders headers = getHeaders( headerTbl );
try try
{ {
URI uri = WebsocketConnector.checkURI( address ); URI uri = Websocket.checkUri( address );
int port = WebsocketConnector.getPort( uri ); new Websocket( websockets, m_apiEnvironment, uri, address, headers ).queue( Websocket::connect );
Future<?> connector = WebsocketConnector.createConnector( m_apiEnvironment, this, uri, address, port, headers );
synchronized( m_httpTasks )
{
m_httpTasks.add( connector );
}
return new Object[] { true }; return new Object[] { true };
} }
catch( HTTPRequestException e ) catch( HTTPRequestException e )
@@ -237,19 +205,25 @@ public class HTTPAPI implements ILuaAPI
} }
} }
public void addCloseable( Closeable closeable ) @Nonnull
private static HttpHeaders getHeaders( @Nonnull Map<?, ?> headerTable ) throws LuaException
{ {
synchronized( m_closeables ) HttpHeaders headers = new DefaultHttpHeaders();
for( Object key : headerTable.keySet() )
{ {
m_closeables.add( closeable ); Object value = headerTable.get( key );
} if( key instanceof String && value instanceof String )
} {
try
public void removeCloseable( Closeable closeable ) {
{ headers.add( (String) key, value );
synchronized( m_closeables ) }
{ catch( IllegalArgumentException e )
m_closeables.remove( closeable ); {
throw new LuaException( e.getMessage() );
}
}
} }
return headers;
} }
} }

11
src/main/java/dan200/computercraft/core/apis/handles/HandleGeneric.java
View File

@@ -8,6 +8,7 @@ package dan200.computercraft.core.apis.handles;
import dan200.computercraft.api.lua.ILuaObject; import dan200.computercraft.api.lua.ILuaObject;
import dan200.computercraft.api.lua.LuaException; import dan200.computercraft.api.lua.LuaException;
import dan200.computercraft.shared.util.IoUtil;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.Closeable; import java.io.Closeable;
@@ -35,14 +36,8 @@ public abstract class HandleGeneric implements ILuaObject
protected final void close() protected final void close()
{ {
try m_open = false;
{ IoUtil.closeQuietly( m_closable );
m_closable.close();
m_open = false;
}
catch( IOException ignored )
{
}
m_closable = null; m_closable = null;
} }

65
src/main/java/dan200/computercraft/core/apis/http/CheckUrl.java Normal file
View File

@@ -0,0 +1,65 @@
/*
* This file is part of ComputerCraft - http://www.computercraft.info
* Copyright Daniel Ratcliffe, 2011-2019. Do not distribute without permission.
* Send enquiries to dratcliffe@gmail.com
*/
package dan200.computercraft.core.apis.http;
import dan200.computercraft.core.apis.IAPIEnvironment;
import java.net.URI;
import java.util.concurrent.Future;
/**
* Checks a URL using {@link NetworkUtils#getAddress(String, int, boolean)}}
*
* This requires a DNS lookup, and so needs to occur off-thread.
*/
public class CheckUrl extends Resource<CheckUrl>
{
private static final String EVENT = "http_check";
private Future<?> future;
private final IAPIEnvironment environment;
private final String address;
private final String host;
public CheckUrl( ResourceQueue<CheckUrl> limiter, IAPIEnvironment environment, String address, URI uri )
{
super( limiter );
this.environment = environment;
this.address = address;
this.host = uri.getHost();
}
public void run()
{
if( isClosed() ) return;
future = NetworkUtils.EXECUTOR.submit( this::doRun );
checkClosed();
}
private void doRun()
{
if( isClosed() ) return;
try
{
NetworkUtils.getAddress( host, 80, false );
if( tryClose() ) environment.queueEvent( EVENT, new Object[] { address, true } );
}
catch( HTTPRequestException e )
{
if( tryClose() ) environment.queueEvent( EVENT, new Object[] { address, false, e.getMessage() } );
}
}
@Override
protected void dispose()
{
super.dispose();
future = closeFuture( future );
}
}

39
src/main/java/dan200/computercraft/core/apis/http/HTTPCheck.java
View File

@@ -1,39 +0,0 @@
/*
* This file is part of ComputerCraft - http://www.computercraft.info
* Copyright Daniel Ratcliffe, 2011-2019. Do not distribute without permission.
* Send enquiries to dratcliffe@gmail.com
*/
package dan200.computercraft.core.apis.http;
import dan200.computercraft.core.apis.IAPIEnvironment;
import java.net.URL;
public class HTTPCheck implements Runnable
{
private final IAPIEnvironment environment;
private final String urlString;
private final URL url;
public HTTPCheck( IAPIEnvironment environment, String urlString, URL url )
{
this.environment = environment;
this.urlString = urlString;
this.url = url;
}
@Override
public void run()
{
try
{
HTTPRequest.checkHost( url.getHost() );
environment.queueEvent( "http_check", new Object[] { urlString, true } );
}
catch( HTTPRequestException e )
{
environment.queueEvent( "http_check", new Object[] { urlString, false, e.getMessage() } );
}
}
}

41
src/main/java/dan200/computercraft/core/apis/http/HTTPExecutor.java
View File

@@ -1,41 +0,0 @@
/*
* This file is part of ComputerCraft - http://www.computercraft.info
* Copyright Daniel Ratcliffe, 2011-2019. Do not distribute without permission.
* Send enquiries to dratcliffe@gmail.com
*/
package dan200.computercraft.core.apis.http;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import dan200.computercraft.shared.util.ThreadUtils;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* Just a shared object for executing simple HTTP related tasks.
*/
public final class HTTPExecutor
{
public static final ListeningExecutorService EXECUTOR = MoreExecutors.listeningDecorator( new ThreadPoolExecutor(
4, Integer.MAX_VALUE,
60L, TimeUnit.SECONDS,
new SynchronousQueue<>(),
ThreadUtils.builder( "HTTP" )
.setPriority( Thread.MIN_PRIORITY + (Thread.NORM_PRIORITY - Thread.MIN_PRIORITY) / 2 )
.build()
) );
public static final EventLoopGroup LOOP_GROUP = new NioEventLoopGroup( 4, ThreadUtils.builder( "Netty" )
.setPriority( Thread.MIN_PRIORITY + (Thread.NORM_PRIORITY - Thread.MIN_PRIORITY) / 2 )
.build()
);
private HTTPExecutor()
{
}
}

287
src/main/java/dan200/computercraft/core/apis/http/HTTPRequest.java
View File

@@ -1,287 +0,0 @@
/*
* This file is part of ComputerCraft - http://www.computercraft.info
* Copyright Daniel Ratcliffe, 2011-2019. Do not distribute without permission.
* Send enquiries to dratcliffe@gmail.com
*/
package dan200.computercraft.core.apis.http;
import com.google.common.base.Joiner;
import com.google.common.io.ByteStreams;
import dan200.computercraft.ComputerCraft;
import dan200.computercraft.api.lua.ILuaContext;
import dan200.computercraft.api.lua.ILuaObject;
import dan200.computercraft.api.lua.LuaException;
import dan200.computercraft.core.apis.IAPIEnvironment;
import dan200.computercraft.core.apis.handles.ArrayByteChannel;
import dan200.computercraft.core.apis.handles.BinaryReadableHandle;
import dan200.computercraft.core.apis.handles.EncodedReadableHandle;
import dan200.computercraft.core.tracking.TrackingField;
import javax.annotation.Nonnull;
import java.io.*;
import java.net.*;
import java.nio.channels.SeekableByteChannel;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class HTTPRequest implements Runnable
{
public static URL checkURL( String urlString ) throws HTTPRequestException
{
URL url;
try
{
url = new URL( urlString );
}
catch( MalformedURLException e )
{
throw new HTTPRequestException( "URL malformed" );
}
// Validate the URL
String protocol = url.getProtocol().toLowerCase();
if( !protocol.equals( "http" ) && !protocol.equals( "https" ) )
{
throw new HTTPRequestException( "URL not http" );
}
// Compare the URL to the whitelist
if( !ComputerCraft.http_whitelist.matches( url.getHost() ) || ComputerCraft.http_blacklist.matches( url.getHost() ) )
{
throw new HTTPRequestException( "Domain not permitted" );
}
return url;
}
public static InetAddress checkHost( String host ) throws HTTPRequestException
{
try
{
InetAddress resolved = InetAddress.getByName( host );
if( !ComputerCraft.http_whitelist.matches( resolved ) || ComputerCraft.http_blacklist.matches( resolved ) )
{
throw new HTTPRequestException( "Domain not permitted" );
}
return resolved;
}
catch( UnknownHostException e )
{
throw new HTTPRequestException( "Unknown host" );
}
}
private final IAPIEnvironment m_environment;
private final URL m_url;
private final String m_urlString;
private final String m_postText;
private final Map<String, String> m_headers;
private boolean m_binary;
private final String m_method;
private final boolean m_followRedirects;
public HTTPRequest( IAPIEnvironment environment, String urlString, URL url, final String postText, final Map<String, String> headers, boolean binary, final String method, final boolean followRedirects ) throws HTTPRequestException
{
m_environment = environment;
m_urlString = urlString;
m_url = url;
m_binary = binary;
m_postText = postText;
m_headers = headers;
m_method = method;
m_followRedirects = followRedirects;
}
@Override
public void run()
{
// First verify the address is allowed.
try
{
checkHost( m_url.getHost() );
}
catch( HTTPRequestException e )
{
// Queue the failure event if not.
String error = e.getMessage();
m_environment.queueEvent( "http_failure", new Object[] { m_urlString, error == null ? "Could not connect" : error, null } );
return;
}
try
{
// Connect to the URL
HttpURLConnection connection = (HttpURLConnection) m_url.openConnection();
if( m_postText != null )
{
connection.setRequestMethod( "POST" );
connection.setDoOutput( true );
}
else
{
connection.setRequestMethod( "GET" );
}
if( m_method != null ) connection.setRequestMethod( m_method );
connection.setInstanceFollowRedirects( m_followRedirects );
// Set headers
connection.setRequestProperty( "accept-charset", "UTF-8" );
if( m_postText != null )
{
connection.setRequestProperty( "content-type", "application/x-www-form-urlencoded; charset=utf-8" );
}
if( m_headers != null )
{
for( Map.Entry<String, String> header : m_headers.entrySet() )
{
connection.setRequestProperty( header.getKey(), header.getValue() );
}
}
// Add request size and count to the tracker before opening the connection
m_environment.addTrackingChange( TrackingField.HTTP_REQUESTS );
m_environment.addTrackingChange( TrackingField.HTTP_UPLOAD,
getHeaderSize( connection.getRequestProperties() ) + (m_postText == null ? 0 : m_postText.length()) );
// Send POST text
if( m_postText != null )
{
OutputStream os = connection.getOutputStream();
OutputStreamWriter osw;
try
{
osw = new OutputStreamWriter( os, "UTF-8" );
}
catch( UnsupportedEncodingException e )
{
osw = new OutputStreamWriter( os );
}
BufferedWriter writer = new BufferedWriter( osw );
writer.write( m_postText, 0, m_postText.length() );
writer.close();
}
// Read response
InputStream is;
int code = connection.getResponseCode();
boolean responseSuccess;
if( code >= 200 && code < 400 )
{
is = connection.getInputStream();
responseSuccess = true;
}
else
{
is = connection.getErrorStream();
responseSuccess = false;
}
byte[] result = ByteStreams.toByteArray( is );
is.close();
String encoding = connection.getContentEncoding();
Charset charset = encoding != null && Charset.isSupported( encoding )
? Charset.forName( encoding ) : StandardCharsets.UTF_8;
// We've got some sort of response, so let's build a resulting object.
Joiner joiner = Joiner.on( ',' );
Map<String, String> headers = new HashMap<>();
for( Map.Entry<String, List<String>> header : connection.getHeaderFields().entrySet() )
{
headers.put( header.getKey(), joiner.join( header.getValue() ) );
}
m_environment.addTrackingChange( TrackingField.HTTP_DOWNLOAD,
getHeaderSize( connection.getHeaderFields() ) + result.length );
SeekableByteChannel contents = new ArrayByteChannel( result );
ILuaObject stream = wrapStream(
m_binary
? new BinaryReadableHandle( contents )
: new EncodedReadableHandle( EncodedReadableHandle.open( contents, charset ) ),
connection.getResponseCode(), headers
);
connection.disconnect(); // disconnect
// Queue the appropriate event.
if( responseSuccess )
{
m_environment.queueEvent( "http_success", new Object[] { m_urlString, stream } );
}
else
{
m_environment.queueEvent( "http_failure", new Object[] { m_urlString, "Could not connect", stream } );
}
}
catch( IOException e )
{
// There was an error
m_environment.queueEvent( "http_failure", new Object[] { m_urlString, "Could not connect", null } );
}
}
private static ILuaObject wrapStream( final ILuaObject reader, final int responseCode, final Map<String, String> responseHeaders )
{
String[] oldMethods = reader.getMethodNames();
final int methodOffset = oldMethods.length;
final String[] newMethods = Arrays.copyOf( oldMethods, oldMethods.length + 2 );
newMethods[methodOffset + 0] = "getResponseCode";
newMethods[methodOffset + 1] = "getResponseHeaders";
return new ILuaObject()
{
@Nonnull
@Override
public String[] getMethodNames()
{
return newMethods;
}
@Override
public Object[] callMethod( @Nonnull ILuaContext context, int method, @Nonnull Object[] args ) throws LuaException, InterruptedException
{
if( method < methodOffset )
{
return reader.callMethod( context, method, args );
}
switch( method - methodOffset )
{
case 0:
{
// getResponseCode
return new Object[] { responseCode };
}
case 1:
{
// getResponseHeaders
return new Object[] { responseHeaders };
}
default:
{
return null;
}
}
}
};
}
private static long getHeaderSize( Map<String, List<String>> headers )
{
long size = 0;
for( Map.Entry<String, List<String>> header : headers.entrySet() )
{
size += header.getKey() == null ? 0 : header.getKey().length();
for( String value : header.getValue() ) size += value == null ? 0 : value.length() + 1;
}
return size;
}
}

6
src/main/java/dan200/computercraft/core/apis/http/HTTPRequestException.java
View File

@@ -14,4 +14,10 @@ public class HTTPRequestException extends Exception
{ {
super( s ); super( s );
} }
@Override
public Throwable fillInStackTrace()
{
return this;
}
} }

154
src/main/java/dan200/computercraft/core/apis/http/NetworkUtils.java Normal file
View File

@@ -0,0 +1,154 @@
/*
* This file is part of ComputerCraft - http://www.computercraft.info
* Copyright Daniel Ratcliffe, 2011-2019. Do not distribute without permission.
* Send enquiries to dratcliffe@gmail.com
*/
package dan200.computercraft.core.apis.http;
import dan200.computercraft.ComputerCraft;
import dan200.computercraft.shared.util.ThreadUtils;
import io.netty.buffer.ByteBuf;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import javax.net.ssl.SSLException;
import javax.net.ssl.TrustManagerFactory;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.security.KeyStore;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* Just a shared object for executing simple HTTP related tasks.
*/
public final class NetworkUtils
{
public static final ExecutorService EXECUTOR = new ThreadPoolExecutor(
4, Integer.MAX_VALUE,
60L, TimeUnit.SECONDS,
new SynchronousQueue<>(),
ThreadUtils.builder( "Network" )
.setPriority( Thread.MIN_PRIORITY + (Thread.NORM_PRIORITY - Thread.MIN_PRIORITY) / 2 )
.build()
);
public static final EventLoopGroup LOOP_GROUP = new NioEventLoopGroup( 4, ThreadUtils.builder( "Netty" )
.setPriority( Thread.MIN_PRIORITY + (Thread.NORM_PRIORITY - Thread.MIN_PRIORITY) / 2 )
.build()
);
private NetworkUtils()
{
}
private static final Object sslLock = new Object();
private static TrustManagerFactory trustManager;
private static SslContext sslContext;
private static boolean triedSslContext = false;
private static TrustManagerFactory getTrustManager()
{
if( trustManager != null ) return trustManager;
synchronized( sslLock )
{
if( trustManager != null ) return trustManager;
TrustManagerFactory tmf = null;
try
{
tmf = TrustManagerFactory.getInstance( TrustManagerFactory.getDefaultAlgorithm() );
tmf.init( (KeyStore) null );
}
catch( Exception e )
{
ComputerCraft.log.error( "Cannot setup trust manager", e );
}
return trustManager = tmf;
}
}
public static SslContext getSslContext() throws HTTPRequestException
{
if( sslContext != null || triedSslContext ) return sslContext;
synchronized( sslLock )
{
if( sslContext != null || triedSslContext ) return sslContext;
try
{
return sslContext = SslContextBuilder
.forClient()
.trustManager( getTrustManager() )
.build();
}
catch( SSLException e )
{
ComputerCraft.log.error( "Cannot construct SSL context", e );
triedSslContext = true;
sslContext = null;
throw new HTTPRequestException( "Cannot create a secure connection" );
}
}
}
/**
* Checks a host is allowed
*
* @param host The domain to check against
* @throws HTTPRequestException If the host is not permitted.
*/
public static void checkHost( String host ) throws HTTPRequestException
{
if( !ComputerCraft.http_whitelist.matches( host ) || ComputerCraft.http_blacklist.matches( host ) )
{
throw new HTTPRequestException( "Domain not permitted" );
}
}
/**
* Create a {@link InetSocketAddress} from the resolved {@code host} and port.
*
* Note, this may require a DNS lookup, and so should not be executed on the main CC thread.
*
* @param host The host to resolve.
* @param port The port, or -1 if not defined.
* @param ssl Whether to connect with SSL. This is used to find the default port if not otherwise specified.
* @return The resolved address.
* @throws HTTPRequestException If the host is not permitted.
*/
public static InetSocketAddress getAddress( String host, int port, boolean ssl ) throws HTTPRequestException
{
if( port < 0 ) port = ssl ? 443 : 80;
InetSocketAddress socketAddress = new InetSocketAddress( host, port );
if( socketAddress.isUnresolved() ) throw new HTTPRequestException( "Unknown host" );
InetAddress address = socketAddress.getAddress();
if( !ComputerCraft.http_whitelist.matches( address ) || ComputerCraft.http_blacklist.matches( address ) )
{
throw new HTTPRequestException( "Domain not permitted" );
}
return socketAddress;
}
/**
* Read a {@link ByteBuf} into a byte array.
*
* @param buffer The buffer to read.
* @return The resulting bytes.
*/
public static byte[] toBytes( ByteBuf buffer )
{
byte[] bytes = new byte[buffer.readableBytes()];
buffer.readBytes( bytes );
return bytes;
}
}

145
src/main/java/dan200/computercraft/core/apis/http/Resource.java Normal file
View File

@@ -0,0 +1,145 @@
/*
* This file is part of ComputerCraft - http://www.computercraft.info
* Copyright Daniel Ratcliffe, 2011-2019. Do not distribute without permission.
* Send enquiries to dratcliffe@gmail.com
*/
package dan200.computercraft.core.apis.http;
import dan200.computercraft.shared.util.IoUtil;
import io.netty.channel.Channel;
import io.netty.channel.ChannelFuture;
import java.io.Closeable;
import java.lang.ref.Reference;
import java.lang.ref.ReferenceQueue;
import java.lang.ref.WeakReference;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
/**
* A holder for one or more resources, with a lifetime.
*/
public abstract class Resource<T extends Resource<T>> implements Closeable
{
private final AtomicBoolean closed = new AtomicBoolean( false );
private final ResourceQueue<T> limiter;
protected Resource( ResourceQueue<T> limiter )
{
this.limiter = limiter;
}
/**
* Whether this resource is closed.
*
* @return Whether this resource is closed.
*/
public final boolean isClosed()
{
return closed.get();
}
/**
* Checks if this has been cancelled. If so, it'll clean up any
* existing resources and cancel any pending futures.
*/
public final boolean checkClosed()
{
if( !closed.get() ) return false;
dispose();
return true;
}
/**
* Try to close the current resource.
*
* @return Whether this was successfully closed, or {@code false} if it has already been closed.
*/
protected final boolean tryClose()
{
if( closed.getAndSet( true ) ) return false;
dispose();
return true;
}
/**
* Clean up any pending resources
*
* Note, this may be called multiple times, and so should be thread-safe and
* avoid any major side effects.
*/
protected void dispose()
{
@SuppressWarnings( "unchecked" ) T thisT = (T) this;
limiter.release( thisT );
}
/**
* Create a {@link WeakReference} which will close {@code this} when collected.
*
* @param object The object to reference to
* @return The weak reference.
*/
protected <R> WeakReference<R> createOwnerReference( R object )
{
return new CloseReference<>( this, object );
}
@Override
public final void close()
{
tryClose();
}
public void queue( Consumer<T> task )
{
@SuppressWarnings( "unchecked" ) T thisT = (T) this;
limiter.queue( thisT, () -> task.accept( thisT ) );
}
protected static <T extends Closeable> T closeCloseable( T closeable )
{
if( closeable != null ) IoUtil.closeQuietly( closeable );
return null;
}
protected static ChannelFuture closeChannel( ChannelFuture future )
{
if( future != null )
{
future.cancel( false );
Channel channel = future.channel();
if( channel != null && channel.isOpen() ) channel.close();
}
return null;
}
protected static <T extends Future<?>> T closeFuture( T future )
{
if( future != null ) future.cancel( true );
return null;
}
private static final ReferenceQueue<Object> QUEUE = new ReferenceQueue<>();
private static class CloseReference<T> extends WeakReference<T>
{
final Resource<?> resource;
CloseReference( Resource<?> resource, T referent )
{
super( referent, QUEUE );
this.resource = resource;
}
}
public static void cleanup()
{
Reference<?> reference;
while( (reference = QUEUE.poll()) != null ) ((CloseReference) reference).resource.close();
}
}

92
src/main/java/dan200/computercraft/core/apis/http/ResourceQueue.java Normal file
View File

@@ -0,0 +1,92 @@
/*
* This file is part of ComputerCraft - http://www.computercraft.info
* Copyright Daniel Ratcliffe, 2011-2019. Do not distribute without permission.
* Send enquiries to dratcliffe@gmail.com
*/
package dan200.computercraft.core.apis.http;
import java.util.ArrayDeque;
import java.util.HashSet;
import java.util.Set;
import java.util.function.IntSupplier;
import java.util.function.Supplier;
/**
* A queue for {@link Resource}s, with built-in rate-limiting.
*/
public class ResourceQueue<T extends Resource<T>>
{
private static final IntSupplier ZERO = () -> 0;
private final IntSupplier limit;
private boolean active = false;
private final Set<T> resources = new HashSet<>();
private final ArrayDeque<Supplier<T>> pending = new ArrayDeque<>();
public ResourceQueue( IntSupplier limit )
{
this.limit = limit;
}
public ResourceQueue()
{
this.limit = ZERO;
}
public void startup()
{
active = true;
}
public synchronized void shutdown()
{
active = false;
pending.clear();
for( T resource : resources ) resource.close();
resources.clear();
Resource.cleanup();
}
public void queue( T resource, Runnable setup )
{
queue( () -> {
setup.run();
return resource;
} );
}
public synchronized void queue( Supplier<T> resource )
{
Resource.cleanup();
if( !active ) return;
int limit = this.limit.getAsInt();
if( limit <= 0 || resources.size() < limit )
{
resources.add( resource.get() );
}
else
{
pending.add( resource );
}
}
public synchronized void release( T resource )
{
if( !active ) return;
resources.remove( resource );
int limit = this.limit.getAsInt();
if( limit <= 0 || resources.size() < limit )
{
Supplier<T> next = pending.poll();
if( next != null ) resources.add( next.get() );
}
}
}

207
src/main/java/dan200/computercraft/core/apis/http/WebsocketConnection.java
View File

@@ -1,207 +0,0 @@
/*
* This file is part of ComputerCraft - http://www.computercraft.info
* Copyright Daniel Ratcliffe, 2011-2019. Do not distribute without permission.
* Send enquiries to dratcliffe@gmail.com
*/
package dan200.computercraft.core.apis.http;
import com.google.common.base.Objects;
import com.google.common.base.Strings;
import dan200.computercraft.api.lua.ILuaContext;
import dan200.computercraft.api.lua.ILuaObject;
import dan200.computercraft.api.lua.LuaException;
import dan200.computercraft.core.apis.HTTPAPI;
import dan200.computercraft.core.apis.IAPIEnvironment;
import dan200.computercraft.core.tracking.TrackingField;
import dan200.computercraft.shared.util.StringUtil;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.handler.codec.http.FullHttpResponse;
import io.netty.handler.codec.http.websocketx.*;
import io.netty.util.CharsetUtil;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.Closeable;
import java.util.Arrays;
import static dan200.computercraft.core.apis.ArgumentHelper.optBoolean;
public class WebsocketConnection extends SimpleChannelInboundHandler<Object> implements ILuaObject, Closeable
{
public static final String SUCCESS_EVENT = "websocket_success";
public static final String FAILURE_EVENT = "websocket_failure";
public static final String CLOSE_EVENT = "websocket_closed";
public static final String MESSAGE_EVENT = "websocket_message";
private final String url;
private final IAPIEnvironment computer;
private final HTTPAPI api;
private boolean open = true;
private Channel channel;
private final WebSocketClientHandshaker handshaker;
public WebsocketConnection( IAPIEnvironment computer, HTTPAPI api, WebSocketClientHandshaker handshaker, String url )
{
this.computer = computer;
this.api = api;
this.handshaker = handshaker;
this.url = url;
api.addCloseable( this );
}
private void close( boolean remove )
{
open = false;
if( remove ) api.removeCloseable( this );
if( channel != null )
{
channel.close();
channel = null;
}
}
@Override
public void close()
{
close( false );
}
private void onClosed( int status, String reason )
{
close( true );
computer.queueEvent( CLOSE_EVENT, new Object[] {
url,
Strings.isNullOrEmpty( reason ) ? null : reason,
status < 0 ? null : status,
} );
}
@Override
public void handlerAdded( ChannelHandlerContext ctx ) throws Exception
{
channel = ctx.channel();
super.handlerAdded( ctx );
}
@Override
public void channelActive( ChannelHandlerContext ctx ) throws Exception
{
handshaker.handshake( ctx.channel() );
super.channelActive( ctx );
}
@Override
public void channelInactive( ChannelHandlerContext ctx ) throws Exception
{
onClosed( -1, "Websocket is inactive" );
super.channelInactive( ctx );
}
@Override
public void channelRead0( ChannelHandlerContext ctx, Object msg )
{
Channel ch = ctx.channel();
if( !handshaker.isHandshakeComplete() )
{
handshaker.finishHandshake( ch, (FullHttpResponse) msg );
computer.queueEvent( SUCCESS_EVENT, new Object[] { url, this } );
return;
}
if( msg instanceof FullHttpResponse )
{
FullHttpResponse response = (FullHttpResponse) msg;
throw new IllegalStateException( "Unexpected FullHttpResponse (getStatus=" + response.status() + ", content=" + response.content().toString( CharsetUtil.UTF_8 ) + ')' );
}
WebSocketFrame frame = (WebSocketFrame) msg;
if( frame instanceof TextWebSocketFrame )
{
String data = ((TextWebSocketFrame) frame).text();
computer.addTrackingChange( TrackingField.WEBSOCKET_INCOMING, data.length() );
computer.queueEvent( MESSAGE_EVENT, new Object[] { url, data, false } );
}
else if( frame instanceof BinaryWebSocketFrame )
{
ByteBuf data = frame.content();
byte[] converted = new byte[data.readableBytes()];
data.readBytes( converted );
computer.addTrackingChange( TrackingField.WEBSOCKET_INCOMING, converted.length );
computer.queueEvent( MESSAGE_EVENT, new Object[] { url, converted, true } );
}
else if( frame instanceof CloseWebSocketFrame )
{
CloseWebSocketFrame closeFrame = (CloseWebSocketFrame) frame;
ch.close();
onClosed( closeFrame.statusCode(), closeFrame.reasonText() );
}
}
@Override
public void exceptionCaught( ChannelHandlerContext ctx, Throwable cause )
{
ctx.close();
computer.queueEvent( FAILURE_EVENT, new Object[] {
url,
cause instanceof WebSocketHandshakeException ? cause.getMessage() : "Could not connect"
} );
}
@Nonnull
@Override
public String[] getMethodNames()
{
return new String[] { "receive", "send", "close" };
}
@Nullable
@Override
public Object[] callMethod( @Nonnull ILuaContext context, int method, @Nonnull Object[] arguments ) throws LuaException, InterruptedException
{
switch( method )
{
case 0: // receive
while( true )
{
checkOpen();
Object[] event = context.pullEvent( MESSAGE_EVENT );
if( event.length >= 3 && Objects.equal( event[1], url ) )
{
return Arrays.copyOfRange( event, 2, event.length );
}
}
case 1: // send
{
checkOpen();
String text = arguments.length > 0 && arguments[0] != null ? arguments[0].toString() : "";
boolean binary = optBoolean( arguments, 1, false );
computer.addTrackingChange( TrackingField.WEBSOCKET_OUTGOING, text.length() );
channel.writeAndFlush( binary
? new BinaryWebSocketFrame( Unpooled.wrappedBuffer( StringUtil.encodeString( text ) ) )
: new TextWebSocketFrame( text ) );
return null;
}
case 2:
close( true );
return null;
default:
return null;
}
}
private void checkOpen() throws LuaException
{
if( !open ) throw new LuaException( "attempt to use a closed file" );
}
}

206
src/main/java/dan200/computercraft/core/apis/http/WebsocketConnector.java
View File

@@ -1,206 +0,0 @@
/*
* This file is part of ComputerCraft - http://www.computercraft.info
* Copyright Daniel Ratcliffe, 2011-2019. Do not distribute without permission.
* Send enquiries to dratcliffe@gmail.com
*/
package dan200.computercraft.core.apis.http;
import dan200.computercraft.ComputerCraft;
import dan200.computercraft.core.apis.HTTPAPI;
import dan200.computercraft.core.apis.IAPIEnvironment;
import io.netty.bootstrap.Bootstrap;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.http.DefaultHttpHeaders;
import io.netty.handler.codec.http.HttpClientCodec;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpObjectAggregator;
import io.netty.handler.codec.http.websocketx.WebSocketClientHandshaker;
import io.netty.handler.codec.http.websocketx.WebSocketClientHandshakerFactory;
import io.netty.handler.codec.http.websocketx.WebSocketVersion;
import io.netty.handler.codec.http.websocketx.extensions.compression.WebSocketClientCompressionHandler;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import javax.net.ssl.SSLException;
import javax.net.ssl.TrustManagerFactory;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.KeyStore;
import java.util.Map;
import java.util.concurrent.Future;
/**
* Provides functionality to verify and connect to a remote websocket.
*/
public final class WebsocketConnector
{
private static final Object lock = new Object();
private static TrustManagerFactory trustManager;
private WebsocketConnector()
{
}
private static TrustManagerFactory getTrustManager()
{
if( trustManager != null ) return trustManager;
synchronized( lock )
{
if( trustManager != null ) return trustManager;
TrustManagerFactory tmf = null;
try
{
tmf = TrustManagerFactory.getInstance( TrustManagerFactory.getDefaultAlgorithm() );
tmf.init( (KeyStore) null );
}
catch( Exception e )
{
ComputerCraft.log.error( "Cannot setup trust manager", e );
}
return trustManager = tmf;
}
}
public static URI checkURI( String address ) throws HTTPRequestException
{
URI uri = null;
try
{
uri = new URI( address );
}
catch( URISyntaxException ignored )
{
}
if( uri == null || uri.getHost() == null )
{
try
{
uri = new URI( "ws://" + address );
}
catch( URISyntaxException ignored )
{
}
}
if( uri == null || uri.getHost() == null ) throw new HTTPRequestException( "URL malformed" );
String scheme = uri.getScheme();
if( scheme == null )
{
try
{
uri = new URI( "ws://" + uri.toString() );
}
catch( URISyntaxException e )
{
throw new HTTPRequestException( "URL malformed" );
}
}
else if( !scheme.equalsIgnoreCase( "wss" ) && !scheme.equalsIgnoreCase( "ws" ) )
{
throw new HTTPRequestException( "Invalid scheme '" + scheme + "'" );
}
if( !ComputerCraft.http_whitelist.matches( uri.getHost() ) || ComputerCraft.http_blacklist.matches( uri.getHost() ) )
{
throw new HTTPRequestException( "Domain not permitted" );
}
return uri;
}
public static int getPort( URI uri ) throws HTTPRequestException
{
int port = uri.getPort();
if( port >= 0 ) return port;
String scheme = uri.getScheme();
if( scheme.equalsIgnoreCase( "ws" ) )
{
return 80;
}
else if( scheme.equalsIgnoreCase( "wss" ) )
{
return 443;
}
else
{
throw new HTTPRequestException( "Invalid scheme '" + scheme + "'" );
}
}
public static Future<?> createConnector( final IAPIEnvironment environment, final HTTPAPI api, final URI uri, final String address, final int port, final Map<String, String> headers )
{
return HTTPExecutor.EXECUTOR.submit( () -> {
InetAddress resolved;
try
{
resolved = HTTPRequest.checkHost( uri.getHost() );
}
catch( HTTPRequestException e )
{
environment.queueEvent( WebsocketConnection.FAILURE_EVENT, new Object[] { address, e.getMessage() } );
return;
}
InetSocketAddress socketAddress = new InetSocketAddress( resolved, uri.getPort() == -1 ? port : uri.getPort() );
final SslContext ssl;
if( uri.getScheme().equalsIgnoreCase( "wss" ) )
{
try
{
ssl = SslContextBuilder.forClient().trustManager( getTrustManager() ).build();
}
catch( SSLException e )
{
environment.queueEvent( WebsocketConnection.FAILURE_EVENT, new Object[] { address, "Cannot create secure socket" } );
return;
}
}
else
{
ssl = null;
}
HttpHeaders httpHeaders = new DefaultHttpHeaders();
for( Map.Entry<String, String> header : headers.entrySet() )
{
httpHeaders.add( header.getKey(), header.getValue() );
}
WebSocketClientHandshaker handshaker = WebSocketClientHandshakerFactory.newHandshaker( uri, WebSocketVersion.V13, null, true, httpHeaders );
final WebsocketConnection connection = new WebsocketConnection( environment, api, handshaker, address );
new Bootstrap()
.group( HTTPExecutor.LOOP_GROUP )
.channel( NioSocketChannel.class )
.handler( new ChannelInitializer<SocketChannel>()
{
@Override
protected void initChannel( SocketChannel ch ) throws Exception
{
ChannelPipeline p = ch.pipeline();
if( ssl != null ) p.addLast( ssl.newHandler( ch.alloc(), uri.getHost(), port ) );
p.addLast(
new HttpClientCodec(),
new HttpObjectAggregator( 8192 ),
WebSocketClientCompressionHandler.INSTANCE,
connection
);
}
} )
.remoteAddress( socketAddress )
.connect();
} );
}
}

270
src/main/java/dan200/computercraft/core/apis/http/request/HttpRequest.java Normal file
View File

@@ -0,0 +1,270 @@
/*
* This file is part of ComputerCraft - http://www.computercraft.info
* Copyright Daniel Ratcliffe, 2011-2019. Do not distribute without permission.
* Send enquiries to dratcliffe@gmail.com
*/
package dan200.computercraft.core.apis.http.request;
import dan200.computercraft.ComputerCraft;
import dan200.computercraft.api.lua.ILuaObject;
import dan200.computercraft.core.apis.IAPIEnvironment;
import dan200.computercraft.core.apis.http.HTTPRequestException;
import dan200.computercraft.core.apis.http.NetworkUtils;
import dan200.computercraft.core.apis.http.Resource;
import dan200.computercraft.core.apis.http.ResourceQueue;
import dan200.computercraft.core.tracking.TrackingField;
import io.netty.bootstrap.Bootstrap;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.ConnectTimeoutException;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.TooLongFrameException;
import io.netty.handler.codec.http.*;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.timeout.ReadTimeoutException;
import io.netty.handler.timeout.ReadTimeoutHandler;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Represents one or more
*/
public class HttpRequest extends Resource<HttpRequest>
{
private static final String SUCCESS_EVENT = "http_success";
private static final String FAILURE_EVENT = "http_failure";
private static final int MAX_REDIRECTS = 16;
private static final int TIMEOUT = 30000;
private Future<?> executorFuture;
private ChannelFuture connectFuture;
private HttpRequestHandler currentRequest;
private final IAPIEnvironment environment;
private final String address;
private final ByteBuf postBuffer;
private final HttpHeaders headers;
private final boolean binary;
final AtomicInteger redirects;
public HttpRequest( ResourceQueue<HttpRequest> limiter, IAPIEnvironment environment, String address, String postText, HttpHeaders headers, boolean binary, boolean followRedirects )
{
super( limiter );
this.environment = environment;
this.address = address;
this.postBuffer = postText != null
? Unpooled.wrappedBuffer( postText.getBytes( StandardCharsets.UTF_8 ) )
: Unpooled.buffer( 0 );
this.headers = headers;
this.binary = binary;
this.redirects = new AtomicInteger( followRedirects ? MAX_REDIRECTS : 0 );
if( postText != null )
{
if( !headers.contains( HttpHeaderNames.CONTENT_TYPE ) )
{
headers.set( HttpHeaderNames.CONTENT_TYPE, "application/x-www-form-urlencoded; charset=utf-8" );
}
if( !headers.contains( HttpHeaderNames.CONTENT_LENGTH ) )
{
headers.set( HttpHeaderNames.CONTENT_LENGTH, postBuffer.readableBytes() );
}
}
}
public IAPIEnvironment environment()
{
return environment;
}
public static URI checkUri( String address ) throws HTTPRequestException
{
URI url;
try
{
url = new URI( address );
}
catch( URISyntaxException e )
{
throw new HTTPRequestException( "URL malformed" );
}
checkUri( url );
return url;
}
public static void checkUri( URI url ) throws HTTPRequestException
{
// Validate the URL
if( url.getScheme() == null ) throw new HTTPRequestException( "Must specify http or https" );
String scheme = url.getScheme().toLowerCase( Locale.ROOT );
if( !scheme.equalsIgnoreCase( "http" ) && !scheme.equalsIgnoreCase( "https" ) )
{
throw new HTTPRequestException( "Invalid protocol '" + scheme + "'" );
}
NetworkUtils.checkHost( url.getHost() );
}
public void request( URI uri, HttpMethod method )
{
if( isClosed() ) return;
executorFuture = NetworkUtils.EXECUTOR.submit( () -> doRequest( uri, method ) );
checkClosed();
}
private void doRequest( URI uri, HttpMethod method )
{
// If we're cancelled, abort.
if( isClosed() ) return;
try
{
boolean ssl = uri.getScheme().equalsIgnoreCase( "https" );
InetSocketAddress socketAddress = NetworkUtils.getAddress( uri.getHost(), uri.getPort(), ssl );
SslContext sslContext = ssl ? NetworkUtils.getSslContext() : null;
// getAddress may have a slight delay, so let's perform another cancellation check.
if( isClosed() ) return;
// Add request size to the tracker before opening the connection
environment.addTrackingChange( TrackingField.HTTP_REQUESTS, 1 );
environment.addTrackingChange( TrackingField.HTTP_UPLOAD, getHeaderSize( headers ) + postBuffer.capacity() );
HttpRequestHandler handler = currentRequest = new HttpRequestHandler( this, uri, method );
connectFuture = new Bootstrap()
.group( NetworkUtils.LOOP_GROUP )
.channelFactory( NioSocketChannel::new )
.handler( new ChannelInitializer<SocketChannel>()
{
@Override
protected void initChannel( SocketChannel ch )
{
ch.config().setConnectTimeoutMillis( TIMEOUT );
ChannelPipeline p = ch.pipeline();
if( sslContext != null )
{
p.addLast( sslContext.newHandler( ch.alloc(), uri.getHost(), socketAddress.getPort() ) );
}
p.addLast(
new ReadTimeoutHandler( TIMEOUT, TimeUnit.MILLISECONDS ),
new HttpClientCodec(),
new HttpContentDecompressor(),
handler
);
}
} )
.remoteAddress( socketAddress )
.connect()
.addListener( c -> {
if( !c.isSuccess() ) failure( c.cause() );
} );
// Do an additional check for cancellation
checkClosed();
}
catch( HTTPRequestException e )
{
failure( e.getMessage() );
}
catch( Exception e )
{
failure( "Could not connect" );
if( ComputerCraft.logPeripheralErrors ) ComputerCraft.log.error( "Error in HTTP request", e );
}
}
void failure( String message )
{
if( tryClose() ) environment.queueEvent( FAILURE_EVENT, new Object[] { address, message } );
}
void failure( Throwable cause )
{
String message;
if( cause instanceof HTTPRequestException )
{
message = cause.getMessage();
}
else if( cause instanceof TooLongFrameException )
{
message = "Response is too large";
}
else if( cause instanceof ReadTimeoutException || cause instanceof ConnectTimeoutException )
{
message = "Timed out";
}
else
{
message = "Could not connect";
}
failure( message );
}
void failure( String message, ILuaObject object )
{
if( tryClose() ) environment.queueEvent( FAILURE_EVENT, new Object[] { address, message, object } );
}
void success( ILuaObject object )
{
if( tryClose() ) environment.queueEvent( SUCCESS_EVENT, new Object[] { address, object } );
}
protected void dispose()
{
super.dispose();
executorFuture = closeFuture( executorFuture );
connectFuture = closeChannel( connectFuture );
currentRequest = closeCloseable( currentRequest );
}
public static long getHeaderSize( HttpHeaders headers )
{
long size = 0;
for( Map.Entry<String, String> header : headers )
{
size += header.getKey() == null ? 0 : header.getKey().length();
size += header.getValue() == null ? 0 : header.getValue().length() + 1;
}
return size;
}
public ByteBuf body()
{
return postBuffer;
}
public HttpHeaders headers()
{
return headers;
}
public boolean isBinary()
{
return binary;
}
}

259
src/main/java/dan200/computercraft/core/apis/http/request/HttpRequestHandler.java Normal file
View File

@@ -0,0 +1,259 @@
/*
* This file is part of ComputerCraft - http://www.computercraft.info
* Copyright Daniel Ratcliffe, 2011-2019. Do not distribute without permission.
* Send enquiries to dratcliffe@gmail.com
*/
package dan200.computercraft.core.apis.http.request;
import dan200.computercraft.ComputerCraft;
import dan200.computercraft.api.lua.ILuaObject;
import dan200.computercraft.core.apis.handles.ArrayByteChannel;
import dan200.computercraft.core.apis.handles.BinaryReadableHandle;
import dan200.computercraft.core.apis.handles.EncodedReadableHandle;
import dan200.computercraft.core.apis.http.HTTPRequestException;
import dan200.computercraft.core.apis.http.NetworkUtils;
import dan200.computercraft.core.tracking.TrackingField;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.CompositeByteBuf;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.handler.codec.http.*;
import java.io.Closeable;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import static dan200.computercraft.core.apis.http.request.HttpRequest.getHeaderSize;
public final class HttpRequestHandler extends SimpleChannelInboundHandler<HttpObject> implements Closeable
{
/**
* Same as {@link io.netty.handler.codec.MessageAggregator}.
*/
private static final int DEFAULT_MAX_COMPOSITEBUFFER_COMPONENTS = 1024;
private static final byte[] EMPTY_BYTES = new byte[0];
private final HttpRequest request;
private boolean closed = false;
private final URI uri;
private final HttpMethod method;
private Charset responseCharset;
private final HttpHeaders responseHeaders = new DefaultHttpHeaders();
private HttpResponseStatus responseStatus;
private CompositeByteBuf responseBody;
HttpRequestHandler( HttpRequest request, URI uri, HttpMethod method )
{
this.request = request;
this.uri = uri;
this.method = method;
}
@Override
public void channelActive( ChannelHandlerContext ctx ) throws Exception
{
if( request.checkClosed() ) return;
ByteBuf body = request.body();
body.resetReaderIndex().retain();
String requestUri = uri.getRawPath();
if( uri.getRawQuery() != null ) requestUri += "?" + uri.getRawQuery();
FullHttpRequest request = new DefaultFullHttpRequest( HttpVersion.HTTP_1_1, HttpMethod.GET, requestUri, body );
request.setMethod( method );
request.headers().set( this.request.headers() );
// We force some headers to be always applied
if( !request.headers().contains( HttpHeaderNames.ACCEPT_CHARSET ) )
{
request.headers().set( HttpHeaderNames.ACCEPT_CHARSET, "UTF-8" );
}
if( !request.headers().contains( HttpHeaderNames.USER_AGENT ) )
{
request.headers().set( HttpHeaderNames.USER_AGENT, ComputerCraft.MOD_ID + "/" + ComputerCraft.getVersion() );
}
request.headers().set( HttpHeaderNames.HOST, uri.getHost() );
request.headers().set( HttpHeaderNames.CONNECTION, HttpHeaderValues.CLOSE );
ctx.channel().writeAndFlush( request );
super.channelActive( ctx );
}
@Override
public void channelInactive( ChannelHandlerContext ctx ) throws Exception
{
if( !closed ) request.failure( "Could not connect" );
super.channelInactive( ctx );
}
@Override
public void channelRead0( ChannelHandlerContext ctx, HttpObject message )
{
if( closed || request.checkClosed() ) return;
if( message instanceof HttpResponse )
{
HttpResponse response = (HttpResponse) message;
if( request.redirects.get() > 0 )
{
URI redirect = getRedirect( response.status(), response.headers() );
if( redirect != null && !uri.equals( redirect ) && request.redirects.getAndDecrement() > 0 )
{
// If we have a redirect, and don't end up at the same place, then follow it.
// We mark ourselves as disposed first though, to avoid firing events when the channel
// becomes inactive or disposed.
closed = true;
ctx.close();
try
{
HttpRequest.checkUri( redirect );
}
catch( HTTPRequestException e )
{
// If we cannot visit this uri, then fail.
request.failure( e.getMessage() );
return;
}
request.request( redirect, response.status().code() == 303 ? HttpMethod.GET : method );
return;
}
}
responseCharset = HttpUtil.getCharset( response, StandardCharsets.UTF_8 );
responseStatus = response.status();
responseHeaders.add( response.headers() );
}
if( message instanceof HttpContent )
{
HttpContent content = (HttpContent) message;
if( responseBody == null )
{
responseBody = ctx.alloc().compositeBuffer( DEFAULT_MAX_COMPOSITEBUFFER_COMPONENTS );
}
ByteBuf partial = content.content();
if( partial.isReadable() )
{
// If we've read more than we're allowed to handle, abort as soon as possible.
if( ComputerCraft.httpMaxDownload != 0 && responseBody.readableBytes() + partial.readableBytes() > ComputerCraft.httpMaxDownload )
{
closed = true;
ctx.close();
request.failure( "Response is too large" );
return;
}
responseBody.addComponent( true, partial.retain() );
}
if( message instanceof LastHttpContent )
{
LastHttpContent last = (LastHttpContent) message;
responseHeaders.add( last.trailingHeaders() );
// Set the content length, if not already given.
if( responseHeaders.contains( HttpHeaderNames.CONTENT_LENGTH ) )
{
responseHeaders.set( HttpHeaderNames.CONTENT_LENGTH, responseBody.readableBytes() );
}
ctx.close();
sendResponse();
}
}
}
@Override
public void exceptionCaught( ChannelHandlerContext ctx, Throwable cause )
{
if( ComputerCraft.logPeripheralErrors ) ComputerCraft.log.error( "Error handling HTTP response", cause );
request.failure( cause );
}
private void sendResponse()
{
// Read the ByteBuf into a channel.
CompositeByteBuf body = responseBody;
byte[] bytes = body == null ? EMPTY_BYTES : NetworkUtils.toBytes( body );
// Decode the headers
HttpResponseStatus status = responseStatus;
Map<String, String> headers = new HashMap<>();
for( Map.Entry<String, String> header : responseHeaders )
{
String existing = headers.get( header.getKey() );
headers.put( header.getKey(), existing == null ? header.getValue() : existing + "," + header.getValue() );
}
// Fire off a stats event
request.environment().addTrackingChange( TrackingField.HTTP_DOWNLOAD, getHeaderSize( responseHeaders ) + bytes.length );
// Prepare to queue an event
ArrayByteChannel contents = new ArrayByteChannel( bytes );
final ILuaObject reader = request.isBinary()
? new BinaryReadableHandle( contents )
: new EncodedReadableHandle( EncodedReadableHandle.open( contents, responseCharset ) );
ILuaObject stream = new HttpResponseHandle( reader, status.code(), status.reasonPhrase(), headers );
if( status.code() >= 200 && status.code() < 400 )
{
request.success( stream );
}
else
{
request.failure( status.reasonPhrase(), stream );
}
}
/**
* Determine the redirect from this response
*/
private URI getRedirect( HttpResponseStatus status, HttpHeaders headers )
{
int code = status.code();
if( code < 300 || code > 307 || code == 304 || code == 306 ) return null;
String location = headers.get( HttpHeaderNames.LOCATION );
if( location == null ) return null;
try
{
return uri.resolve( new URI( URLDecoder.decode( location, "UTF-8" ) ) );
}
catch( UnsupportedEncodingException | IllegalArgumentException | URISyntaxException e )
{
return null;
}
}
@Override
public void close()
{
closed = true;
if( responseBody != null )
{
responseBody.release();
responseBody = null;
}
}
}

67
src/main/java/dan200/computercraft/core/apis/http/request/HttpResponseHandle.java Normal file
View File

@@ -0,0 +1,67 @@
/*
* This file is part of ComputerCraft - http://www.computercraft.info
* Copyright Daniel Ratcliffe, 2011-2019. Do not distribute without permission.
* Send enquiries to dratcliffe@gmail.com
*/
package dan200.computercraft.core.apis.http.request;
import dan200.computercraft.api.lua.ILuaContext;
import dan200.computercraft.api.lua.ILuaObject;
import dan200.computercraft.api.lua.LuaException;
import javax.annotation.Nonnull;
import java.util.Arrays;
import java.util.Map;
/**
* Wraps a {@link dan200.computercraft.core.apis.handles.HandleGeneric} and provides additional methods for
* getting the response code and headers.
*/
public class HttpResponseHandle implements ILuaObject
{
private final String[] newMethods;
private final int methodOffset;
private final ILuaObject reader;
private final int responseCode;
private final String responseStatus;
private final Map<String, String> responseHeaders;
public HttpResponseHandle(@Nonnull ILuaObject reader, int responseCode, String responseStatus, @Nonnull Map<String, String> responseHeaders)
{
this.reader = reader;
this.responseCode = responseCode;
this.responseStatus = responseStatus;
this.responseHeaders = responseHeaders;
String[] oldMethods = reader.getMethodNames();
final int methodOffset = this.methodOffset = oldMethods.length;
final String[] newMethods = this.newMethods = Arrays.copyOf( oldMethods, oldMethods.length + 2 );
newMethods[methodOffset + 0] = "getResponseCode";
newMethods[methodOffset + 1] = "getResponseHeaders";
}
@Nonnull
@Override
public String[] getMethodNames()
{
return newMethods;
}
@Override
public Object[] callMethod( @Nonnull ILuaContext context, int method, @Nonnull Object[] args ) throws LuaException, InterruptedException
{
if( method < methodOffset ) return reader.callMethod( context, method, args );
switch( method - methodOffset )
{
case 0: // getResponseCode
return new Object[] { responseCode, responseStatus };
case 1: // getResponseHeaders
return new Object[] { responseHeaders };
default:
return null;
}
}
}

231
src/main/java/dan200/computercraft/core/apis/http/websocket/Websocket.java Normal file
View File

@@ -0,0 +1,231 @@
/*
* This file is part of ComputerCraft - http://www.computercraft.info
* Copyright Daniel Ratcliffe, 2011-2019. Do not distribute without permission.
* Send enquiries to dratcliffe@gmail.com
*/
package dan200.computercraft.core.apis.http.websocket;
import com.google.common.base.Strings;
import dan200.computercraft.ComputerCraft;
import dan200.computercraft.core.apis.IAPIEnvironment;
import dan200.computercraft.core.apis.http.HTTPRequestException;
import dan200.computercraft.core.apis.http.NetworkUtils;
import dan200.computercraft.core.apis.http.Resource;
import dan200.computercraft.core.apis.http.ResourceQueue;
import dan200.computercraft.shared.util.IoUtil;
import io.netty.bootstrap.Bootstrap;
import io.netty.channel.Channel;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.http.HttpClientCodec;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpObjectAggregator;
import io.netty.handler.codec.http.websocketx.WebSocketClientHandshaker;
import io.netty.handler.codec.http.websocketx.WebSocketClientHandshakerFactory;
import io.netty.handler.codec.http.websocketx.WebSocketVersion;
import io.netty.handler.codec.http.websocketx.extensions.compression.WebSocketClientCompressionHandler;
import io.netty.handler.ssl.SslContext;
import java.lang.ref.WeakReference;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.concurrent.Future;
/**
* Provides functionality to verify and connect to a remote websocket.
*/
public class Websocket extends Resource<Websocket>
{
public static final int MAX_MESSAGE_SIZE = 64 * 1024;
static final String SUCCESS_EVENT = "websocket_success";
static final String FAILURE_EVENT = "websocket_failure";
static final String CLOSE_EVENT = "websocket_closed";
static final String MESSAGE_EVENT = "websocket_message";
private Future<?> executorFuture;
private ChannelFuture connectFuture;
private WeakReference<WebsocketHandle> websocketHandle;
private final IAPIEnvironment environment;
private final URI uri;
private final String address;
private final HttpHeaders headers;
public Websocket( ResourceQueue<Websocket> limiter, IAPIEnvironment environment, URI uri, String address, HttpHeaders headers )
{
super( limiter );
this.environment = environment;
this.uri = uri;
this.address = address;
this.headers = headers;
}
public static URI checkUri( String address ) throws HTTPRequestException
{
URI uri = null;
try
{
uri = new URI( address );
}
catch( URISyntaxException ignored )
{
}
if( uri == null || uri.getHost() == null )
{
try
{
uri = new URI( "ws://" + address );
}
catch( URISyntaxException ignored )
{
}
}
if( uri == null || uri.getHost() == null ) throw new HTTPRequestException( "URL malformed" );
String scheme = uri.getScheme();
if( scheme == null )
{
try
{
uri = new URI( "ws://" + uri.toString() );
}
catch( URISyntaxException e )
{
throw new HTTPRequestException( "URL malformed" );
}
}
else if( !scheme.equalsIgnoreCase( "wss" ) && !scheme.equalsIgnoreCase( "ws" ) )
{
throw new HTTPRequestException( "Invalid scheme '" + scheme + "'" );
}
NetworkUtils.checkHost( uri.getHost() );
return uri;
}
public void connect()
{
if( isClosed() ) return;
executorFuture = NetworkUtils.EXECUTOR.submit( this::doConnect );
checkClosed();
}
private void doConnect()
{
// If we're cancelled, abort.
if( isClosed() ) return;
try
{
boolean ssl = uri.getScheme().equalsIgnoreCase( "wss" );
InetSocketAddress socketAddress = NetworkUtils.getAddress( uri.getHost(), uri.getPort(), ssl );
SslContext sslContext = ssl ? NetworkUtils.getSslContext() : null;
// getAddress may have a slight delay, so let's perform another cancellation check.
if( isClosed() ) return;
connectFuture = new Bootstrap()
.group( NetworkUtils.LOOP_GROUP )
.channel( NioSocketChannel.class )
.handler( new ChannelInitializer<SocketChannel>()
{
@Override
protected void initChannel( SocketChannel ch )
{
ChannelPipeline p = ch.pipeline();
if( sslContext != null )
{
p.addLast( sslContext.newHandler( ch.alloc(), uri.getHost(), socketAddress.getPort() ) );
}
WebSocketClientHandshaker handshaker = WebSocketClientHandshakerFactory.newHandshaker(
uri, WebSocketVersion.V13, null, true, headers,
ComputerCraft.httpMaxWebsocketMessage == 0 ? MAX_MESSAGE_SIZE : ComputerCraft.httpMaxWebsocketMessage
);
p.addLast(
new HttpClientCodec(),
new HttpObjectAggregator( 8192 ),
WebSocketClientCompressionHandler.INSTANCE,
new WebsocketHandler( Websocket.this, handshaker )
);
}
} )
.remoteAddress( socketAddress )
.connect();
// Do an additional check for cancellation
checkClosed();
}
catch( HTTPRequestException e )
{
failure( e.getMessage() );
}
catch( Exception e )
{
failure( "Could not connect" );
if( ComputerCraft.logPeripheralErrors ) ComputerCraft.log.error( "Error in websocket", e );
}
}
void success( Channel channel )
{
if( isClosed() ) return;
WebsocketHandle handle = new WebsocketHandle( this, channel );
environment().queueEvent( SUCCESS_EVENT, new Object[] { address, handle } );
this.websocketHandle = createOwnerReference( handle );
checkClosed();
}
void failure( String message )
{
if( tryClose() ) environment.queueEvent( FAILURE_EVENT, new Object[] { address, message } );
}
void close( int status, String reason )
{
if( tryClose() )
{
environment.queueEvent( CLOSE_EVENT, new Object[] {
address,
Strings.isNullOrEmpty( reason ) ? null : reason,
status < 0 ? null : status,
} );
}
}
@Override
protected void dispose()
{
super.dispose();
executorFuture = closeFuture( executorFuture );
connectFuture = closeChannel( connectFuture );
WeakReference<WebsocketHandle> websocketHandleRef = this.websocketHandle;
WebsocketHandle websocketHandle = websocketHandleRef == null ? null : websocketHandleRef.get();
if( websocketHandle != null ) IoUtil.closeQuietly( websocketHandle );
this.websocketHandle = null;
}
public IAPIEnvironment environment()
{
return environment;
}
public String address()
{
return address;
}
}

117
src/main/java/dan200/computercraft/core/apis/http/websocket/WebsocketHandle.java Normal file
View File

@@ -0,0 +1,117 @@
/*
* This file is part of ComputerCraft - http://www.computercraft.info
* Copyright Daniel Ratcliffe, 2011-2019. Do not distribute without permission.
* Send enquiries to dratcliffe@gmail.com
*/
package dan200.computercraft.core.apis.http.websocket;
import com.google.common.base.Objects;
import dan200.computercraft.ComputerCraft;
import dan200.computercraft.api.lua.ILuaContext;
import dan200.computercraft.api.lua.ILuaObject;
import dan200.computercraft.api.lua.LuaException;
import dan200.computercraft.core.tracking.TrackingField;
import dan200.computercraft.shared.util.StringUtil;
import io.netty.buffer.Unpooled;
import io.netty.channel.Channel;
import io.netty.handler.codec.http.websocketx.BinaryWebSocketFrame;
import io.netty.handler.codec.http.websocketx.TextWebSocketFrame;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.Closeable;
import java.util.Arrays;
import static dan200.computercraft.core.apis.ArgumentHelper.optBoolean;
import static dan200.computercraft.core.apis.http.websocket.Websocket.MESSAGE_EVENT;
public class WebsocketHandle implements ILuaObject, Closeable
{
private final Websocket websocket;
private boolean closed = false;
private Channel channel;
public WebsocketHandle( Websocket websocket, Channel channel )
{
this.websocket = websocket;
this.channel = channel;
}
@Nonnull
@Override
public String[] getMethodNames()
{
return new String[] { "receive", "send", "close" };
}
@Nullable
@Override
public Object[] callMethod( @Nonnull ILuaContext context, int method, @Nonnull Object[] arguments ) throws LuaException, InterruptedException
{
switch( method )
{
case 0: // receive
while( true )
{
checkOpen();
Object[] event = context.pullEvent( MESSAGE_EVENT );
if( event.length >= 3 && Objects.equal( event[1], websocket.address() ) )
{
return Arrays.copyOfRange( event, 2, event.length );
}
}
case 1: // send
{
checkOpen();
String text = arguments.length > 0 && arguments[0] != null ? arguments[0].toString() : "";
if( ComputerCraft.httpMaxWebsocketMessage != 0 && text.length() > ComputerCraft.httpMaxWebsocketMessage )
{
throw new LuaException( "Message is too large" );
}
boolean binary = optBoolean( arguments, 1, false );
websocket.environment().addTrackingChange( TrackingField.WEBSOCKET_OUTGOING, text.length() );
Channel channel = this.channel;
if( channel != null )
{
channel.writeAndFlush( binary
? new BinaryWebSocketFrame( Unpooled.wrappedBuffer( StringUtil.encodeString( text ) ) )
: new TextWebSocketFrame( text ) );
}
return null;
}
case 2: // close
close();
websocket.close();
return null;
default:
return null;
}
}
private void checkOpen() throws LuaException
{
if( closed ) throw new LuaException( "attempt to use a closed file" );
}
@Override
public void close()
{
closed = true;
Channel channel = this.channel;
if( channel != null )
{
channel.close();
this.channel = null;
}
}
}

113
src/main/java/dan200/computercraft/core/apis/http/websocket/WebsocketHandler.java Normal file
View File

@@ -0,0 +1,113 @@
/*
* This file is part of ComputerCraft - http://www.computercraft.info
* Copyright Daniel Ratcliffe, 2011-2019. Do not distribute without permission.
* Send enquiries to dratcliffe@gmail.com
*/
package dan200.computercraft.core.apis.http.websocket;
import dan200.computercraft.core.apis.http.HTTPRequestException;
import dan200.computercraft.core.apis.http.NetworkUtils;
import dan200.computercraft.core.tracking.TrackingField;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ConnectTimeoutException;
import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.handler.codec.TooLongFrameException;
import io.netty.handler.codec.http.FullHttpResponse;
import io.netty.handler.codec.http.websocketx.*;
import io.netty.handler.timeout.ReadTimeoutException;
import io.netty.util.CharsetUtil;
import static dan200.computercraft.core.apis.http.websocket.Websocket.MESSAGE_EVENT;
public class WebsocketHandler extends SimpleChannelInboundHandler<Object>
{
private final Websocket websocket;
private final WebSocketClientHandshaker handshaker;
public WebsocketHandler( Websocket websocket, WebSocketClientHandshaker handshaker )
{
this.handshaker = handshaker;
this.websocket = websocket;
}
@Override
public void channelActive( ChannelHandlerContext ctx ) throws Exception
{
handshaker.handshake( ctx.channel() );
super.channelActive( ctx );
}
@Override
public void channelInactive( ChannelHandlerContext ctx ) throws Exception
{
websocket.close( -1, "Websocket is inactive" );
super.channelInactive( ctx );
}
@Override
public void channelRead0( ChannelHandlerContext ctx, Object msg )
{
if( websocket.isClosed() ) return;
if( !handshaker.isHandshakeComplete() )
{
handshaker.finishHandshake( ctx.channel(), (FullHttpResponse) msg );
websocket.success( ctx.channel() );
return;
}
if( msg instanceof FullHttpResponse )
{
FullHttpResponse response = (FullHttpResponse) msg;
throw new IllegalStateException( "Unexpected FullHttpResponse (getStatus=" + response.status() + ", content=" + response.content().toString( CharsetUtil.UTF_8 ) + ')' );
}
WebSocketFrame frame = (WebSocketFrame) msg;
if( frame instanceof TextWebSocketFrame )
{
String data = ((TextWebSocketFrame) frame).text();
websocket.environment().addTrackingChange( TrackingField.WEBSOCKET_INCOMING, data.length() );
websocket.environment().queueEvent( MESSAGE_EVENT, new Object[] { websocket.address(), data, false } );
}
else if( frame instanceof BinaryWebSocketFrame )
{
byte[] converted = NetworkUtils.toBytes( frame.content() );
websocket.environment().addTrackingChange( TrackingField.WEBSOCKET_INCOMING, converted.length );
websocket.environment().queueEvent( MESSAGE_EVENT, new Object[] { websocket.address(), converted, true } );
}
else if( frame instanceof CloseWebSocketFrame )
{
CloseWebSocketFrame closeFrame = (CloseWebSocketFrame) frame;
websocket.close( closeFrame.statusCode(), closeFrame.reasonText() );
}
}
@Override
public void exceptionCaught( ChannelHandlerContext ctx, Throwable cause )
{
ctx.close();
String message;
if( cause instanceof WebSocketHandshakeException || cause instanceof HTTPRequestException )
{
message = cause.getMessage();
}
else if( cause instanceof TooLongFrameException )
{
message = "Message is too large";
}
else if( cause instanceof ReadTimeoutException || cause instanceof ConnectTimeoutException )
{
message = "Timed out";
}
else
{
message = "Could not connect";
}
websocket.failure( message );
}
}

18
src/main/java/dan200/computercraft/core/filesystem/FileSystem.java
View File

@@ -11,6 +11,7 @@ import dan200.computercraft.ComputerCraft;
import dan200.computercraft.api.filesystem.IFileSystem; import dan200.computercraft.api.filesystem.IFileSystem;
import dan200.computercraft.api.filesystem.IMount; import dan200.computercraft.api.filesystem.IMount;
import dan200.computercraft.api.filesystem.IWritableMount; import dan200.computercraft.api.filesystem.IWritableMount;
import dan200.computercraft.shared.util.IoUtil;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.Closeable; import java.io.Closeable;
@@ -334,7 +335,7 @@ public class FileSystem
// Close all dangling open files // Close all dangling open files
synchronized( m_openFiles ) synchronized( m_openFiles )
{ {
for( Closeable file : m_openFiles.values() ) closeQuietly( file ); for( Closeable file : m_openFiles.values() ) IoUtil.closeQuietly( file );
m_openFiles.clear(); m_openFiles.clear();
while( m_openFileQueue.poll() != null ) ; while( m_openFileQueue.poll() != null ) ;
} }
@@ -655,7 +656,7 @@ public class FileSystem
while( (ref = m_openFileQueue.poll()) != null ) while( (ref = m_openFileQueue.poll()) != null )
{ {
Closeable file = m_openFiles.remove( ref ); Closeable file = m_openFiles.remove( ref );
if( file != null ) closeQuietly( file ); if( file != null ) IoUtil.closeQuietly( file );
} }
} }
} }
@@ -667,7 +668,7 @@ public class FileSystem
if( ComputerCraft.maximumFilesOpen > 0 && if( ComputerCraft.maximumFilesOpen > 0 &&
m_openFiles.size() >= ComputerCraft.maximumFilesOpen ) m_openFiles.size() >= ComputerCraft.maximumFilesOpen )
{ {
closeQuietly( file ); IoUtil.closeQuietly( file );
throw new FileSystemException( "Too many files already open" ); throw new FileSystemException( "Too many files already open" );
} }
@@ -881,15 +882,4 @@ public class FileSystem
return local; return local;
} }
} }
private static void closeQuietly( Closeable c )
{
try
{
c.close();
}
catch( IOException ignored )
{
}
}
} }

59
src/main/java/dan200/computercraft/shared/Config.java
View File

@@ -11,6 +11,7 @@ import com.google.common.base.Converter;
import dan200.computercraft.ComputerCraft; import dan200.computercraft.ComputerCraft;
import dan200.computercraft.api.turtle.event.TurtleAction; import dan200.computercraft.api.turtle.event.TurtleAction;
import dan200.computercraft.core.apis.AddressPredicate; import dan200.computercraft.core.apis.AddressPredicate;
import dan200.computercraft.core.apis.http.websocket.Websocket;
import net.minecraftforge.common.config.ConfigCategory; import net.minecraftforge.common.config.ConfigCategory;
import net.minecraftforge.common.config.ConfigElement; import net.minecraftforge.common.config.ConfigElement;
import net.minecraftforge.common.config.Configuration; import net.minecraftforge.common.config.Configuration;
@@ -35,11 +36,6 @@ public class Config
private static Configuration config; private static Configuration config;
private static Property httpEnable;
private static Property httpWebsocketEnable;
private static Property httpWhitelist;
private static Property httpBlacklist;
private static Property computerSpaceLimit; private static Property computerSpaceLimit;
private static Property floppySpaceLimit; private static Property floppySpaceLimit;
private static Property maximumFilesOpen; private static Property maximumFilesOpen;
@@ -49,12 +45,16 @@ public class Config
private static Property computerThreads; private static Property computerThreads;
private static Property logComputerErrors; private static Property logComputerErrors;
private static Property turtlesNeedFuel; private static Property httpEnable;
private static Property turtleFuelLimit; private static Property httpWebsocketEnable;
private static Property advancedTurtleFuelLimit; private static Property httpWhitelist;
private static Property turtlesObeyBlockProtection; private static Property httpBlacklist;
private static Property turtlesCanPush;
private static Property turtleDisabledActions; private static Property httpMaxRequests;
private static Property httpMaxDownload;
private static Property httpMaxUpload;
private static Property httpMaxWebsockets;
private static Property httpMaxWebsocketMessage;
private static Property commandBlockEnabled; private static Property commandBlockEnabled;
private static Property modemRange; private static Property modemRange;
@@ -63,6 +63,13 @@ public class Config
private static Property modemHighAltitudeRangeDuringStorm; private static Property modemHighAltitudeRangeDuringStorm;
private static Property maxNotesPerTick; private static Property maxNotesPerTick;
private static Property turtlesNeedFuel;
private static Property turtleFuelLimit;
private static Property advancedTurtleFuelLimit;
private static Property turtlesObeyBlockProtection;
private static Property turtlesCanPush;
private static Property turtleDisabledActions;
public static void load( File configFile ) public static void load( File configFile )
{ {
config = new Configuration( configFile ); config = new Configuration( configFile );
@@ -135,9 +142,31 @@ public class Config
"If this is empty then all whitelisted domains will be accessible. Example: \"*.github.com\" will block access to all subdomains of github.com.\n" + "If this is empty then all whitelisted domains will be accessible. Example: \"*.github.com\" will block access to all subdomains of github.com.\n" +
"You can use domain names (\"pastebin.com\"), wilcards (\"*.pastebin.com\") or CIDR notation (\"127.0.0.0/8\")." ); "You can use domain names (\"pastebin.com\"), wilcards (\"*.pastebin.com\") or CIDR notation (\"127.0.0.0/8\")." );
httpMaxRequests = config.get( CATEGORY_HTTP, "max_requests", ComputerCraft.httpMaxRequests );
httpMaxRequests.setComment( "The number of http requests a computer can make at one time. Additional requests will be queued, and sent when the running requests have finished. Set to 0 for unlimited." );
httpMaxRequests.setMinValue( 0 );
httpMaxDownload = config.get( CATEGORY_HTTP, "max_download", (int) ComputerCraft.httpMaxDownload );
httpMaxDownload.setComment( "The maximum size (in bytes) that a computer can download in a single request. Note that responses may receive more data than allowed, but this data will not be returned to the client." );
httpMaxDownload.setMinValue( 0 );
httpMaxUpload = config.get( CATEGORY_HTTP, "max_upload", (int) ComputerCraft.httpMaxUpload );
httpMaxUpload.setComment( "The maximum size (in bytes) that a computer can upload in a single request. This includes headers and POST text." );
httpMaxUpload.setMinValue( 0 );
httpMaxWebsockets = config.get( CATEGORY_HTTP, "max_websockets", ComputerCraft.httpMaxWebsockets );
httpMaxWebsockets.setComment( "The number of websockets a computer can have open at one time. Set to 0 for unlimited." );
httpMaxWebsockets.setMinValue( 1 );
httpMaxWebsocketMessage = config.get( CATEGORY_HTTP, "max_websocket_message", ComputerCraft.httpMaxWebsocketMessage );
httpMaxWebsocketMessage.setComment( "The maximum size (in bytes) that a computer can send or receive in one websocket packet." );
httpMaxWebsocketMessage.setMinValue( 0 );
httpMaxWebsocketMessage.setMaxValue( Websocket.MAX_MESSAGE_SIZE );
setOrder( setOrder(
CATEGORY_HTTP, CATEGORY_HTTP,
httpEnable, httpWebsocketEnable, httpWhitelist, httpBlacklist httpEnable, httpWebsocketEnable, httpWhitelist, httpBlacklist,
httpMaxRequests, httpMaxDownload, httpMaxUpload, httpMaxWebsockets, httpMaxWebsocketMessage
); );
} }
@@ -286,6 +315,12 @@ public class Config
ComputerCraft.http_whitelist = new AddressPredicate( httpWhitelist.getStringList() ); ComputerCraft.http_whitelist = new AddressPredicate( httpWhitelist.getStringList() );
ComputerCraft.http_blacklist = new AddressPredicate( httpBlacklist.getStringList() ); ComputerCraft.http_blacklist = new AddressPredicate( httpBlacklist.getStringList() );
ComputerCraft.httpMaxRequests = Math.max( 1, httpMaxRequests.getInt() );
ComputerCraft.httpMaxDownload = Math.max( 0, httpMaxDownload.getLong() );
ComputerCraft.httpMaxUpload = Math.max( 0, httpMaxUpload.getLong() );
ComputerCraft.httpMaxWebsockets = Math.max( 1, httpMaxWebsockets.getInt() );
ComputerCraft.httpMaxWebsocketMessage = Math.max( 0, httpMaxWebsocketMessage.getInt() );
// Peripheral // Peripheral
ComputerCraft.enableCommandBlock = commandBlockEnabled.getBoolean(); ComputerCraft.enableCommandBlock = commandBlockEnabled.getBoolean();
ComputerCraft.maxNotesPerTick = Math.max( 1, maxNotesPerTick.getInt() ); ComputerCraft.maxNotesPerTick = Math.max( 1, maxNotesPerTick.getInt() );

7
src/main/java/dan200/computercraft/shared/computer/core/ClientComputer.java
View File

@@ -64,6 +64,13 @@ public class ClientComputer extends ClientTerminal implements IComputer
return m_instanceID; return m_instanceID;
} }
@Override
@Deprecated
public int getID()
{
return -1;
}
@Override @Override
@Deprecated @Deprecated
public String getLabel() public String getLabel()

3
src/main/java/dan200/computercraft/shared/computer/core/IComputer.java
View File

@@ -13,6 +13,9 @@ public interface IComputer extends ITerminal
{ {
int getInstanceID(); int getInstanceID();
@Deprecated
int getID();
@Deprecated @Deprecated
String getLabel(); String getLabel();

2
src/main/java/dan200/computercraft/shared/computer/core/ServerComputer.java
View File

@@ -226,6 +226,8 @@ public class ServerComputer extends ServerTerminal implements IComputer, IComput
return m_instanceID; return m_instanceID;
} }
@Override
@SuppressWarnings( "deprecation" )
public int getID() public int getID()
{ {
return m_computer.getID(); return m_computer.getID();

10
src/main/java/dan200/computercraft/shared/util/IDAssigner.java
View File

@@ -9,6 +9,7 @@ package dan200.computercraft.shared.util;
import dan200.computercraft.ComputerCraft; import dan200.computercraft.ComputerCraft;
import java.io.*; import java.io.*;
import java.nio.charset.StandardCharsets;
public class IDAssigner public class IDAssigner
{ {
@@ -71,14 +72,7 @@ public class IDAssigner
{ {
FileInputStream in = new FileInputStream( lastidFile ); FileInputStream in = new FileInputStream( lastidFile );
InputStreamReader isr; InputStreamReader isr;
try isr = new InputStreamReader( in, StandardCharsets.UTF_8 );
{
isr = new InputStreamReader( in, "UTF-8" );
}
catch( UnsupportedEncodingException e )
{
isr = new InputStreamReader( in );
}
try( BufferedReader br = new BufferedReader( isr ) ) try( BufferedReader br = new BufferedReader( isr ) )
{ {
idString = br.readLine(); idString = br.readLine();

24
src/main/java/dan200/computercraft/shared/util/IoUtil.java Normal file
View File

@@ -0,0 +1,24 @@
/*
* This file is part of ComputerCraft - http://www.computercraft.info
* Copyright Daniel Ratcliffe, 2011-2019. Do not distribute without permission.
* Send enquiries to dratcliffe@gmail.com
*/
package dan200.computercraft.shared.util;
import java.io.Closeable;
import java.io.IOException;
public class IoUtil
{
public static void closeQuietly( Closeable closeable )
{
try
{
closeable.close();
}
catch( IOException ignored )
{
}
}
}

6
src/main/resources/assets/computercraft/lang/en_us.lang
View File

@@ -59,6 +59,12 @@ gui.computercraft:config.http.websocket_enabled=Enable websockets
gui.computercraft:config.http.whitelist=HTTP whitelist gui.computercraft:config.http.whitelist=HTTP whitelist
gui.computercraft:config.http.blacklist=HTTP blacklist gui.computercraft:config.http.blacklist=HTTP blacklist
gui.computercraft:config.http.max_requests=Maximum concurrent requests
gui.computercraft:config.http.max_download=Maximum response size
gui.computercraft:config.http.max_upload=Maximum request
gui.computercraft:config.http.max_websockets=Maximum concurrent websockets
gui.computercraft:config.http.max_websocket_message=Maximum websocket message size
gui.computercraft:config.peripheral=Peripherals gui.computercraft:config.peripheral=Peripherals
gui.computercraft:config.peripheral.command_block_enabled=Enable command block peripheral gui.computercraft:config.peripheral.command_block_enabled=Enable command block peripheral
gui.computercraft:config.peripheral.modem_range=Modem range (default) gui.computercraft:config.peripheral.modem_range=Modem range (default)