Redo how http block/allow lists are stored. (#396)

This replaces the allow/block lists with a series of rules. Each rule
takes the form

    [[http.rules]]
    host = "127.0.0.0/8"
    action = "block"

This is pretty much the same as the previous config style, in that hosts
may be domains, wildcards or in CIDR notation. However, they may also be
mixed, so you could allow a specific IP, and then block all others.

src/main/java/dan200/computercraft/ComputerCraft.java

@@ -6,7 +6,7 @@
 package dan200.computercraft;
 
 import dan200.computercraft.api.turtle.event.TurtleAction;
-import dan200.computercraft.core.apis.AddressPredicate;
+import dan200.computercraft.core.apis.http.AddressRule;
 import dan200.computercraft.core.apis.http.websocket.Websocket;
 import dan200.computercraft.shared.Config;
 import dan200.computercraft.shared.computer.blocks.BlockComputer;
@@ -39,8 +39,13 @@ import org.apache.logging.log4j.Logger;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Collections;
 import java.util.EnumSet;
+import java.util.List;
+import java.util.Objects;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 @Mod( ComputerCraft.MOD_ID )
 public final class ComputerCraft
@@ -50,8 +55,8 @@ public final class ComputerCraft
     public static final int DATAFIXER_VERSION = 0;
 
     // Configuration options
-    public static final String[] DEFAULT_HTTP_WHITELIST = new String[] { "*" };
-    public static final String[] DEFAULT_HTTP_BLACKLIST = new String[] {
+    public static final String[] DEFAULT_HTTP_ALLOW = new String[] { "*" };
+    public static final String[] DEFAULT_HTTP_DENY = new String[] {
         "127.0.0.0/8",
         "10.0.0.0/8",
         "172.16.0.0/12",
@@ -71,10 +76,12 @@ public final class ComputerCraft
     public static long maxMainGlobalTime = TimeUnit.MILLISECONDS.toNanos( 10 );
     public static long maxMainComputerTime = TimeUnit.MILLISECONDS.toNanos( 5 );
 
-    public static boolean http_enable = true;
-    public static boolean http_websocket_enable = true;
-    public static AddressPredicate http_whitelist = new AddressPredicate( DEFAULT_HTTP_WHITELIST );
-    public static AddressPredicate http_blacklist = new AddressPredicate( DEFAULT_HTTP_BLACKLIST );
+    public static boolean httpEnabled = true;
+    public static boolean httpWebsocketEnabled = true;
+    public static List<AddressRule> httpRules = Collections.unmodifiableList( Stream.concat(
+        Stream.of( DEFAULT_HTTP_DENY ).map( x -> AddressRule.parse( x, AddressRule.Action.DENY ) ).filter( Objects::nonNull ),
+        Stream.of( DEFAULT_HTTP_ALLOW ).map( x -> AddressRule.parse( x, AddressRule.Action.ALLOW ) ).filter( Objects::nonNull )
+    ).collect( Collectors.toList() ) );
 
     public static int httpTimeout = 30000;
     public static int httpMaxRequests = 16;

src/main/java/dan200/computercraft/core/apis/AddressPredicate.java

@@ -1,181 +0,0 @@
-/*
- * This file is part of ComputerCraft - http://www.computercraft.info
- * Copyright Daniel Ratcliffe, 2011-2020. Do not distribute without permission.
- * Send enquiries to dratcliffe@gmail.com
- */
-package dan200.computercraft.core.apis;
-
-import com.google.common.net.InetAddresses;
-import dan200.computercraft.ComputerCraft;
-
-import java.net.Inet6Address;
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.regex.Pattern;
-
-/**
- * Used to determine whether a domain or IP address matches a series of patterns.
- */
-public class AddressPredicate
-{
-    private static final class HostRange
-    {
-        private final byte[] min;
-        private final byte[] max;
-
-        private HostRange( byte[] min, byte[] max )
-        {
-            this.min = min;
-            this.max = max;
-        }
-
-        public boolean contains( InetAddress address )
-        {
-            byte[] entry = address.getAddress();
-            if( entry.length != min.length ) return false;
-
-            for( int i = 0; i < entry.length; i++ )
-            {
-                int value = 0xFF & entry[i];
-                if( value < (0xFF & min[i]) || value > (0xFF & max[i]) ) return false;
-            }
-
-            return true;
-        }
-    }
-
-    private final List<Pattern> wildcards;
-    private final List<HostRange> ranges;
-
-    public AddressPredicate( String... filters )
-    {
-        this( Arrays.asList( filters ) );
-    }
-
-    public AddressPredicate( Iterable<? extends String> filters )
-    {
-        List<Pattern> wildcards = this.wildcards = new ArrayList<>();
-        List<HostRange> ranges = this.ranges = new ArrayList<>();
-
-        for( String filter : filters )
-        {
-            int cidr = filter.indexOf( '/' );
-            if( cidr >= 0 )
-            {
-                String addressStr = filter.substring( 0, cidr );
-                String prefixSizeStr = filter.substring( cidr + 1 );
-
-                int prefixSize;
-                try
-                {
-                    prefixSize = Integer.parseInt( prefixSizeStr );
-                }
-                catch( NumberFormatException e )
-                {
-                    ComputerCraft.log.error(
-                        "Malformed http whitelist/blacklist entry '{}': Cannot extract size of CIDR mask from '{}'.",
-                        filter, prefixSizeStr
-                    );
-                    continue;
-                }
-
-                InetAddress address;
-                try
-                {
-                    address = InetAddresses.forString( addressStr );
-                }
-                catch( IllegalArgumentException e )
-                {
-                    ComputerCraft.log.error(
-                        "Malformed http whitelist/blacklist entry '{}': Cannot extract IP address from '{}'.",
-                        filter, prefixSizeStr
-                    );
-                    continue;
-                }
-
-                // Mask the bytes of the IP address.
-                byte[] minBytes = address.getAddress(), maxBytes = address.getAddress();
-                int size = prefixSize;
-                for( int i = 0; i < minBytes.length; i++ )
-                {
-                    if( size <= 0 )
-                    {
-                        minBytes[i] &= 0;
-                        maxBytes[i] |= 0xFF;
-                    }
-                    else if( size < 8 )
-                    {
-                        minBytes[i] &= 0xFF << (8 - size);
-                        maxBytes[i] |= ~(0xFF << (8 - size));
-                    }
-
-                    size -= 8;
-                }
-
-                ranges.add( new HostRange( minBytes, maxBytes ) );
-            }
-            else
-            {
-                wildcards.add( Pattern.compile( "^\\Q" + filter.replaceAll( "\\*", "\\\\E.*\\\\Q" ) + "\\E$" ) );
-            }
-        }
-    }
-
-    /**
-     * Determine whether a host name matches a series of patterns.
-     *
-     * This is intended to allow early exiting, before one has to look up the IP address. You should use
-     * {@link #matches(InetAddress)} instead of/in addition to this one.
-     *
-     * @param domain The domain to match.
-     * @return Whether the patterns were matched.
-     */
-    public boolean matches( String domain )
-    {
-        for( Pattern domainPattern : wildcards )
-        {
-            if( domainPattern.matcher( domain ).matches() ) return true;
-        }
-
-        return false;
-    }
-
-    private boolean matchesAddress( InetAddress address )
-    {
-        String addressString = address.getHostAddress();
-        for( Pattern domainPattern : wildcards )
-        {
-            if( domainPattern.matcher( addressString ).matches() ) return true;
-        }
-
-        for( HostRange range : ranges )
-        {
-            if( range.contains( address ) ) return true;
-        }
-
-        return false;
-    }
-
-    /**
-     * Determine whether the given address matches a series of patterns.
-     *
-     * @param address The address to check.
-     * @return Whether it matches any of these patterns.
-     */
-    public boolean matches( InetAddress address )
-    {
-        // Match the host name
-        String host = address.getHostName();
-        if( host != null && matches( host ) ) return true;
-
-        // Match the normal address
-        if( matchesAddress( address ) ) return true;
-
-        // If we're an IPv4 address in disguise then let's check that.
-        return address instanceof Inet6Address && InetAddresses.is6to4Address( (Inet6Address) address )
-            && matchesAddress( InetAddresses.get6to4IPv4Address( (Inet6Address) address ) );
-
-    }
-}

src/main/java/dan200/computercraft/core/apis/HTTPAPI.java

@@ -173,7 +173,7 @@ public class HTTPAPI implements ILuaAPI
                 String address = getString( args, 0 );
                 Map<?, ?> headerTbl = optTable( args, 1, Collections.emptyMap() );
 
-                if( !ComputerCraft.http_websocket_enable )
+                if( !ComputerCraft.httpWebsocketEnabled )
                 {
                     throw new LuaException( "Websocket connections are disabled" );
                 }

src/main/java/dan200/computercraft/core/apis/http/AddressRule.java

@@ -0,0 +1,167 @@
+/*
+ * This file is part of ComputerCraft - http://www.computercraft.info
+ * Copyright Daniel Ratcliffe, 2011-2020. Do not distribute without permission.
+ * Send enquiries to dratcliffe@gmail.com
+ */
+package dan200.computercraft.core.apis.http;
+
+import com.google.common.net.InetAddresses;
+import dan200.computercraft.ComputerCraft;
+
+import javax.annotation.Nullable;
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.util.regex.Pattern;
+
+/**
+ * A pattern which matches an address, and controls whether it is accessible or not.
+ */
+public final class AddressRule
+{
+    private static final class HostRange
+    {
+        private final byte[] min;
+        private final byte[] max;
+
+        private HostRange( byte[] min, byte[] max )
+        {
+            this.min = min;
+            this.max = max;
+        }
+
+        public boolean contains( InetAddress address )
+        {
+            byte[] entry = address.getAddress();
+            if( entry.length != min.length ) return false;
+
+            for( int i = 0; i < entry.length; i++ )
+            {
+                int value = 0xFF & entry[i];
+                if( value < (0xFF & min[i]) || value > (0xFF & max[i]) ) return false;
+            }
+
+            return true;
+        }
+    }
+
+    public enum Action
+    {
+        ALLOW,
+        DENY,
+    }
+
+    private final HostRange ip;
+    private final Pattern domainPattern;
+    private final Action action;
+
+    private AddressRule( HostRange ip, Pattern domainPattern, Action action )
+    {
+        this.ip = ip;
+        this.domainPattern = domainPattern;
+        this.action = action;
+    }
+
+    @Nullable
+    public static AddressRule parse( String filter, Action action )
+    {
+        int cidr = filter.indexOf( '/' );
+        if( cidr >= 0 )
+        {
+            String addressStr = filter.substring( 0, cidr );
+            String prefixSizeStr = filter.substring( cidr + 1 );
+
+            int prefixSize;
+            try
+            {
+                prefixSize = Integer.parseInt( prefixSizeStr );
+            }
+            catch( NumberFormatException e )
+            {
+                ComputerCraft.log.error(
+                    "Malformed http whitelist/blacklist entry '{}': Cannot extract size of CIDR mask from '{}'.",
+                    filter, prefixSizeStr
+                );
+                return null;
+            }
+
+            InetAddress address;
+            try
+            {
+                address = InetAddresses.forString( addressStr );
+            }
+            catch( IllegalArgumentException e )
+            {
+                ComputerCraft.log.error(
+                    "Malformed http whitelist/blacklist entry '{}': Cannot extract IP address from '{}'.",
+                    filter, prefixSizeStr
+                );
+                return null;
+            }
+
+            // Mask the bytes of the IP address.
+            byte[] minBytes = address.getAddress(), maxBytes = address.getAddress();
+            int size = prefixSize;
+            for( int i = 0; i < minBytes.length; i++ )
+            {
+                if( size <= 0 )
+                {
+                    minBytes[i] &= 0;
+                    maxBytes[i] |= 0xFF;
+                }
+                else if( size < 8 )
+                {
+                    minBytes[i] &= 0xFF << (8 - size);
+                    maxBytes[i] |= ~(0xFF << (8 - size));
+                }
+
+                size -= 8;
+            }
+
+            return new AddressRule( new HostRange( minBytes, maxBytes ), null, action );
+        }
+        else
+        {
+            Pattern pattern = Pattern.compile( "^\\Q" + filter.replaceAll( "\\*", "\\\\E.*\\\\Q" ) + "\\E$" );
+            return new AddressRule( null, pattern, action );
+        }
+    }
+
+    /**
+     * Determine whether the given address matches a series of patterns.
+     *
+     * @param domain  The domain to match
+     * @param address The address to check.
+     * @return Whether it matches any of these patterns.
+     */
+    public boolean matches( String domain, InetAddress address )
+    {
+        if( domainPattern != null )
+        {
+            if( domainPattern.matcher( domain ).matches() ) return true;
+            if( domainPattern.matcher( address.getHostName() ).matches() ) return true;
+        }
+
+        // Match the normal address
+        if( matchesAddress( address ) ) return true;
+
+        // If we're an IPv4 address in disguise then let's check that.
+        return address instanceof Inet6Address && InetAddresses.is6to4Address( (Inet6Address) address )
+            && matchesAddress( InetAddresses.get6to4IPv4Address( (Inet6Address) address ) );
+    }
+
+    private boolean matchesAddress( InetAddress address )
+    {
+        if( domainPattern != null && domainPattern.matcher( address.getHostAddress() ).matches() ) return true;
+        return ip != null && ip.contains( address );
+    }
+
+    public static Action apply( Iterable<? extends AddressRule> rules, String domain, InetAddress address )
+    {
+        for( AddressRule rule : rules )
+        {
+            if( rule.matches( domain, address ) ) return rule.action;
+        }
+
+        return Action.DENY;
+    }
+}

src/main/java/dan200/computercraft/core/apis/http/NetworkUtils.java

@@ -97,20 +97,6 @@ public final class NetworkUtils
         }
     }
 
-    /**
-     * Checks a host is allowed.
-     *
-     * @param host The domain to check against
-     * @throws HTTPRequestException If the host is not permitted.
-     */
-    public static void checkHost( String host ) throws HTTPRequestException
-    {
-        if( !ComputerCraft.http_whitelist.matches( host ) || ComputerCraft.http_blacklist.matches( host ) )
-        {
-            throw new HTTPRequestException( "Domain not permitted" );
-        }
-    }
-
     /**
      * Create a {@link InetSocketAddress} from the resolved {@code host} and port.
      *
@@ -130,7 +116,7 @@ public final class NetworkUtils
         if( socketAddress.isUnresolved() ) throw new HTTPRequestException( "Unknown host" );
 
         InetAddress address = socketAddress.getAddress();
-        if( !ComputerCraft.http_whitelist.matches( address ) || ComputerCraft.http_blacklist.matches( address ) )
+        if( AddressRule.apply( ComputerCraft.httpRules, host, address ) == AddressRule.Action.DENY )
         {
             throw new HTTPRequestException( "Domain not permitted" );
         }

src/main/java/dan200/computercraft/core/apis/http/request/HttpRequest.java

@@ -119,8 +119,6 @@ public class HttpRequest extends Resource<HttpRequest>
         {
             throw new HTTPRequestException( "Invalid protocol '" + scheme + "'" );
         }
-
-        NetworkUtils.checkHost( url.getHost() );
     }
 
     public void request( URI uri, HttpMethod method )

src/main/java/dan200/computercraft/core/apis/http/websocket/Websocket.java

@@ -106,7 +106,6 @@ public class Websocket extends Resource<Websocket>
             throw new HTTPRequestException( "Invalid scheme '" + scheme + "'" );
         }
 
-        NetworkUtils.checkHost( uri.getHost() );
         return uri;
     }
 

src/main/java/dan200/computercraft/core/computer/ComputerExecutor.java

@@ -171,7 +171,7 @@ final class ComputerExecutor
         apis.add( new FSAPI( environment ) );
         apis.add( new PeripheralAPI( environment ) );
         apis.add( new OSAPI( environment ) );
-        if( ComputerCraft.http_enable ) apis.add( new HTTPAPI( environment ) );
+        if( ComputerCraft.httpEnabled ) apis.add( new HTTPAPI( environment ) );
 
         // Load in the externally registered APIs.
         for( ILuaAPIFactory factory : ApiFactories.getAll() )

src/main/java/dan200/computercraft/shared/Config.java

@@ -6,12 +6,13 @@
 package dan200.computercraft.shared;
 
 import com.electronwill.nightconfig.core.CommentedConfig;
+import com.electronwill.nightconfig.core.UnmodifiableConfig;
 import com.electronwill.nightconfig.core.file.CommentedFileConfig;
 import com.google.common.base.CaseFormat;
 import com.google.common.base.Converter;
 import dan200.computercraft.ComputerCraft;
 import dan200.computercraft.api.turtle.event.TurtleAction;
-import dan200.computercraft.core.apis.AddressPredicate;
+import dan200.computercraft.core.apis.http.AddressRule;
 import dan200.computercraft.core.apis.http.websocket.Websocket;
 import net.minecraftforge.common.ForgeConfigSpec;
 import net.minecraftforge.eventbus.api.SubscribeEvent;
@@ -19,13 +20,14 @@ import net.minecraftforge.fml.ModLoadingContext;
 import net.minecraftforge.fml.common.Mod;
 import net.minecraftforge.fml.config.ModConfig;
 
-import java.util.Arrays;
+import javax.annotation.Nullable;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
-import static dan200.computercraft.ComputerCraft.DEFAULT_HTTP_BLACKLIST;
-import static dan200.computercraft.ComputerCraft.DEFAULT_HTTP_WHITELIST;
 import static net.minecraftforge.common.ForgeConfigSpec.Builder;
 import static net.minecraftforge.common.ForgeConfigSpec.ConfigValue;
 
@@ -50,8 +52,7 @@ public final class Config
 
     private static ConfigValue<Boolean> httpEnabled;
     private static ConfigValue<Boolean> httpWebsocketEnabled;
-    private static ConfigValue<List<? extends String>> httpWhitelist;
-    private static ConfigValue<List<? extends String>> httpBlacklist;
+    private static ConfigValue<List<? extends UnmodifiableConfig>> httpRules;
 
     private static ConfigValue<Integer> httpTimeout;
     private static ConfigValue<Integer> httpMaxRequests;
@@ -151,25 +152,25 @@ public final class Config
             builder.push( "http" );
 
             httpEnabled = builder
-                .comment( "Enable the \"http\" API on Computers (see \"http_whitelist\" and \"http_blacklist\" for more " +
-                    "fine grained control than this)" )
-                .define( "enabled", ComputerCraft.http_enable );
+                .comment( "Enable the \"http\" API on Computers (see \"rules\" for more fine grained control than this)." )
+                .define( "enabled", ComputerCraft.httpEnabled );
 
             httpWebsocketEnabled = builder
                 .comment( "Enable use of http websockets. This requires the \"http_enable\" option to also be true." )
-                .define( "websocket_enabled", ComputerCraft.http_websocket_enable );
+                .define( "websocket_enabled", ComputerCraft.httpWebsocketEnabled );
 
-            httpWhitelist = builder
-                .comment( "A list of wildcards for domains or IP ranges that can be accessed through the \"http\" API on Computers.\n" +
-                    "Set this to \"*\" to access to the entire internet. Example: \"*.pastebin.com\" will restrict access to just subdomains of pastebin.com.\n" +
-                    "You can use domain names (\"pastebin.com\"), wilcards (\"*.pastebin.com\") or CIDR notation (\"127.0.0.0/8\")." )
-                .defineList( "whitelist", Arrays.asList( DEFAULT_HTTP_WHITELIST ), x -> true );
-
-            httpBlacklist = builder
-                .comment( "A list of wildcards for domains or IP ranges that cannot be accessed through the \"http\" API on Computers.\n" +
-                    "If this is empty then all whitelisted domains will be accessible. Example: \"*.github.com\" will block access to all subdomains of github.com.\n" +
-                    "You can use domain names (\"pastebin.com\"), wilcards (\"*.pastebin.com\") or CIDR notation (\"127.0.0.0/8\")." )
-                .defineList( "blacklist", Arrays.asList( DEFAULT_HTTP_BLACKLIST ), x -> true );
+            httpRules = builder
+                .comment( "A list of rules which control which domains or IPs are allowed through the \"http\" API on computers.\n" +
+                    "Each rule is an item with a 'host' to match against, and an action. " +
+                    "The host may be a domain name (\"pastebin.com\"),\n" +
+                    "wildcard (\"*.pastebin.com\") or CIDR notation (\"127.0.0.0/8\"). 'action' maybe 'allow' or 'block'. If no rules" +
+                    "match, the domain will be blocked." )
+                .defineList( "rules",
+                    Stream.concat(
+                        Stream.of( ComputerCraft.DEFAULT_HTTP_DENY ).map( x -> makeRule( x, "deny" ) ),
+                        Stream.of( ComputerCraft.DEFAULT_HTTP_ALLOW ).map( x -> makeRule( x, "allow" ) )
+                    ).collect( Collectors.toList() ),
+                    x -> x instanceof UnmodifiableConfig && parseRule( (UnmodifiableConfig) x ) != null );
 
             httpTimeout = builder
                 .comment( "The period of time (in milliseconds) to wait before a HTTP request times out. Set to 0 for unlimited." )
@@ -286,10 +287,10 @@ public final class Config
         ComputerCraft.maxMainComputerTime = TimeUnit.MILLISECONDS.toNanos( maxMainComputerTime.get() );
 
         // HTTP
-        ComputerCraft.http_enable = httpEnabled.get();
-        ComputerCraft.http_websocket_enable = httpWebsocketEnabled.get();
-        ComputerCraft.http_whitelist = new AddressPredicate( httpWhitelist.get() );
-        ComputerCraft.http_blacklist = new AddressPredicate( httpBlacklist.get() );
+        ComputerCraft.httpEnabled = httpEnabled.get();
+        ComputerCraft.httpWebsocketEnabled = httpWebsocketEnabled.get();
+        ComputerCraft.httpRules = Collections.unmodifiableList( httpRules.get().stream()
+            .map( Config::parseRule ).filter( Objects::nonNull ).collect( Collectors.toList() ) );
 
         ComputerCraft.httpTimeout = httpTimeout.get();
         ComputerCraft.httpMaxRequests = httpMaxRequests.get();
@@ -346,4 +347,28 @@ public final class Config
             return null;
         }
     }
+
+    private static UnmodifiableConfig makeRule( String host, String action )
+    {
+        com.electronwill.nightconfig.core.Config config = com.electronwill.nightconfig.core.Config.inMemory();
+        config.add( "host", host );
+        config.add( "action", action );
+        return config;
+    }
+
+    @Nullable
+    private static AddressRule parseRule( UnmodifiableConfig builder )
+    {
+        Object hostObj = builder.get( "host" );
+        Object actionObj = builder.get( "action" );
+        if( !(hostObj instanceof String) || !(actionObj instanceof String) ) return null;
+
+        String host = (String) hostObj, action = (String) actionObj;
+        for( AddressRule.Action candiate : AddressRule.Action.values() )
+        {
+            if( candiate.name().equalsIgnoreCase( action ) ) return AddressRule.parse( host, candiate );
+        }
+
+        return null;
+    }
 }