Allow $private HTTP rule to block any private IP

This is a little magic compared with our previous approach of "list
every private IP range", but given then the sheer number we were
missing[1][2] this feels more reasonable.

Also refactor out some of the logic into separate classes, hopefully to
make things a little cleaner.

build.gradle

@@ -51,8 +51,9 @@ dependencies {
 
     shade 'org.squiddev:Cobalt:0.5.1-SNAPSHOT'
 
-    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.1.0'
-    testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.1.0'
+    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.7.0'
+    testImplementation 'org.junit.jupiter:junit-jupiter-params:5.7.0'
+    testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.7.0'
 
     modRuntime "me.shedaniel:RoughlyEnoughItems-api:5.2.10"
     modRuntime "me.shedaniel:RoughlyEnoughItems:5.2.10"

patchwork.md

@@ -408,3 +408,10 @@ Remove <!-- -->s in usages
 Added documentation for global functions (#592)
 ```
 Didn't port the docs over.
+
+```
+d83a68f3ff6e3833278a38798d06215293656e85
+
+Allow $private HTTP rule to block any private IP
+```
+The config still uses a `blacklist` and `whitelist` array.

src/main/java/dan200/computercraft/ComputerCraft.java

@@ -10,12 +10,12 @@ import static dan200.computercraft.shared.ComputerCraftRegistry.ModBlocks;
 import static dan200.computercraft.shared.ComputerCraftRegistry.init;
 
 import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.List;
-import java.util.Objects;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
+
 
 import dan200.computercraft.api.turtle.event.TurtleAction;
 import dan200.computercraft.core.apis.http.options.Action;
@@ -32,20 +32,10 @@ import dan200.computercraft.shared.data.PlayerCreativeLootCondition;
 import dan200.computercraft.shared.media.recipes.DiskRecipe;
 import dan200.computercraft.shared.media.recipes.PrintoutRecipe;
 import dan200.computercraft.shared.peripheral.monitor.MonitorRenderer;
-import dan200.computercraft.shared.pocket.peripherals.PocketModem;
-import dan200.computercraft.shared.pocket.peripherals.PocketSpeaker;
 import dan200.computercraft.shared.pocket.recipes.PocketComputerUpgradeRecipe;
 import dan200.computercraft.shared.proxy.ComputerCraftProxyCommon;
 import dan200.computercraft.shared.turtle.recipes.TurtleRecipe;
 import dan200.computercraft.shared.turtle.recipes.TurtleUpgradeRecipe;
-import dan200.computercraft.shared.turtle.upgrades.TurtleAxe;
-import dan200.computercraft.shared.turtle.upgrades.TurtleCraftingTable;
-import dan200.computercraft.shared.turtle.upgrades.TurtleHoe;
-import dan200.computercraft.shared.turtle.upgrades.TurtleModem;
-import dan200.computercraft.shared.turtle.upgrades.TurtleShovel;
-import dan200.computercraft.shared.turtle.upgrades.TurtleSpeaker;
-import dan200.computercraft.shared.turtle.upgrades.TurtleSword;
-import dan200.computercraft.shared.turtle.upgrades.TurtleTool;
 import dan200.computercraft.shared.util.Config;
 import dan200.computercraft.shared.util.ImpostorRecipe;
 import dan200.computercraft.shared.util.ImpostorShapelessRecipe;
@@ -65,15 +55,6 @@ import net.fabricmc.loader.api.FabricLoader;
 public final class ComputerCraft implements ModInitializer {
     public static final String MOD_ID = "computercraft";
     // Configuration options
-    public static final String[] DEFAULT_HTTP_WHITELIST = new String[] {"*"};
-    public static final String[] DEFAULT_HTTP_BLACKLIST = new String[] {
-        "127.0.0.0/8",
-        "10.0.0.0/8",
-        "172.16.0.0/12",
-        "192.168.0.0/16",
-        "fd00::/8",
-        "0.0.0.0/8"
-        };
     public static final int terminalWidth_computer = 51;
     public static final int terminalHeight_computer = 19;
     public static final int terminalWidth_turtle = 39;
@@ -122,17 +103,10 @@ public final class ComputerCraft implements ModInitializer {
     public static int monitorHeight = 6;
     public static double monitorDistanceSq = 4096;
 
-    public static List<AddressRule> httpRules = buildHttpRulesFromConfig(DEFAULT_HTTP_BLACKLIST, DEFAULT_HTTP_WHITELIST);
-
-    public static List<AddressRule> buildHttpRulesFromConfig(String[] blacklist, String[] whitelist) {
-        return Stream.concat(Stream.of(blacklist)
-                        .map( x -> AddressRule.parse( x, null, Action.DENY.toPartial()))
-                        .filter(Objects::nonNull),
-                Stream.of(whitelist)
-                        .map( x -> AddressRule.parse( x, null, Action.ALLOW.toPartial()))
-                        .filter(Objects::nonNull))
-                .collect(Collectors.toList());
-    }
+    public static List<AddressRule> httpRules = Collections.unmodifiableList( Arrays.asList(
+        AddressRule.parse( "$private", null, Action.DENY.toPartial() ),
+        AddressRule.parse( "*", null, Action.ALLOW.toPartial() )
+    ) );
 
     @Override
     public void onInitialize() {

src/main/java/dan200/computercraft/core/apis/http/options/AddressPredicate.java

@@ -0,0 +1,148 @@
+/*
+ * This file is part of ComputerCraft - http://www.computercraft.info
+ * Copyright Daniel Ratcliffe, 2011-2021. Do not distribute without permission.
+ * Send enquiries to dratcliffe@gmail.com
+ */
+package dan200.computercraft.core.apis.http.options;
+
+import com.google.common.net.InetAddresses;
+import dan200.computercraft.ComputerCraft;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.regex.Pattern;
+
+/**
+ * A predicate on an address. Matches against a domain and an ip address.
+ *
+ * @see AddressRule#apply(Iterable, String, InetSocketAddress) for the actual handling of this rule.
+ */
+interface AddressPredicate
+{
+    default boolean matches( String domain )
+    {
+        return false;
+    }
+
+    default boolean matches( InetAddress socketAddress )
+    {
+        return false;
+    }
+
+    final class HostRange implements AddressPredicate
+    {
+        private final byte[] min;
+        private final byte[] max;
+
+        HostRange( byte[] min, byte[] max )
+        {
+            this.min = min;
+            this.max = max;
+        }
+
+        @Override
+        public boolean matches( InetAddress address )
+        {
+            byte[] entry = address.getAddress();
+            if( entry.length != min.length ) return false;
+
+            for( int i = 0; i < entry.length; i++ )
+            {
+                int value = 0xFF & entry[i];
+                if( value < (0xFF & min[i]) || value > (0xFF & max[i]) ) return false;
+            }
+
+            return true;
+        }
+
+        public static HostRange parse( String addressStr, String prefixSizeStr )
+        {
+            int prefixSize;
+            try
+            {
+                prefixSize = Integer.parseInt( prefixSizeStr );
+            }
+            catch( NumberFormatException e )
+            {
+                ComputerCraft.log.error(
+                    "Malformed http whitelist/blacklist entry '{}': Cannot extract size of CIDR mask from '{}'.",
+                    addressStr + '/' + prefixSizeStr, prefixSizeStr
+                );
+                return null;
+            }
+
+            InetAddress address;
+            try
+            {
+                address = InetAddresses.forString( addressStr );
+            }
+            catch( IllegalArgumentException e )
+            {
+                ComputerCraft.log.error(
+                    "Malformed http whitelist/blacklist entry '{}': Cannot extract IP address from '{}'.",
+                    addressStr + '/' + prefixSizeStr, prefixSizeStr
+                );
+                return null;
+            }
+
+            // Mask the bytes of the IP address.
+            byte[] minBytes = address.getAddress(), maxBytes = address.getAddress();
+            int size = prefixSize;
+            for( int i = 0; i < minBytes.length; i++ )
+            {
+                if( size <= 0 )
+                {
+                    minBytes[i] &= 0;
+                    maxBytes[i] |= 0xFF;
+                }
+                else if( size < 8 )
+                {
+                    minBytes[i] &= 0xFF << (8 - size);
+                    maxBytes[i] |= ~(0xFF << (8 - size));
+                }
+
+                size -= 8;
+            }
+
+            return new HostRange( minBytes, maxBytes );
+        }
+    }
+
+    final class DomainPattern implements AddressPredicate
+    {
+        private final Pattern pattern;
+
+        DomainPattern( Pattern pattern )
+        {
+            this.pattern = pattern;
+        }
+
+        @Override
+        public boolean matches( String domain )
+        {
+            return pattern.matcher( domain ).matches();
+        }
+
+        @Override
+        public boolean matches( InetAddress socketAddress )
+        {
+            return pattern.matcher( socketAddress.getHostAddress() ).matches();
+        }
+    }
+
+
+    final class PrivatePattern implements AddressPredicate
+    {
+        static final PrivatePattern INSTANCE = new PrivatePattern();
+
+        @Override
+        public boolean matches( InetAddress socketAddress )
+        {
+            return socketAddress.isAnyLocalAddress()
+                || socketAddress.isLoopbackAddress()
+                || socketAddress.isLinkLocalAddress()
+                || socketAddress.isSiteLocalAddress();
+        }
+    }
+
+}

src/main/java/dan200/computercraft/core/apis/http/options/AddressRule.java

@@ -6,6 +6,7 @@
 
 package dan200.computercraft.core.apis.http.options;
 
+import java.net.Inet4Address;
 import java.net.Inet6Address;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
@@ -15,7 +16,10 @@ import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import com.google.common.net.InetAddresses;
-import dan200.computercraft.ComputerCraft;
+
+import dan200.computercraft.core.apis.http.options.AddressPredicate.DomainPattern;
+import dan200.computercraft.core.apis.http.options.AddressPredicate.HostRange;
+import dan200.computercraft.core.apis.http.options.AddressPredicate.PrivatePattern;
 
 /**
  * A pattern which matches an address, and controls whether it is accessible or not.
@@ -25,18 +29,12 @@ public final class AddressRule {
     public static final long MAX_UPLOAD = 4 * 1024 * 1024;
     public static final int TIMEOUT = 30_000;
     public static final int WEBSOCKET_MESSAGE = 128 * 1024;
-    private final HostRange ip;
-    private final Pattern domainPattern;
+
+    private final AddressPredicate predicate;
     private final Integer port;
     private final PartialOptions partial;
-	private AddressRule(
-        @Nullable HostRange ip,
-        @Nullable Pattern domainPattern,
-        @Nullable Integer port,
-        @Nonnull PartialOptions partial )
-	{
-        this.ip = ip;
-        this.domainPattern = domainPattern;
+    private AddressRule( @Nonnull AddressPredicate predicate, @Nullable Integer port, @Nonnull PartialOptions partial ) {
+        this.predicate = predicate;
         this.partial = partial;
 		this.port = port;
     }
@@ -47,55 +45,29 @@ public final class AddressRule {
         if (cidr >= 0) {
             String addressStr = filter.substring(0, cidr);
             String prefixSizeStr = filter.substring(cidr + 1);
-
-            int prefixSize;
-            try {
-                prefixSize = Integer.parseInt(prefixSizeStr);
-            } catch (NumberFormatException e) {
-                ComputerCraft.log.error("Malformed http whitelist/blacklist entry '{}': Cannot extract size of CIDR mask from '{}'.",
-                                        filter,
-                                        prefixSizeStr);
-                return null;
-            }
-
-            InetAddress address;
-            try {
-                address = InetAddresses.forString(addressStr);
-            } catch (IllegalArgumentException e) {
-                ComputerCraft.log.error("Malformed http whitelist/blacklist entry '{}': Cannot extract IP address from '{}'.", filter, prefixSizeStr);
-                return null;
-            }
-
-            // Mask the bytes of the IP address.
-            byte[] minBytes = address.getAddress(), maxBytes = address.getAddress();
-            int size = prefixSize;
-            for (int i = 0; i < minBytes.length; i++) {
-                if (size <= 0) {
-                    minBytes[i] &= 0;
-                    maxBytes[i] |= 0xFF;
-                } else if (size < 8) {
-                    minBytes[i] &= 0xFF << (8 - size);
-                    maxBytes[i] |= ~(0xFF << (8 - size));
-                }
-
-                size -= 8;
-            }
-
-            return new AddressRule(new HostRange(minBytes, maxBytes), null, port, partial);
+            HostRange range = HostRange.parse( addressStr, prefixSizeStr );
+            return range == null ? null : new AddressRule( range, port, partial );
+        }
+        else if( filter.equalsIgnoreCase( "$private" ) )
+        {
+            return new AddressRule( PrivatePattern.INSTANCE, port, partial );
         } else {
-            Pattern pattern = Pattern.compile("^\\Q" + filter.replaceAll("\\*", "\\\\E.*\\\\Q") + "\\E$");
-            return new AddressRule(null, pattern, port, partial);
+            Pattern pattern = Pattern.compile( "^\\Q" + filter.replaceAll( "\\*", "\\\\E.*\\\\Q" ) + "\\E$", Pattern.CASE_INSENSITIVE );
+            return new AddressRule( new DomainPattern( pattern ), port, partial );
         }
     }
 
-    public static Options apply(Iterable<? extends AddressRule> rules, String domain, InetSocketAddress address) {
+    public static Options apply( Iterable<? extends AddressRule> rules, String domain, InetSocketAddress socketAddress ) {
         PartialOptions options = null;
         boolean hasMany = false;
 
+        int port = socketAddress.getPort();
+        InetAddress address = socketAddress.getAddress();
+        Inet4Address ipv4Address = address instanceof Inet6Address && InetAddresses.is6to4Address( (Inet6Address) address )
+            ? InetAddresses.get6to4IPv4Address( (Inet6Address) address ) : null;
+
         for (AddressRule rule : rules) {
-            if (!rule.matches(domain, address)) {
-                continue;
-            }
+            if( !rule.matches( domain, port, address, ipv4Address ) ) continue;
 
             if (options == null) {
                 options = rule.partial;
@@ -116,65 +88,16 @@ public final class AddressRule {
     /**
      * Determine whether the given address matches a series of patterns.
      *
-     * @param domain        The domain to match
-     * @param socketAddress The address to check.
+     * @param domain      The domain to match
+     * @param port        The port of the address.
+     * @param address     The address to check.
+     * @param ipv4Address An ipv4 version of the address, if the original was an ipv6 address.
      * @return Whether it matches any of these patterns.
      */
-    private boolean matches(String domain, InetSocketAddress socketAddress) {
-		InetAddress address = socketAddress.getAddress();
-        if( port != null && port != socketAddress.getPort() ) return false;
-
-        if (this.domainPattern != null) {
-            if (this.domainPattern.matcher(domain)
-                                  .matches()) {
-                return true;
-            }
-            if (this.domainPattern.matcher(address.getHostName())
-                                  .matches()) {
-                return true;
-            }
-        }
-
-        // Match the normal address
-        if (this.matchesAddress(address)) {
-            return true;
-        }
-
-        // If we're an IPv4 address in disguise then let's check that.
-        return address instanceof Inet6Address && InetAddresses.is6to4Address((Inet6Address) address) && this.matchesAddress(InetAddresses.get6to4IPv4Address((Inet6Address) address));
-    }
-
-    private boolean matchesAddress(InetAddress address) {
-        if (this.domainPattern != null && this.domainPattern.matcher(address.getHostAddress())
-                                                            .matches()) {
-            return true;
-        }
-        return this.ip != null && this.ip.contains(address);
-    }
-
-    private static final class HostRange {
-        private final byte[] min;
-        private final byte[] max;
-
-        private HostRange(byte[] min, byte[] max) {
-            this.min = min;
-            this.max = max;
-        }
-
-        public boolean contains(InetAddress address) {
-            byte[] entry = address.getAddress();
-            if (entry.length != this.min.length) {
-                return false;
-            }
-
-            for (int i = 0; i < entry.length; i++) {
-                int value = 0xFF & entry[i];
-                if (value < (0xFF & this.min[i]) || value > (0xFF & this.max[i])) {
-                    return false;
-                }
-            }
-
-            return true;
-        }
+    private boolean matches(String domain, int port, InetAddress address, Inet4Address ipv4Address) {
+		if( this.port != null && this.port != port ) return false;
+        return predicate.matches( domain )
+            || predicate.matches( address )
+            || (ipv4Address != null && predicate.matches( ipv4Address ));
     }
 }

src/main/java/dan200/computercraft/core/apis/http/options/AddressRuleConfig.java

@@ -8,7 +8,6 @@ package dan200.computercraft.core.apis.http.options;
 
 
 import com.electronwill.nightconfig.core.CommentedConfig;
-import com.electronwill.nightconfig.core.Config;
 import com.electronwill.nightconfig.core.InMemoryCommentedFormat;
 import com.electronwill.nightconfig.core.UnmodifiableConfig;
 import dan200.computercraft.ComputerCraft;

src/main/java/dan200/computercraft/shared/util/Config.java

@@ -3,7 +3,10 @@ package dan200.computercraft.shared.util;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Objects;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import blue.endless.jankson.Comment;
 import blue.endless.jankson.Jankson;
@@ -13,6 +16,8 @@ import com.google.common.base.CaseFormat;
 import com.google.common.base.Converter;
 import dan200.computercraft.ComputerCraft;
 import dan200.computercraft.api.turtle.event.TurtleAction;
+import dan200.computercraft.core.apis.http.options.Action;
+import dan200.computercraft.core.apis.http.options.AddressRule;
 import dan200.computercraft.core.apis.http.websocket.Websocket;
 
 public class Config {
@@ -83,7 +88,13 @@ public class Config {
         // HTTP
         ComputerCraft.http_enable = config.http.enabled;
         ComputerCraft.http_websocket_enable = config.http.websocket_enabled;
-        ComputerCraft.httpRules = ComputerCraft.buildHttpRulesFromConfig(config.http.blacklist, config.http.whitelist);
+        ComputerCraft.httpRules = Stream.concat(Stream.of(config.http.blacklist)
+                .map( x -> AddressRule.parse( x, null, Action.DENY.toPartial()))
+                .filter(Objects::nonNull),
+        Stream.of(config.http.whitelist)
+                .map( x -> AddressRule.parse( x, null, Action.ALLOW.toPartial()))
+                .filter(Objects::nonNull))
+        .collect(Collectors.toList());
 
         ComputerCraft.httpTimeout = Math.max(0, config.http.timeout);
         ComputerCraft.httpMaxRequests = Math.max(1, config.http.max_requests);
@@ -158,9 +169,9 @@ public class Config {
             ComputerCraft.http_websocket_enable;
 
         @Comment ("\nA list of wildcards for domains or IP ranges that can be accessed through the " + "\"http\" API on Computers.\n" + "Set this to " +
-                  "\"*\" to access to the entire internet. Example: \"*.pastebin.com\" will restrict access to " + "just subdomains of pastebin.com.\n" + "You can use domain names (\"pastebin.com\"), wilcards (\"*.pastebin.com\") or CIDR notation (\"127.0.0.0/8\").") public String[] whitelist = ComputerCraft.DEFAULT_HTTP_WHITELIST.clone();
+                  "\"*\" to access to the entire internet. Example: \"*.pastebin.com\" will restrict access to " + "just subdomains of pastebin.com.\n" + "You can use domain names (\"pastebin.com\"), wilcards (\"*.pastebin.com\") or CIDR notation (\"127.0.0.0/8\").") public String[] whitelist = new String[] {"*"};
 
-        @Comment ("\nA list of wildcards for domains or IP ranges that cannot be accessed through the " + "\"http\" API on Computers.\n" + "If this is " + "empty then all whitelisted domains will be accessible. Example: \"*.github.com\" will block " + "access to all subdomains of github" + ".com.\n" + "You can use domain names (\"pastebin.com\"), wilcards (\"*.pastebin.com\") or CIDR notation (\"127.0.0.0/8\").") public String[] blacklist = ComputerCraft.DEFAULT_HTTP_BLACKLIST.clone();
+        @Comment ("\nA list of wildcards for domains or IP ranges that cannot be accessed through the " + "\"http\" API on Computers.\n" + "If this is " + "empty then all whitelisted domains will be accessible. Example: \"*.github.com\" will block " + "access to all subdomains of github" + ".com.\n" + "You can use domain names (\"pastebin.com\"), wilcards (\"*.pastebin.com\") or CIDR notation (\"127.0.0.0/8\").") public String[] blacklist = new String[] {"$private"};
 
         @Comment ("\nThe period of time (in milliseconds) to wait before a HTTP request times out. Set to 0 for unlimited.") public int timeout =
             ComputerCraft.httpTimeout;