diff --git a/flake.nix b/flake.nix index 0cff5be403f..294b9ed5b1f 100644 --- a/flake.nix +++ b/flake.nix @@ -12,12 +12,23 @@ nixosModule = { config, ... }: with nixpkgs.lib; let + inherit (nixpkgs) lib; cfg = config.networking.stevenBlackHosts; alternatesList = (if cfg.blockFakenews then [ "fakenews" ] else []) ++ (if cfg.blockGambling then [ "gambling" ] else []) ++ (if cfg.blockPorn then [ "porn" ] else []) ++ (if cfg.blockSocial then [ "social" ] else []); - alternatesPath = "alternates/" + builtins.concatStringsSep "-" alternatesList + "/"; + alternatesPath = "/alternates/" + builtins.concatStringsSep "-" alternatesList; + orig = builtins.readFile ( + self.outPath + (if alternatesList != [ ] then alternatesPath else "") + "/hosts" + ); + filtered = + if cfg.enableIPv6 then + orig + else + lib.concatStringsSep "\n" ( + lib.filter (line: !(lib.strings.hasPrefix "::" line)) (lib.strings.splitString "\n" orig) + ); in { options.networking.stevenBlackHosts = { @@ -32,11 +43,7 @@ blockSocial = mkEnableOption "social hosts entries"; }; config = mkIf cfg.enable { - networking.extraHosts = - let - orig = builtins.readFile ("${self}/" + (if alternatesList != [] then alternatesPath else "") + "hosts"); - ipv6 = builtins.replaceStrings [ "0.0.0.0" ] [ "::" ] orig; - in orig + (optionalString cfg.enableIPv6 ("\n" + ipv6)); + networking.extraHosts = filtered; }; }; diff --git a/makeHosts.py b/makeHosts.py index 6c7b5a21cfb..fec11fd1f67 100644 --- a/makeHosts.py +++ b/makeHosts.py @@ -67,10 +67,10 @@ def recursively_loop_extensions(extension, extensions, current_extensions): name = "-".join(c_current_extensions) - params = ("-a", "-n", "-o", "alternates/"+name, "-e") + tuple(c_current_extensions) + params = ("-a", "-d", "-n", "-o", "alternates/"+name, "-e") + tuple(c_current_extensions) update_hosts_file(*params) - params = ("-a", "-n", "-s", "--nounifiedhosts", "-o", "alternates/"+name+"-only", "-e") + tuple(c_current_extensions) + params = ("-a", "-d", "-n", "-s", "--nounifiedhosts", "-o", "alternates/"+name+"-only", "-e") + tuple(c_current_extensions) update_hosts_file(*params) while len(c_extensions) > 0: @@ -83,10 +83,22 @@ def main(): "file from hosts stored in " "data subfolders." ) + parser.add_argument( + "--noupdate", + "-n", + dest="noupdate", + default=False, + action="store_true", + help="Don't update from host data sources.", + ) parser.parse_args() + options = vars(parser.parse_args()) # Update the unified hosts file - update_hosts_file("-a") + if options["noupdate"]: + update_hosts_file("-a", "-d", "-n") + else: + update_hosts_file("-a", "-d") # List of extensions we want to generate, we will loop over them recursively to prevent manual definitions # Only add new extensions to the end of the array, to avoid relocating existing hosts-files diff --git a/testUpdateHostsFile.py b/testUpdateHostsFile.py index 9de043affb5..e81c9cbd468 100644 --- a/testUpdateHostsFile.py +++ b/testUpdateHostsFile.py @@ -118,7 +118,7 @@ def test_get_defaults(self): "minimise": False, "outputsubfolder": "", "hostfilename": "hosts", - "targetip": "0.0.0.0", + "targetips": ["0.0.0.0", "::"], "sourcedatafilename": "update.json", "sourcesdata": [], "readmefilename": "readme.md", @@ -131,6 +131,7 @@ def test_get_defaults(self): "commonexclusions": ["hulu.com"], "blacklistfile": "foo" + self.sep + "blacklist", "whitelistfile": "foo" + self.sep + "whitelist", + "addsystemhostname": True, } self.assertDictEqual(actual, expected) @@ -1092,7 +1093,7 @@ def test_basic_include_static_hosts(self): def test_basic_include_static_hosts_linux(self): kwargs = dict( - extensions="", outputsubfolder="", numberofrules=5, skipstatichosts=False, nounifiedhosts=False + extensions="", outputsubfolder="", numberofrules=5, skipstatichosts=False, nounifiedhosts=False, addsystemhostname=True ) with self.mock_property("platform.system") as system: system.return_value = "Linux" diff --git a/updateHostsFile.py b/updateHostsFile.py index 2a79fcef5fd..1be2996175a 100755 --- a/updateHostsFile.py +++ b/updateHostsFile.py @@ -78,7 +78,7 @@ def get_defaults(): "minimise": False, "outputsubfolder": "", "hostfilename": "hosts", - "targetip": "0.0.0.0", + "targetips": ["0.0.0.0", "::"], "sourcedatafilename": "update.json", "sourcesdata": [], "readmefilename": "readme.md", @@ -91,6 +91,7 @@ def get_defaults(): "commonexclusions": ["hulu.com"], "blacklistfile": path_join_robust(BASEDIR_PATH, "blacklist"), "whitelistfile": path_join_robust(BASEDIR_PATH, "whitelist"), + "addsystemhostname": True, } @@ -136,9 +137,10 @@ def main(): parser.add_argument( "--ip", "-i", - dest="targetip", - default="0.0.0.0", - help="Target IP address. Default is 0.0.0.0.", + dest="targetips", + nargs="+", + default=["0.0.0.0", "::"], + help="""Target IP address(es). Default is ["0.0.0.0", "::"].""", ) parser.add_argument( "--keepdomaincomments", @@ -228,6 +230,14 @@ def main(): default=path_join_robust(BASEDIR_PATH, "blacklist"), help="Blacklist file to use while generating hosts files.", ) + parser.add_argument( + "--dont-add-system-hostname", + "-d", + dest="addsystemhostname", + default=True, + action="store_false", + help="Don't add the current system hostname while generating hosts files.", + ) global settings @@ -236,7 +246,8 @@ def main(): options["outputpath"] = path_join_robust(BASEDIR_PATH, options["outputsubfolder"]) options["freshen"] = not options["noupdate"] - settings = get_defaults() + defaults = get_defaults() + settings = defaults.copy() settings.update(options) datapath = settings["datapath"] @@ -274,6 +285,18 @@ def main(): exclusionregexes=exclusionregexes, ) + targetips = set(settings["targetips"]) + targetips_updated = targetips != set(defaults["targetips"]) + generatev4, generatev6 = prompt_for_v4_v6(skipprompt=(auto or targetips_updated)) + if not generatev4: + targetips.remove("0.0.0.0") + if not generatev6: + targetips.remove("::") + print() + if len(targetips) == 0: + print_failure("Target IP list is empty!") + exit(1) + extensions = settings["extensions"] sourcesdata = update_sources_data( settings["sourcesdata"], @@ -284,22 +307,23 @@ def main(): nounifiedhosts=nounifiedhosts, ) - mergefile = create_initial_file( - nounifiedhosts=nounifiedhosts, - ) remove_old_hosts_file(settings["outputpath"], "hosts", settings["backup"]) - if settings["compress"]: - finalfile = open(path_join_robust(settings["outputpath"], "hosts"), "w+b") - compressedfile = tempfile.NamedTemporaryFile() - remove_dups_and_excl(mergefile, exclusionregexes, compressedfile) - compress_file(compressedfile, settings["targetip"], finalfile) - elif settings["minimise"]: - finalfile = open(path_join_robust(settings["outputpath"], "hosts"), "w+b") - minimisedfile = tempfile.NamedTemporaryFile() - remove_dups_and_excl(mergefile, exclusionregexes, minimisedfile) - minimise_file(minimisedfile, settings["targetip"], finalfile) - else: - finalfile = remove_dups_and_excl(mergefile, exclusionregexes) + + if not os.path.exists(settings["outputpath"]): + os.makedirs(settings["outputpath"]) + + finalfile = open(path_join_robust(settings["outputpath"], "hosts"), "w+b") + with create_initial_file(nounifiedhosts=nounifiedhosts) as mergefile: + if settings["compress"]: + with tempfile.NamedTemporaryFile() as uncompressedfile: + remove_dups_and_excl(mergefile, exclusionregexes, targetips, uncompressedfile) + compress_file(uncompressedfile, targetips, finalfile) + elif settings["minimise"]: + with tempfile.NamedTemporaryFile() as unminimisedfile: + remove_dups_and_excl(mergefile, exclusionregexes, targetips, unminimisedfile) + minimise_file(unminimisedfile, targetips, finalfile) + else: + remove_dups_and_excl(mergefile, exclusionregexes, targetips, finalfile) numberofrules = settings["numberofrules"] outputsubfolder = settings["outputsubfolder"] @@ -312,6 +336,7 @@ def main(): outputsubfolder=outputsubfolder, skipstatichosts=skipstatichosts, nounifiedhosts=nounifiedhosts, + addsystemhostname=settings["addsystemhostname"], ) finalfile.close() @@ -430,6 +455,47 @@ def prompt_for_exclusions(skipprompt): return False +def prompt_for_v4_v6(skipprompt): + """ + Prompt the user to add v4 or v6 rules. + + Parameters + ---------- + skipprompt : bool + Whether or not to skip prompting. + If true, the function returns immediately. + + Returns + ------- + generatev4 : bool + Whether or not we should generate v4 rules. + generatev6 : bool + Whether or not we should generate v6 rules. + """ + + prompt_unformatted = ( + "Do you want to add {} rules?\n" + "This roughly doubles the file size, but may be " + "necessary to actually block hosts\n" + "depending on your network configuration." + ) + prompt_v4 = prompt_unformatted.format("IPv4") + prompt_v6 = prompt_unformatted.format("IPv6") + + generatev4 = True + generatev6 = True + + if not skipprompt: + if not query_yes_no(prompt_v4): + print("OK, we'll skip adding v4 rules.") + generatev4 = False + if not query_yes_no(prompt_v6): + print("OK, we'll skip adding v6 rules.") + generatev6 = False + + return generatev4, generatev6 + + def prompt_for_flush_dns_cache(flushcache, promptflush): """ Prompt the user to flush the DNS cache. @@ -843,7 +909,7 @@ def create_initial_file(**initial_file_params): return mergefile -def compress_file(inputfile, targetip, outputfile): +def compress_file(inputfile, targetips, outputfile, maxdomainsperline=9): """ Reduce the file dimension removing non-necessary lines (empty lines and comments) and putting multiple domains in each line. @@ -854,38 +920,43 @@ def compress_file(inputfile, targetip, outputfile): ---------- inputfile : file The file object that contains the hostnames that we are reducing. - targetip : str + targetips : list[str] The target IP address. outputfile : file The file object that will contain the reduced hostnames. + maxdomainsperline : int + The maximum number of domains per line. Defaults to 9. """ inputfile.seek(0) # reset file pointer write_data(outputfile, "\n") - targetip_len = len(targetip) - lines = [targetip] - lines_index = 0 + ip_lines = {ip: [] for ip in targetips} + for line in inputfile.readlines(): - line = line.decode("UTF-8") + line = line.decode("UTF-8").strip() - if line.startswith(targetip): - if lines[lines_index].count(" ") < 9: - lines[lines_index] += ( - " " + line[targetip_len : line.find("#")].strip() # noqa: E203 - ) - else: - lines[lines_index] += "\n" - lines.append(line[: line.find("#")].strip()) - lines_index += 1 + if not line or line.startswith("#"): + continue - for line in lines: - write_data(outputfile, line) + for targetip in targetips: + if line.startswith(targetip): + domain_part = line[len(targetip) : line.find("#")].strip() if "#" in line else line[len[targetip] :].strip() + + if len(ip_lines[targetip]) == 0 or ip_lines[targetip][-1].count(" ") >= maxdomainsperline: + ip_lines[targetip].append(targetip + " " + domain_part) + else: + ip_lines[targetip][-1] += " " + domain_part + break + + for targetip in targetips: + for line in ip_lines[targetip]: + write_data(outputfile, line + "\n") inputfile.close() -def minimise_file(inputfile, targetip, outputfile): +def minimise_file(inputfile, targetips, outputfile): """ Reduce the file dimension removing non-necessary lines (empty lines and comments). @@ -894,8 +965,8 @@ def minimise_file(inputfile, targetip, outputfile): ---------- inputfile : file The file object that contains the hostnames that we are reducing. - targetip : str - The target IP address. + targetips : list[str] + The target IP addresses. outputfile : file The file object that will contain the reduced hostnames. """ @@ -907,8 +978,9 @@ def minimise_file(inputfile, targetip, outputfile): for line in inputfile.readlines(): line = line.decode("UTF-8") - if line.startswith(targetip): - lines.append(line[: line.find("#")].strip() + "\n") + for targetip in targetips: + if line.startswith(targetip): + lines.append(line[: line.find("#")].strip() + "\n") for line in lines: write_data(outputfile, line) @@ -916,7 +988,7 @@ def minimise_file(inputfile, targetip, outputfile): inputfile.close() -def remove_dups_and_excl(mergefile, exclusionregexes, outputfile=None): +def remove_dups_and_excl(mergefile, exclusionregexes, targetips, outputfile): """ Remove duplicates and remove hosts that we are excluding. @@ -929,9 +1001,10 @@ def remove_dups_and_excl(mergefile, exclusionregexes, outputfile=None): The file object that contains the hostnames that we are pruning. exclusionregexes : list The list of regex patterns used to exclude domains. + targetips : list[str] + The list of target IP addresses outputfile : file - The file object in which the result is written. If None, the file - 'settings["outputpath"]' will be created. + The file object to which the result is written. """ numberofrules = settings["numberofrules"] @@ -944,13 +1017,7 @@ def remove_dups_and_excl(mergefile, exclusionregexes, outputfile=None): if line and not line.startswith("#"): settings["exclusions"].append(line) - if not os.path.exists(settings["outputpath"]): - os.makedirs(settings["outputpath"]) - - if outputfile is None: - finalfile = open(path_join_robust(settings["outputpath"], "hosts"), "w+b") - else: - finalfile = outputfile + finalfile = outputfile # analyze any post.json here post_json_path = os.path.join(os.path.dirname(finalfile.name), "post.json") @@ -998,28 +1065,29 @@ def remove_dups_and_excl(mergefile, exclusionregexes, outputfile=None): if "@" in strippedrule: continue - # Normalize rule - hostname, normalized_rule = normalize_rule( - strippedrule, - targetip=settings["targetip"], - keep_domain_comments=settings["keepdomaincomments"], - ) - for exclude in exclusions: if re.search(r"(^|[\s\.])" + re.escape(exclude) + r"\s", line): write_line = False break - if normalized_rule and (hostname not in hostnames) and write_line: - write_data(finalfile, normalized_rule) + # Normalize rule + hostname, has_rule = normalize_rule( + strippedrule, + targetip="0.0.0.0", + keep_domain_comments=settings["keepdomaincomments"], + ) + if write_line and (hostname not in hostnames) and has_rule: hostnames.add(hostname) numberofrules += 1 + for targetip in targetips: + _, normalized_rule = normalize_rule( + strippedrule, + targetip=targetip, + keep_domain_comments=settings["keepdomaincomments"], + ) + write_data(finalfile, normalized_rule) settings["numberofrules"] = numberofrules - mergefile.close() - - if outputfile is None: - return finalfile def normalize_rule(rule, targetip, keep_domain_comments): @@ -1064,7 +1132,9 @@ def normalize_response( and spacing reformatted. """ - rule = "%s %s" % (targetip, extracted_hostname) + hostname = extracted_hostname.lower() + + rule = "%s %s" % (targetip, hostname) if keep_domain_comments and extracted_suffix: if not extracted_suffix.strip().startswith("#"): @@ -1073,7 +1143,7 @@ def normalize_response( else: rule += " %s" % extracted_suffix - return extracted_hostname, rule + "\n" + return hostname, rule + "\n" def is_ip(dataset: str) -> bool: """ @@ -1097,6 +1167,37 @@ def is_ip(dataset: str) -> bool: except ValueError: return False + def is_valid_hostname(dataset: str, min_labels: int = 1) -> bool: + """ + Validates a hostname according to RFC 1123 Section 2.5. + + - Hostname must be <= 255 characters. + - Each label (dot-separated part) must be <= 63 characters. + - Labels may contain letters, digits, and hyphens. + - Labels must not start or end with a hyphen (but may start with a digit!). + + Parameters + ---------- + + dataset : str + The dataset to validate + min_labels : int + The minimum number of labels to require. + + Returns + ------ + is_valid_hostname: bool + Whether the dataset is a hostname + """ + if len(dataset) > 255: + return False + if dataset.endswith("."): + dataset = dataset[:-1] + + label_regex = re.compile(r"^(?!-)[A-Za-z0-9-]{1,63}(?= min_labels + def belch_unwanted(unwanted: str) -> Tuple[None, None]: """ Belches unwanted to screen. @@ -1122,68 +1223,28 @@ def belch_unwanted(unwanted: str) -> Tuple[None, None]: first try: IP followed by domain """ - static_ip_regex = r"^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$" split_rule = rule.split(maxsplit=1) if is_ip(split_rule[0]): - # Assume that the first item is an IP address following the rule. + # Assume that the first item is an IP address, discard as we're replacing it with targetip - if " " or "\t" in split_rule[-1]: - try: - # Example: 0.0.0.0 example.org # hello, world! - hostname, suffix = split_rule[-1].split(maxsplit=1) - except ValueError: - # Example: 0.0.0.0 example.org[:space:] - hostname, suffix = split_rule[-1], None - else: - # Example: 0.0.0.0 example.org - hostname, suffix = split_rule[-1], None - - hostname = hostname.lower() - - if ( - is_ip(hostname) - or re.search(static_ip_regex, hostname) - or "." not in hostname - or ".." in hostname - or "." in hostname[-1] - or "/" in hostname - or ":" in hostname - ): - # Example: 0.0.0.0 127.0.0.1 - - # If the hostname is: - # - an IP - or looks like it, - # - doesn't contain dots, or - # - contains repeated dots, - # - ends in a dot, or - # - contains a slash, or - # - contains a colon, - # - contains an underscore, - # we don't want to normalize it. - return belch_unwanted(rule) - - return normalize_response(hostname, suffix) - - if ( - not re.search(static_ip_regex, split_rule[0]) - and ":" not in split_rule[0] - and ".." not in split_rule[0] - and "/" not in split_rule[0] - and "." in split_rule[0] - ): - # Deny anything that looks like an IP; doesn't container dots or INVALID. + split_rule = split_rule[-1].split(maxsplit=1) - try: - hostname, suffix = split_rule - except ValueError: - hostname, suffix = split_rule[0], None + if is_valid_hostname(split_rule[0]): + # Assume that the next item is the hostname - hostname = hostname.lower() + hostname = split_rule[0] + suffix = split_rule[1] if len(split_rule) > 1 else None + else: + return belch_unwanted(rule) - return normalize_response(hostname, suffix) + # If the hostname is invalid or looks like an IP, we don't want to normalize it. + # If we were wanting 100% compliance with the RFC we'd use min_labels=1, + # but we want to stop TLDs from being blocked so we set this to 2. + if not is_valid_hostname(hostname, 2) or is_ip(hostname): + return belch_unwanted(rule) - return belch_unwanted(rule) + return normalize_response(hostname, suffix) def strip_rule(line): @@ -1221,6 +1282,7 @@ def write_opening_header(finalfile, **headerparams): 3) outputsubfolder 4) skipstatichosts 5) nounifiedhosts + 6) hostfilename """ finalfile.seek(0) # Reset file pointer. @@ -1338,7 +1400,7 @@ def write_opening_header(finalfile, **headerparams): write_data(finalfile, "ff02::3 ip6-allhosts\n") write_data(finalfile, "0.0.0.0 0.0.0.0\n") - if platform.system() == "Linux": + if platform.system() == "Linux" and "addsystemhostname" in headerparams.keys() and headerparams["addsystemhostname"]: write_data(finalfile, "127.0.1.1 " + socket.gethostname() + "\n") write_data(finalfile, "127.0.0.53 " + socket.gethostname() + "\n")