diff --git a/scripts/gen_wordlist.py b/scripts/gen_wordlist.py index dd10212..3f8e23b 100755 --- a/scripts/gen_wordlist.py +++ b/scripts/gen_wordlist.py @@ -23,6 +23,8 @@ def main(args): words = [ word[0] for word in words if word[1] > args.frequency_min and word[0].isalpha() and len(word[0]) == args.word_length ] words.sort(key=lambda word: word[1]) + # remove duplicates + words = list(set(words)) print(f"wordlist = {json.dumps(words)}") print(f"{args}", file=sys.stderr) print(f"{len(words)=}", file=sys.stderr)