From 595a70e40c96e0b5373e0517c7b506cdaa896765 Mon Sep 17 00:00:00 2001
From: Alvie Rahman <alvierahman90@gmail.com>
Date: Thu, 27 Jan 2022 20:16:33 +0000
Subject: [PATCH] remove duplicate words

---
 scripts/gen_wordlist.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/gen_wordlist.py b/scripts/gen_wordlist.py
index dd10212..3f8e23b 100755
--- a/scripts/gen_wordlist.py
+++ b/scripts/gen_wordlist.py
@@ -23,6 +23,8 @@ def main(args):
     words = [ word[0] for word in words if word[1] > args.frequency_min and word[0].isalpha() and len(word[0]) == args.word_length ]
 
     words.sort(key=lambda word: word[1])
+    # remove duplicates
+    words = list(set(words))
     print(f"wordlist = {json.dumps(words)}")
     print(f"{args}", file=sys.stderr)
     print(f"{len(words)=}", file=sys.stderr)