words/scripts/gen_wordlist.py

#!/usr/bin/env python3

import sys
import json

class WordListItem:
    def __init__(self, line):
        word = line.split('\t')

        self.word = word[1] if word[1].isalpha() else word[3]
        self.pos = word[2]
        self.frequency = int(word[4])


    def __repr__(self):
        return f"<WordListItem {self.word=} {self.pos=} {self.frequency=}>"


def get_args():
    """ Get command line arguments """

    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('wordlist')
    parser.add_argument('word_length', type=int)
    parser.add_argument('frequency_min', type=int)
    parser.add_argument('allowedtypelist')
    return parser.parse_args()


def main(args):
    """ Entry point for script """

    with open(args.allowedtypelist) as fp:
        allowed_types = fp.read().split('\n')

    types = set()
    with open(args.wordlist) as fp:
        words = [ WordListItem(line) for line in fp.read().strip().lower().split('\n') ]

    [ types.add(word.pos) for word in words ]

    words = [ word.word for word in words if word.frequency >= args.frequency_min and word.word.isalpha() and len(word.word) == args.word_length and word.pos in allowed_types ]

    words.sort()
    # remove duplicates
    words = list(set(words))
    print(f"wordlist = {json.dumps(words)}")
    print(f"{args=}", file=sys.stderr)
    print(f"{len(words)=}", file=sys.stderr)
    print(f"{types=}", file=sys.stderr)
    print(f"{'cares' in words=}", file=sys.stderr)
    print(f"{'ideas' in words=}", file=sys.stderr)
    print(f"{'prose' in words=}", file=sys.stderr)

    return 0


if __name__ == '__main__':
    try:
        sys.exit(main(get_args()))
    except KeyboardInterrupt:
        sys.exit(0)