#!/usr/bin/env python3

# Copyright (c) 2021-2024 Chris Pressey, Cat's Eye Technologies
# This file is distributed under the MIT license.  For more information, see
# the file LicenseRef-MIT-X-Dissociated-Parse.txt in the LICENSES directory.
# SPDX-License-Identifier: LicenseRef-MIT-X-Dissociated-Parse

import json
import os
import re
import sys
import subprocess
from tqdm import tqdm

from linktree import make_tree


def main():
    freq = {}

    with open('data/all-sentences.json', 'r') as f:
        data = json.loads(f.read())

    sentences = []
    for n, sent in tqdm(enumerate(data['sentences'])):
        s = ''
        last_word = None
        for word in sent:
            s += ' '
            s += word
            #sys.stdout.write(' ')
            #sys.stdout.write(word)
            last_word = word
        sentences.append(s)
        #sys.stdout.write('\n\n')

    trees = []
    for s in tqdm(sentences):

        if '"' in s:
            continue
        if ';' in s:
            continue
        if ':' in s:
            continue
        if '—' in s:
            continue
        if '!' in s:
            continue
        if 'beautifully' in s:
            continue

        tree = make_tree(s)
        trees.append(tree)

    with open('data/trees.json', 'w') as f:
        f.write(json.dumps({
            'trees': trees
        }, indent=4))

main()