git @ Cat's Eye Technologies The-Dossier / master script / localize-images.py
master

Tree @master (Download .tar.gz)

localize-images.py @masterraw · history · blame

#!/usr/bin/env python3

# SPDX-FileCopyrightText: Chris Pressey, the original author of this work, has dedicated it to the public domain.
# For more information, please refer to <https://unlicense.org/>
# SPDX-License-Identifier: Unlicense

from argparse import ArgumentParser
import json
import logging
import os
import sys
from urllib.parse import quote, urlsplit

import requests

from feedmark.checkers import Schema
from feedmark.loader import read_document_from
from feedmark.formats.markdown import feedmark_markdownize


logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)


SKIPPED_SECTIONS = [
    "Congo Bongo"
]


def main(args):
    argparser = ArgumentParser()

    argparser.add_argument('docs', nargs='+', metavar='FILENAME', type=str)

    options = argparser.parse_args(args)

    for filename in options.docs:
        document = read_document_from(filename)

        logger.info("document: %s", document)
        image_dir = os.path.dirname(filename) + "/images"
        try:
            os.makedirs(image_dir)
        except FileExistsError:
            pass

        schema = None
        schema_name = document.properties.get('schema')
        if schema_name:
            schema_filename = "schema/{}.md".format(schema_name)
            schema_document = read_document_from(schema_filename)
            schema = Schema(schema_document)
            results = schema.check_documents([document])
            if results:
                sys.stdout.write(json.dumps(results, indent=4, sort_keys=True))
                sys.exit(1)
        else:
            continue

        for section in document.sections:
            logger.info("section: %s", section)
            if section.title in SKIPPED_SECTIONS:
                logger.info("SKIPPING because configured so")
                continue
            new_images = []
            for image_record in section.images:
                alt_text = image_record["description"]
                url = image_record["source"]
                logger.info("link: %s %s", alt_text, url)
                if alt_text.startswith(('screenshot', 'cover', 'montage')) and url.startswith('http'):
                    extension = None
                    for possible in [".png", ".jpg", ".JPG", ".gif"]:
                        if url.endswith(possible):
                            extension = possible
                            break
                    assert extension is not None, url
                    local_filename = f"{image_dir}/{section.title}{extension}"
                    path = urlsplit(url).path

                    r = requests.get(url)
                    filesize = None
                    if os.path.exists(local_filename):
                        logger.warning(f"{local_filename} already exists, SKIPPING DOWNLOAD")
                    else:
                        with open(local_filename, 'wb') as f:
                            filesize = len(r.content)
                            f.write(r.content)
                        logger.info(f"downloaded {filesize} bytes to {local_filename}")

                    if url.startswith('https://catseye.tc/static/archive'):
                        path = path[16:]
                        orig_url = "https://" + path.replace("%252F", "/")
                    else:
                        orig_url = url
                    license_filename = f"{local_filename}.license"
                    if os.path.exists(license_filename):
                        logger.warning(f"{license_filename} already exists, SKIPPING CREATION")
                    with open(license_filename, "w") as f:
                        f.write(f"""\
SPDX-FileCopyrightText: This copyrighted screenshot is used under the auspices of Fair Dealing.

SPDX-License-Identifier: LicenseRef-Fair-Dealing

SPDX-PackageDownloadLocation: {orig_url}
""")
                    new_images.append({
                        "description": alt_text,
                        "source": "images/" + quote(os.path.basename(local_filename)) + "?raw=true",
                    })
                else:
                    logger.info("image link unprocessable, SKIPPING this link")
                    new_images.append(image_record)
            section.images = new_images
        s = feedmark_markdownize(document, schema=schema)
        with open(document.filename, 'w') as f:
            f.write(s)


if __name__ == '__main__':
    main(sys.argv[1:])