#!/usr/bin/env python3 """ gronk --- view your notes as a static html site """ import argparse import os from pathlib import Path import shutil import sys import subprocess import copy import time import magic import regex as re import pprint import frontmatter import jinja2 import requests GRONK_COMMIT = "dev" PANDOC_SERVER_URL = os.getenv("PANDOC_SERVER_URL", r"http://localhost:3030/") PANDOC_TIMEOUT = int(os.getenv("PANDOC_TIMEOUT", "120")) GRONK_CSS_DIR = Path(os.getenv("GRONK_CSS_DIR", "/opt/gronk/css")) GRONK_JS_DIR = Path(os.getenv("GRONK_JS_DIR", "/opt/gronk/js")) GRONK_TEMPLATES_DIR = Path( os.getenv("GRONK_TEMPLATES_DIR", "/opt/gronk/templates/")) JINJA_ENV = jinja2.Environment( loader=jinja2.FileSystemLoader(searchpath=GRONK_TEMPLATES_DIR), autoescape=jinja2.select_autoescape) JINJA_TEMPLATE_TEXTARTICLE = JINJA_ENV.get_template("article-text.html") JINJA_TEMPLATE_HOME_INDEX = JINJA_ENV.get_template("home.html") JINJA_TEMPLATE_INDEX = JINJA_ENV.get_template("index.html") JINJA_TEMPLATE_ARTICLE = JINJA_ENV.get_template("article.html") JINJA_TEMPLATE_PERMALINK = JINJA_ENV.get_template("permalink.html") LICENSE = None FILEMAP = None class FileMap: """ this class is used to read file properties, inherit properties, and have a centralised place to access them """ def __init__(self, input_dir, output_dir): self._map = {} self.input_dir = Path(input_dir) self.output_dir = Path(output_dir) @staticmethod def _path_to_key(path): return str(path) @staticmethod def is_plaintext(filename): return re.match(r'^text/', magic.from_file(str(filename), mime=True)) is not None def add(self, filepath): filepath = Path(filepath) if filepath.is_dir(): properties = self._get_directory_properties(filepath) else: properties = self._get_file_properties(filepath) properties['src_path'] = filepath properties['dst_path'] = self._get_output_filepath(filepath) self._map[self._path_to_key(filepath)] = properties def get(self, filepath, default=None, raw=False): """ get the properties of a file at a filepath raw=True to not inherit properties """ # TODO maybe store properties of a file once it's in built and mark it # as built? might save time but also cba if self._path_to_key(filepath) not in self._map.keys(): self.add(filepath) properties = copy.deepcopy( self._map.get(self._path_to_key(filepath), default)) if raw: return properties parent = filepath while True: parent = parent.parent if parent == Path('.'): break parent_properties = self.get(parent, raw=True) # TODO inherit any property that isn't defined, append any lists # that exist properties['tags'] = properties.get( 'tags', []) + parent_properties.get('tags', []) if parent == self.input_dir: break return properties def _get_directory_properties(self, filepath: Path, include_index_entries=True): post = { 'title': filepath.name, 'content_after_search': False, 'automatic_index': True, 'search_bar': True, 'tags': [], } if 'readme.md' in [f.name for f in filepath.iterdir()]: with open(filepath.joinpath('readme.md'), encoding='utf-8') as file_pointer: for key, val in frontmatter.load( file_pointer).to_dict().items(): post[key] = val if 'content' in post.keys(): post['content'] = render_markdown(post['content']) post['is_dir'] = True if include_index_entries: post['index_entries'] = self._get_index_entries(filepath) return post def _get_index_entries(self, filepath): entries = [] for path in filepath.iterdir(): if '.git' in path.parts: continue if path.is_dir(): entry = self._get_directory_properties( path, include_index_entries=False) else: entry = self._get_file_properties(path) entry['path'] = self._get_output_filepath(path)['web'] entries.append(entry) entries.sort(key=lambda entry: str(entry.get('title', '')).lower()) entries.sort(key=lambda entry: entry['is_dir'], reverse=True) return entries def _get_file_properties(self, filepath): post = {'title': filepath.name} if filepath.suffix == '.md': with open(filepath, encoding='utf-8') as file_pointer: post = frontmatter.load(file_pointer).to_dict() # don't store file contents in memory if 'content' in post.keys(): del post['content'] post['is_dir'] = False return post def _get_output_filepath(self, input_filepath): def webpath(filepath): return Path('/notes').joinpath( filepath.relative_to(self.output_dir)) r = {} r['raw'] = self.output_dir.joinpath( input_filepath.relative_to(self.input_dir)) r['web'] = webpath(r['raw']) if input_filepath.is_dir(): return r if input_filepath.suffix == '.md': r['html'] = self.output_dir.joinpath( input_filepath.relative_to( self.input_dir)).with_suffix('.html') r['web'] = webpath(r['html']) elif self.is_plaintext(input_filepath): r['html'] = self.output_dir.joinpath( input_filepath.relative_to( self.input_dir)).with_suffix(input_filepath.suffix + '.html') r['raw'] = self.output_dir.joinpath( input_filepath.relative_to(self.input_dir)) r['web'] = webpath(r['html']) return r def to_list(self): return [val for _, val in self._map.items()] def to_search_data(self): """ returns list of every file in map """ r = [] for _, val in self._map.items(): r.append({ 'title': val.get('title', ''), 'tags': val.get('tags', []), 'path': str(val['dst_path']['web']), 'is_dir': val['is_dir'] }) return r def get_uuid_map(self): d = {} for _, val in self._map.items(): if 'uuid' not in val.keys(): continue d[val['uuid']] = str(val['dst_path']['web']) return d def update_required(src_filepath, output_filepath): """ check if file requires an update, return boolean """ return not output_filepath.exists() or src_filepath.stat( ).st_mtime > output_filepath.stat().st_mtimeme() def get_args(): """ Get command line arguments """ parser = argparse.ArgumentParser() parser.add_argument('notes', type=Path) parser.add_argument('-o', '--output-dir', type=Path, default='web') parser.add_argument( '-F', '--force', action="store_true", help= "Generate new output html even if source file was modified before output html" ) return parser.parse_args() def render_markdown_file(input_filepath): """ render markdown file to file write markdown file to args.output_dir in html, return list of tuple of output filepath, frontmatter post """ with open(input_filepath, encoding='utf-8') as file_pointer: content = frontmatter.load(file_pointer).content properties = FILEMAP.get(input_filepath) html = render_markdown(content) html = JINJA_TEMPLATE_ARTICLE.render( license=LICENSE, content=html, lecture_slides=properties.get("lecture_slides"), lecture_notes=properties.get("lecture_notes"), uuid=properties.get("uuid"), tags=properties.get("tags"), author=properties.get("author"), title=properties.get("title")) properties['dst_path']['html'].write_text(html) def render_plaintext_file(input_filepath): """ render plaintext file to file copy plaintext file into a html preview, copy raw to output dir return list of tuple of output filepath, empty dict """ raw_content = input_filepath.read_text() properties = FILEMAP.get(input_filepath) html = JINJA_TEMPLATE_TEXTARTICLE.render(license=LICENSE, **properties) properties['dst_path']['raw'].write_text(raw_content) properties['dst_path']['html'].write_text(html) def render_generic_file(input_filepath): """ render generic file to file copy generic file into to output_dir return list of tuple of output filepath, empty dict """ properties = FILEMAP.get(input_filepath) output_filepath = properties['dst_path']['raw'] shutil.copyfile(input_filepath, output_filepath) def render_file(input_filepath): """ render any file by detecting type and applying appropriate type write input_filepath to correct file in args.output_dir in appropriate formats, return list of tuples of output filepath, frontmatter post """ if input_filepath.suffix == '.md': return render_markdown_file(input_filepath) if FileMap.is_plaintext(input_filepath): return render_plaintext_file(input_filepath) return render_generic_file(input_filepath) def render_markdown(content): """ render markdown to html """ post_body = { 'text': content, 'toc-depth': 6, 'highlight-style': 'pygments', 'html-math-method': 'mathml', 'to': 'html', 'files': { 'data/data/abbreviations': '', }, 'standalone': False, } headers = {'Accept': 'application/json'} response = requests.post(PANDOC_SERVER_URL, headers=headers, json=post_body, timeout=PANDOC_TIMEOUT) response = response.json() # TODO look at response['messages'] and log them maybe? # https://github.com/jgm/pandoc/blob/main/doc/pandoc-server.md#response return response['output'] def process_home_index(args, notes_git_head_sha1=None): """ create home index.html in output_dir """ post = {'title': 'gronk', 'content': ''} custom_content_file = args.notes.joinpath('readme.md') if custom_content_file.is_file(): fmpost = frontmatter.loads(custom_content_file.read_text()).to_dict() for key, val in fmpost.items(): post[key] = val post['content'] = render_markdown(post['content']) html = JINJA_TEMPLATE_HOME_INDEX.render( gronk_commit=GRONK_COMMIT, search_data=FILEMAP.to_search_data(), notes_git_head_sha1=notes_git_head_sha1, post=post) args.output_dir.joinpath('index.html').write_text(html) def generate_permalink_page(output_dir): """ create the directory and index.html for redirecting permalinks """ dir = output_dir.joinpath('permalink') dir.mkdir(exist_ok=True) dir.joinpath('index.html').write_text( JINJA_TEMPLATE_PERMALINK.render(gronk_commit=GRONK_COMMIT, data=FILEMAP.get_uuid_map())) def generate_tag_browser(output_dir): """ generate a directory that lets you groub by and browse by any given tag. e.g. tags, authors """ tags = {} for post in FILEMAP.to_list(): post['path'] = post['dst_path']['web'] if 'tags' not in post.keys(): continue for tag in post['tags']: if tag not in tags.keys(): tags[tag] = [] tags[tag].append(post) for tag, index_entries in tags.items(): output_file = output_dir.joinpath(tag, 'index.html') output_file.parent.mkdir(exist_ok=True, parents=True) output_file.write_text( JINJA_TEMPLATE_INDEX.render( gronk_commit=GRONK_COMMIT, automatic_index=True, search_bar=True, title=tag, index_entries=[{ 'title': entry.get('title', ''), 'is_dir': entry.get('is_dir', False), 'path': str(entry.get('path', Path(''))), } for entry in index_entries], )) output_file = output_dir.joinpath('index.html') output_file.parent.mkdir(exist_ok=True, parents=True) output_file.write_text( JINJA_TEMPLATE_INDEX.render(automatic_index=True, gronk_commit=GRONK_COMMIT, search_bar=True, title='tags', index_entries=[{ 'path': tag, 'title': tag, 'is_dir': False, } for tag in tags.keys()])) def main(args): """ Entry point for script """ start_time = time.time() global LICENSE global FILEMAP FILEMAP = FileMap(args.notes, args.output_dir.joinpath('notes')) # TODO have some sort of 'site rebuild in progress - come back in a minute # or two!' or auto checking/refreshing page for when site is being built if args.output_dir.is_file(): print(f"Output directory ({args.output_dir}) cannot be a file.") args.output_dir.mkdir(parents=True, exist_ok=True) # attempt to get licensing information license_path = args.notes.joinpath("LICENSE") if license_path.exists(): LICENSE = license_path.read_text() # TODO git commit log integration for root_str, _, files in os.walk(args.notes): root = Path(root_str) if '.git' in root.parts: continue root_properties = FILEMAP.get(root) root_properties['dst_path']['raw'].mkdir(parents=True, exist_ok=True) #pprint.pprint(root_properties) html = JINJA_TEMPLATE_INDEX.render( gronk_commit=GRONK_COMMIT, title=root_properties.get('title', ''), content=root_properties.get('content', ''), content_after_search=root_properties['content_after_search'], automatic_index=root_properties['automatic_index'], search_bar=root_properties['search_bar'], index_entries=[{ 'title': entry.get('title', ''), 'is_dir': entry.get('is_dir', False), 'path': str(entry.get('path', Path(''))), } for entry in root_properties.get('index_entries', '')], ) root_properties['dst_path']['raw'].joinpath('index.html').write_text( html) # render each file for file in files: # don't render readme.md as index as it is used for directory if file == "readme.md": continue render_file(root.joinpath(file)) process_home_index(args) # copy styling and js scripts necessary for function shutil.copytree(GRONK_CSS_DIR, args.output_dir.joinpath('css'), dirs_exist_ok=True) shutil.copytree(GRONK_JS_DIR, args.output_dir.joinpath('js'), dirs_exist_ok=True) generate_tag_browser(args.output_dir.joinpath('tags')) generate_permalink_page(args.output_dir) elapsed_time = time.time() - start_time print(f"generated notes {elapsed_time=}") return 0 def start_pandoc_server(): """ attempt to get the version of pandoc server in a loop until it is successful and return version as string """ start_time = time.time() process = subprocess.Popen(["/usr/bin/pandoc-server"], stdout=subprocess.PIPE) version = None while True: try: resp = requests.get(f"{PANDOC_SERVER_URL}/version") version = resp.content.decode('utf-8') break except requests.ConnectionError: time.sleep(0.1) rc = process.poll() if rc is not None: print(f"PANDOC SERVER FAILED TO START: {rc=}") print(process.stdout.read().decode("utf-8")) raise Exception("Pandoc server failed to start") elapsed_time = time.time() - start_time print(f"pandoc-server started {version=} {elapsed_time=}") return process # TODO implement useful logging and debug printing if __name__ == '__main__': pandoc_process = start_pandoc_server() try: sys.exit(main(get_args())) except KeyboardInterrupt: sys.exit(0) finally: pandoc_process.kill()