-
+ {% if license %}
- Commit log (file history) -
- $filehistory$ -License
-$licenseFull$+
{{ license }}
\n" + "
\n".join(filehistory) + "" - - return filehistory +JINJA_TEMPLATES = {} +JINJA_TEMPLATE_TEXTARTICLE = JINJA_ENV.get_template("textarticle.html") +JINJA_TEMPLATE_HOME_INDEX = JINJA_ENV.get_template("home_index.html") +JINJA_TEMPLATE_DIRECTORY_INDEX = JINJA_ENV.get_template("index.html") +JINJA_TEMPLATE_ARTICLE = JINJA_ENV.get_template("article.html") -def get_dirs_to_index(folder): - r = [] - - for root, folders, files in os.walk(folder): - if pathlib.Path(os.path.join(root, folder)).is_relative_to(folder.joinpath('permalink')): - continue - [r.append(os.path.join(root, folder)) for folder in folders] - - return r +LICENSE = None +GIT_REPO = None +FILEMAP = None -def update_required(src_filename, output_filename): - return not os.path.exists(output_filename) or os.path.getmtime(src_filename) > os.path.getmtime(output_filename) +def update_required(src_filepath, output_filepath): + """ + check if file requires an update, + return boolean + """ + return not output_filepath.exists() or src_filepath.stat().st_mtime > output_filepath.stat().st_mtimeme() + def get_args(): """ Get command line arguments """ - import argparse parser = argparse.ArgumentParser() - parser.add_argument('notes', type=pathlib.Path) - parser.add_argument('-o', '--output-dir', type=pathlib.Path, default='web') - parser.add_argument('-t', '--template', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/article.html')) - parser.add_argument('-H', '--template-text-head', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/textarticlehead.html')) - parser.add_argument('-f', '--template-text-foot', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/textarticlefoot.html')) - parser.add_argument('-i', '--template-index-head', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/indexhead.html')) - parser.add_argument('-I', '--template-index-foot', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/indexfoot.html')) - parser.add_argument('-s', '--stylesheet', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/styles.css')) - parser.add_argument('--home_index', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/home_index.html')) - parser.add_argument('--permalink_index', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/permalink_index.html')) - parser.add_argument('-e', '--extra-index-content', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/extra_index_content.html')) - parser.add_argument('-n', '--index-article-names', action='append', default=['index.md']) - parser.add_argument('-F', '--force', action="store_true", help="Generate new output html even if source file was modified before output html") - parser.add_argument('--fuse', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/fuse.js')) - parser.add_argument('--searchjs', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/search.js')) - parser.add_argument('--indexsearchjs', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/indexsearch.js')) - parser.add_argument('--permalinkjs', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/permalink.js')) - parser.add_argument('--tocsearchjs', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/toc_search.js')) - parser.add_argument('--toc-depth', type=int, default=6, dest='toc_depth') + parser.add_argument('notes', type=Path) + parser.add_argument('-o', '--output-dir', type=Path, default='web') + parser.add_argument('-F', '--force', action="store_true", + help="Generate new output html even if source file was modified before output html") return parser.parse_args() +def render_markdown_file(input_filepath): + """ + render markdown file to file + write markdown file to args.output_dir in html, + return list of tuple of output filepath, frontmatter post + """ + print(f"render_markdown_file({input_filepath})") + with open(input_filepath, encoding='utf-8') as file_pointer: + content = frontmatter.load(file_pointer).content + + properties = FILEMAP.get(input_filepath) + + # TODO pandoc no longer handles template due to metadata passing issues, use jinja to fill in the metadata + html = render_markdown(content) + + with open(properties['dst_path']['html'], 'w+', encoding='utf-8') as file_pointer: + file_pointer.write(html) + + +def render_plaintext_file(input_filepath): + """ + render plaintext file to file + copy plaintext file into a html preview, copy raw to output dir + return list of tuple of output filepath, empty dict + """ + + with open(input_filepath, encoding='utf-8') as file_pointer: + raw_content = file_pointer.read() + + properties = FILEMAP.get(input_filepath) + + html = JINJA_TEMPLATE_TEXTARTICLE.render(license = LICENSE, **properties) + + with open(properties['dst_path']['raw'], "w+", encoding='utf-8') as file_pointer: + file_pointer.write(raw_content) + + with open(properties['dst_path']['html'], "w+", encoding='utf-8') as file_pointer: + file_pointer.write(html) + + +def render_generic_file(input_filepath): + """ + render generic file to file + copy generic file into to output_dir + return list of tuple of output filepath, empty dict + """ + properties = FILEMAP.get(input_filepath) + output_filepath = properties['dst_path']['raw'] + shutil.copyfile(input_filepath, output_filepath) + + +def render_file(input_filepath): + """ + render any file by detecting type and applying appropriate type + write input_filepath to correct file in args.output_dir in appropriate formats, + return list of tuples of output filepath, frontmatter post + """ + + if input_filepath.suffix == '.md': + return render_markdown_file(input_filepath) + + if FileMap.is_plaintext(input_filepath): + return render_plaintext_file(input_filepath) + + return render_generic_file(input_filepath) + + +def render_markdown(content): + """ + render markdown to html + """ + + post_body = { + 'text': content, + 'toc-depth': 6, + 'highlight-style': 'pygments', + 'html-math-method': 'mathml', + 'to': 'html', + 'files': { + 'data/data/abbreviations': '', + }, + 'standalone': False, + } + + headers = { + 'Accept': 'application/json' + } + + response = requests.post( + PANDOC_SERVER_URL, + headers=headers, + json=post_body, + timeout=PANDOC_TIMEOUT + ) + + response = response.json() + + + # TODO look at response['messages'] and log them maybe? + # https://github.com/jgm/pandoc/blob/main/doc/pandoc-server.md#response + + return response['output'] + + + +def process_home_index(output_dir, search_data, notes_git_head_sha1=None): + """ + create home index.html in output_dir + """ + html = JINJA_TEMPLATE_HOME_INDEX.render( + n2w_commit = N2W_COMMIT, + search_data=search_data, + notes_git_head_sha1=notes_git_head_sha1, + ) + with open(output_dir.joinpath('index.html'), 'w+', encoding='utf-8') as file_pointer: + file_pointer.write(html) + + +def generate_variable_browser(output_dir, posts, variable_name) : + """ + generate a directory that lets you groub by and browse by any given tag. e.g. tags, authors + """ + groups = {} + + for key, post in posts.iter(): + group_val = post.get(variable_name, None) + if group_val is None: + continue + + if group_val not in groups.keys(): + groups[group_val] = [] + + groups[group_val].append(post) + + for group_val, index_entries in groups.iter(): + post = { + 'index_entries': index_entries, + 'title': group_val, + } + + # TODO finish writing function, write page to disk + + + def main(args): """ Entry point for script """ - with open(args.template_text_foot) as fp: - TEXT_ARTICLE_TEMPLATE_FOOT = fp.read() + global LICENSE + global GIT_REPO + global FILEMAP - with open(args.template_text_head) as fp: - TEXT_ARTICLE_TEMPLATE_HEAD = fp.read() - - with open(args.template_index_foot) as fp: - INDEX_TEMPLATE_FOOT = fp.read() - - with open(args.template_index_head) as fp: - INDEX_TEMPLATE_HEAD = fp.read() - - with open(args.extra_index_content) as fp: - EXTRA_INDEX_CONTENT = fp.read() + FILEMAP = FileMap(args.notes, args.output_dir.joinpath('notes')) if args.output_dir.is_file(): print(f"Output directory ({args.output_dir}) cannot be a file.") args.output_dir.mkdir(parents=True, exist_ok=True) - notes_license = "This note has no copyright license.", - print(f"{notes_license=}") + # attempt to get licensing information license_path = args.notes.joinpath("LICENSE") if license_path.exists(): - with open(license_path) as fp: - notes_license = fp.read() + with open(license_path, encoding='utf-8') as file_pointer: + LICENSE = file_pointer.read() - markdown_files, plaintext_files, other_files = get_files(args.notes) + # create git.Repo object if notes dir is a git repo + # TODO git commit log integration + if '.git' in args.notes.iterdir(): + GIT_REPO = git.Repo(args.notes) - all_entries=[] - dirs_with_index_article = [] - tag_dict = {} - permalink_to_filepath = {} - - print(f"{markdown_files=}") - for filename in markdown_files: - print(f"{filename=}") - - # calculate output filename - output_filename = args.output_dir.joinpath('notes').joinpath( - pathlib.Path(filename).relative_to(args.notes) - ).with_suffix('.html') - if os.path.basename(filename) in args.index_article_names: - output_filename = output_filename.parent.joinpath('index.html') - dirs_with_index_article.append(str(output_filename.parent)) - print(f"{output_filename=}") - - # extract tags from frontmatter, save to tag_dict - fm = frontmatter.load(filename) - if isinstance(fm.get('tags'), list): - for tag in list(set(fm.get('tags') + get_inherited_tags(filename, args.notes))): - t = { - 'path': str(pathlib.Path(output_filename).relative_to(args.output_dir)), - 'title': fm.get('title') or pathlib.Path(filename).name - } - if tag in tag_dict.keys(): - tag_dict[tag].append(t) - else: - tag_dict[tag] = [t] - - # find headers in markdown - with open(filename) as fp: - lines = fp.read().split('\n') - header_lines = [] - for line in lines: - if re.match('^#{1,6} \S', line): - header_lines.append(" ".join(line.split(" ")[1:])) - - all_entries.append({ - 'path': '/' + str(pathlib.Path(*pathlib.Path(output_filename).parts[1:])), - 'title': fm.get('title') or pathlib.Path(filename).name, - 'tags': list(set(fm.get('tags'))), - 'headers': header_lines, - 'uuid': fm.get('uuid') - }) - - if 'uuid' in fm.keys(): - permalink_to_filepath[fm['uuid']] = all_entries[-1]['path'] - - # update file if required - if update_required(filename, output_filename) or args.force: - filehistory = git_filehistory(args.notes, filename) - with open(filename) as fp: - article = frontmatter.load(fp) - - article['tags'] += get_inherited_tags(filename, args.notes) - article['tags'] = sorted(list(set(article['tags']))) - article['filehistory'] = filehistory - article['licenseFull'] = notes_license - html = pypandoc.convert_text(frontmatter.dumps(article), 'html', format='md', extra_args=[ - f'--template={args.template}', - '--mathjax', - '--toc', f'--toc-depth={args.toc_depth}' - ]) - pathlib.Path(output_filename).parent.mkdir(parents=True, exist_ok=True) - - with open(output_filename, 'w+') as fp: - fp.write(html) - - print(f"{plaintext_files=}") - for filename in plaintext_files: - filehistory = git_filehistory(args.notes, filename) - title = os.path.basename(filename) - output_filename = str( - args.output_dir.joinpath('notes').joinpath( - pathlib.Path(filename).relative_to(args.notes) - ) - ) + '.html' - print(f"{output_filename=}") - - pathlib.Path(output_filename).parent.mkdir(parents=True, exist_ok=True) - html = re.sub(r'\$title\$', title, TEXT_ARTICLE_TEMPLATE_HEAD) - html = re.sub(r'\$h1title\$', title, html) - html = re.sub(r'\$raw\$', os.path.basename(filename), html) - html = re.sub(r'\$licenseFull\$', notes_license, html) - html = html.replace('$filehistory$', filehistory) - with open(filename) as fp: - html += fp.read().replace("<", "<").replace(">", ">") - html += TEXT_ARTICLE_TEMPLATE_FOOT - - with open(output_filename, 'w+') as fp: - fp.write(html) - all_entries.append({ - 'path': str(pathlib.Path(*pathlib.Path(output_filename).parts[1:])), - 'title': title, - 'tags': [get_inherited_tags(filename, args.notes)], - 'headers': [] - }) - - print(f"{other_files=}") - for filename in other_files: - output_filename = str( - args.output_dir.joinpath('notes').joinpath( - pathlib.Path(filename).relative_to(args.notes) - ) - ) - title = os.path.basename(filename) - pathlib.Path(output_filename).parent.mkdir(parents=True, exist_ok=True) - all_entries.append({ - 'path': str(pathlib.Path(*pathlib.Path(output_filename).parts[1:])), - 'title': title, - 'tags': [get_inherited_tags(filename, args.notes)], - 'headers': [] - }) - shutil.copyfile(filename, output_filename) - - tagdir = args.output_dir.joinpath('.tags') - tagdir.mkdir(parents=True, exist_ok=True) - - for tag in tag_dict.keys(): - html = re.sub(r'\$title\$', f'{tag}', INDEX_TEMPLATE_HEAD) - html = re.sub(r'\$h1title\$', f'tag: {tag}', html) - html = re.sub(r'\$extra_content\$', '', html) - - for entry in tag_dict[tag]: - entry['path'] = '/' + entry['path'] - html += f"" - html += re.sub('\$data\$', json.dumps(tag_dict[tag]), INDEX_TEMPLATE_FOOT) - - with open(tagdir.joinpath(f'{tag}.html'), 'w+') as fp: - fp.write(html) - - dirs_to_index = [args.output_dir.name] + get_dirs_to_index(args.output_dir) - print(f"{dirs_to_index=}") - print(f"{dirs_with_index_article=}") - - for d in dirs_to_index: - print(f"{d in dirs_with_index_article=} {d=}") - if d in dirs_with_index_article: + for root_str, subdirectories, files in os.walk(args.notes): + root = Path(root_str) + if '.git' in root.parts: continue - directory = pathlib.Path(d) - paths = os.listdir(directory) - #print(f"{paths=}") + root_properties = FILEMAP.get(root) + root_properties['dst_path']['raw'].mkdir(parents=True, exist_ok=True) - indexentries = [] - - for p in paths: - path = pathlib.Path(p) - #print(f"{path=}") - if p in [ 'index.html', '.git' ]: - continue + pprint.pprint(root_properties) + print(JINJA_TEMPLATE_DIRECTORY_INDEX) + html = JINJA_TEMPLATE_DIRECTORY_INDEX.render(**root_properties) + with open(root_properties['dst_path']['raw'].joinpath('index.html'), 'w+', encoding='utf-8') as file_pointer: + file_pointer.write(html) - fullpath = directory.joinpath(path) - title = path.name - if path.suffix == '.html': - with open(fullpath) as fp: - soup = bs(fp.read(), 'html.parser') + # render each file + for file in files: + render_file(root.joinpath(file)) - try: - title = soup.find('title').get_text() or pathlib.Path(path).name - except AttributeError: - title = pathlib.Path(path).stem - elif fullpath.is_dir(): - title = path - elif is_plaintext(fullpath): - # don't add plaintext files to index, since they have a html wrapper - continue - if str(title).strip() == '': - title = path + process_home_index(args.output_dir, search_data=FILEMAP.to_list()) - indexentries.append({ - 'title': str(title), - 'path': './' + str(path), - 'isdirectory': fullpath.is_dir() - }) + # copy styling and js scripts necessary for function + shutil.copytree(CSS_DIR, args.output_dir.joinpath('css'), dirs_exist_ok=True) + shutil.copytree(JS_DIR, args.output_dir.joinpath('js'), dirs_exist_ok=True) - indexentries.sort(key=lambda entry: str(entry['title']).lower()) - indexentries.sort(key=lambda entry: entry['isdirectory'], reverse=True) - - html = re.sub(r'\$title\$', str(directory), INDEX_TEMPLATE_HEAD) - html = re.sub(r'\$h1title\$', str(directory), html) - html = re.sub(r'\$extra_content\$', - EXTRA_INDEX_CONTENT if directory == args.notes else '', - html - ) - for entry in indexentries: - html += ( - '
' - f'{entry["title"]}{"/" if entry["isdirectory"] else ""}' - '
' - '$licenseFull$+
{{ license }}
-These are my personal notes. Correctness is not guaranteed. -Browse by tag here. -
diff --git a/templates/home_index.html b/templates/home_index.html index 9c50c33..46d75ce 100644 --- a/templates/home_index.html +++ b/templates/home_index.html @@ -2,15 +2,15 @@ - -These are my personal notes. Correctness is not guaranteed. -Browse here or by tag here. +Browse here or by tag here.
page generated by notes2web (commit $n2w_commit$) notes commit $notes_git_head_sha1$
+page generated by notes2web (commit {{ n2w_commit }}) {% if notes_git_head_sha1 %}notes commit {{ notes_git_head_sha1 }}{% endif %}
+ Press (Shift+)Enter to open first result (in new tab) +
+ {% endif %} + +../
{{ entry['title'] }}{% if entry['is_dir'] %}/{% endif %}
page generated by notes2web
page generated by notes2web
-- Press (Shift+)Enter to open first result (in new tab) -
- -This file was not rendered by notes2web because it is a plaintext file, not a markdown +file. +You access the raw file here. +Below is an unformatted representation of the file: +
+{{ raw_content }}
+{% endblock %}
diff --git a/templates/textarticlefoot.html b/templates/textarticlefoot.html
deleted file mode 100644
index 8ef0abe..0000000
--- a/templates/textarticlefoot.html
+++ /dev/null
@@ -1,3 +0,0 @@
-
-
-- page generated by notes2web -
-$licenseFull$-
This file was not rendered by notes2web because it is a plaintext file, not a markdown - file. - You access the raw file here. - Below is an unformatted representation of the file: -
-