begin rewrite

This commit is contained in:
Akbar Rahman 2024-01-02 18:22:15 +00:00
parent 8bd9173847
commit 29529cfd6a
Signed by: alvierahman90
GPG Key ID: 6217899F07CA2BDF
21 changed files with 539 additions and 553 deletions

View File

@ -1,16 +1,9 @@
install: install:
cp n2w_add_uuid.py /usr/local/bin cp n2w_add_uuid.py /usr/local/bin
sed "s/N2W_COMMIT = \"\"/N2W_COMMIT = \"$$(git rev-parse --short HEAD)\"/" notes2web.py > /usr/local/bin/notes2web.py sed "s/N2W_COMMIT = \"\"/N2W_COMMIT = \"$$(git rev-parse --short HEAD)\"/" notes2web.py > /usr/local/bin/notes2web.py
pip3 install -r requirements.txt --break-system-packages
mkdir -p /opt/notes2web mkdir -p /opt/notes2web
cp -r templates /opt/notes2web cp -r templates js css /opt/notes2web
cp styles.css /opt/notes2web pip3 install -r requirements.txt
cp fuse.js /opt/notes2web
cp search.js /opt/notes2web
cp indexsearch.js /opt/notes2web
cp toc_search.js /opt/notes2web
cp permalink.js /opt/notes2web
chmod +x /usr/local/bin/notes2web.py
uninstall: uninstall:
rm -rf /usr/local/bin/notes2web.py /usr/local/bin/n2w_add_uuid.py /opt/notes2web rm -rf /usr/local/bin/notes2web.py /usr/local/bin/n2w_add_uuid.py /opt/notes2web

View File

@ -3,6 +3,7 @@
@import url("https://styles.alv.cx/modules/search.css"); @import url("https://styles.alv.cx/modules/search.css");
@import url("https://styles.alv.cx/modules/buttonlist.css"); @import url("https://styles.alv.cx/modules/buttonlist.css");
@import url("https://styles.alv.cx/modules/darkmode.css"); @import url("https://styles.alv.cx/modules/darkmode.css");
@import url("/notes/styles.css");
html { html {
scroll-behavior: smooth; scroll-behavior: smooth;

184
fileproperties.py Normal file
View File

@ -0,0 +1,184 @@
from pathlib import Path
import frontmatter
import copy
import magic
import regex as re
class FileMap:
"""
this class is used to read file properties, inherit properties, and have a centralised place to access them
"""
def __init__(self, input_dir, output_dir):
self._map = {}
self.input_dir = Path(input_dir)
self.output_dir = Path(output_dir)
@staticmethod
def _path_to_key(path):
return str(Path(path))
def get(self, filepath, default=None, raw=False):
"""
get the properties of a file at a filepath
raw=True to not inherit properties
"""
#print(f"FileMap.get({filepath=}, {default=}, {raw=})")
# TODO maybe store properties of a file once it's in built and mark it as built? might save time but also cba
if self._path_to_key(filepath) not in self._map.keys():
self.add(filepath)
properties = copy.deepcopy(self._map.get(self._path_to_key(filepath), default))
#print(f"FileMap.get({filepath=}, {default=}, {raw=}): {properties=}")
if raw:
return properties
parent = filepath
while True:
parent = parent.parent
if parent == Path('.'):
break
parent_properties = self.get(parent, raw=True)
# TODO inherit any property that isn't defined, append any lists that exist
properties['tags'] = properties.get('tags', []) + parent_properties.get('tags', [])
if parent == self.input_dir:
break
return properties
def add(self, filepath):
filepath = Path(filepath)
#print(f"FileMap.add({filepath=}")
if filepath.is_dir():
properties = self._get_directory_properties(filepath)
else:
properties = self._get_file_properties(filepath)
properties['src_path'] = filepath
properties['dst_path'] = self._get_output_filepath(filepath)
self._map[self._path_to_key(filepath)] = properties
def _get_directory_properties(self, filepath: Path, include_index_entries=True):
"""
return dict of directory properties to be used in pandoc template
"""
post = {
'title': filepath.name,
'content_after_search': False,
'automatic_index': True,
'search_bar': True,
'tags': [],
}
if 'index.md' in filepath.iterdir():
with open(filepath.joinpath('index.md'), encoding='utf-8') as file_pointer:
for key, val in frontmatter.load(file_pointer).to_dict():
post[key] = val
post['is_dir'] = True
if include_index_entries:
post['index_entries'] = self._get_index_entries(filepath)
return post
def _get_index_entries(self, filepath):
"""
return sorted list of index entries. alphabetically sorted, folders first
"""
entries = []
for path in filepath.iterdir():
print(f'{path=}')
if path.is_dir():
entry = self._get_directory_properties(path, include_index_entries=False)
else:
entry = self._get_file_properties(path)
entry['path'] = self._get_output_filepath(path)['web']
entries.append(entry)
#print(f"FileMap._get_index_entries({filepath=}): {entry=}")
entries.sort(key=lambda entry: str(entry['title']).lower())
entries.sort(key=lambda entry: entry['is_dir'], reverse=True)
return entries
def _get_file_properties(self, filepath):
#print(f"FileMap._get_file_properties({filepath=}")
post = { 'title': filepath.name }
if filepath.suffix == '.md':
with open(filepath, encoding='utf-8') as file_pointer:
post = frontmatter.load(file_pointer).to_dict()
# don't store file contents in memory
if 'content' in post.keys():
del post['content']
post['is_dir'] = False
return post
def _get_output_filepath(self, input_filepath):
def webpath(filepath):
return Path('/notes').joinpath(filepath.relative_to(self.output_dir))
r = {}
r['raw'] = self.output_dir.joinpath(input_filepath.relative_to(self.input_dir))
r['web'] = webpath(r['raw'])
if input_filepath.is_dir():
return r
if input_filepath.suffix == '.md':
r['html'] = self.output_dir.joinpath(
input_filepath.relative_to(self.input_dir)
).with_suffix('.html')
r['web'] = webpath(r['html'])
elif self.is_plaintext(input_filepath):
r['html'] = self.output_dir.joinpath(
input_filepath.relative_to(self.input_dir)
).with_suffix(input_filepath.suffix + '.html')
r['raw'] = self.output_dir.joinpath(input_filepath.relative_to(self.input_dir))
r['web'] = webpath(r['html'])
#print(f"{r=}")
return r
def to_list(self):
"""
returns list of every file in map
"""
r = []
for _, val in self._map.items():
r.append({
'title': val.get('title', ''),
'tags': val.get('tags', []),
'path': str(val['dst_path']['web']),
'is_dir': val['is_dir']
})
return r
@staticmethod
def is_plaintext(filename):
"""
check if file is a plaintext format, such as html, css, etc,
return boolean
"""
return re.match(r'^text/', magic.from_file(str(filename), mime=True)) is not None

View File

View File

@ -1,3 +1,13 @@
/*
* search.js expects an array `data` containing objects with the following keys:
* headers: [string]
* path: string
* tags: [string]
* title: string
* uuid: string
*/
const HEADERS = "headers" const HEADERS = "headers"
const PATH = "path" const PATH = "path"
const TAGS = "tags" const TAGS = "tags"
@ -6,11 +16,11 @@ const TITLE = "title"
const SEARCH_TIMEOUT_MS = 100 const SEARCH_TIMEOUT_MS = 100
var SEARCH_TIMEOUT_ID = -1 var SEARCH_TIMEOUT_ID = -1
const fuse = new Fuse(data, { const fuse = new Fuse(search_data, {
keys: [ keys: [
{ {
name: HEADERS, name: HEADERS,
weight: 0.2 weight: 1
}, },
{ {
name: PATH, name: PATH,
@ -18,11 +28,11 @@ const fuse = new Fuse(data, {
}, },
{ {
name: TAGS, name: TAGS,
weight: 0.1 weight: 0.5
}, },
{ {
name: TITLE, name: TITLE,
weight: 4 weight: 2
} }
], ],
includeMatches: true, includeMatches: true,

View File

@ -1,426 +1,274 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""
notes2web --- view your notes as a static html site
"""
from bs4 import BeautifulSoup as bs import argparse
import subprocess
import frontmatter
import magic
import sys
import pathlib
import pypandoc
import shutil
import os import os
import regex as re from pathlib import Path
import shutil
import sys
import pprint
import json import json
import yaml
import frontmatter
import git
import jinja2
import requests
from fileproperties import FileMap
TEXT_ARTICLE_TEMPLATE_FOOT = None
TEXT_ARTICLE_TEMPLATE_HEAD = None
INDEX_TEMPLATE_FOOT = None
INDEX_TEMPLATE_HEAD = None
EXTRA_INDEX_CONTENT = None
N2W_COMMIT = "" N2W_COMMIT = ""
PANDOC_SERVER_URL = os.getenv("PANDOC_SERVER_URL", r"http://localhost:3030/")
def is_plaintext(filename): PANDOC_TIMEOUT = int(os.getenv("PANDOC_TIMEOUT", "120"))
return re.match(r'^text/', magic.from_file(str(filename), mime=True)) is not None CSS_DIR = Path(os.getenv("CSS_DIR", "/opt/notes2web/css"))
JS_DIR = Path(os.getenv("JS_DIR", "/opt/notes2web/js"))
def get_files(folder): TEMPLATES_DIR = Path(os.getenv("TEMPLATES_DIR", "/opt/notes2web/templates"))
markdown = []
plaintext = []
other = []
for root, folders, files in os.walk(folder):
for filename in files:
if '/.git' in root:
continue
name = os.path.join(root, filename)
if pathlib.Path(name).suffix == '.md':
markdown.append(name)
elif is_plaintext(name):
plaintext.append(name)
other.append(name)
else:
other.append(name)
return markdown, plaintext, other
def get_inherited_tags(file, base_folder):
tags = []
folder = pathlib.Path(file)
while folder != base_folder.parent:
print(f"get_inherited_tags {folder=}")
folder = pathlib.Path(folder).parent
folder_metadata = folder.joinpath('.n2w.yml')
if not folder_metadata.exists():
continue
with open(folder.joinpath('.n2w.yml')) as fp:
folder_properties = yaml.safe_load(fp)
tags += folder_properties.get('itags')
print(f"get_inherited_tags {tags=}")
return list(set(tags))
def git_head_sha1(working_dir): JINJA_ENV = jinja2.Environment(
git_response = subprocess.run( loader=jinja2.PackageLoader("notes2web", str(TEMPLATES_DIR)),
[ 'git', f"--git-dir={working_dir.joinpath('.git')}", 'rev-parse', '--short', 'HEAD' ], autoescape=jinja2.select_autoescape
stdout=subprocess.PIPE
).stdout.decode('utf-8')
return git_response.strip()
def git_filehistory(working_dir, filename):
print(f"{pathlib.Path(filename).relative_to(working_dir)=}")
git_response = subprocess.run(
[
'git',
f"--git-dir={working_dir.joinpath('.git')}",
"log",
"-p",
"--",
pathlib.Path(filename).relative_to(working_dir)
],
stdout=subprocess.PIPE
)
filehistory = [f"File history not available: git log returned code {git_response.returncode}."
"\nIf this is not a git repository, this is not a problem."]
if git_response.returncode == 0:
filehistory = git_response.stdout.decode('utf-8')
temp = re.split(
r'(commit [a-f0-9]{40})',
filehistory,
flags=re.IGNORECASE
) )
for t in temp: JINJA_TEMPLATES = {}
if t == '': JINJA_TEMPLATE_TEXTARTICLE = JINJA_ENV.get_template("textarticle.html")
temp.remove(t) JINJA_TEMPLATE_HOME_INDEX = JINJA_ENV.get_template("home_index.html")
filehistory = [] JINJA_TEMPLATE_DIRECTORY_INDEX = JINJA_ENV.get_template("index.html")
for i in range(0, len(temp)-1, 2): JINJA_TEMPLATE_ARTICLE = JINJA_ENV.get_template("article.html")
filehistory.append(f"{temp[i]}{temp[i+1]}")
if filehistory == "":
filehistory = ["This file has no history (it may not be part of the git repository)."]
filehistory = [ x.replace("<", "&lt;").replace(">", "&gt;") for x in filehistory]
filehistory = "<pre>\n" + "</pre><pre>\n".join(filehistory) + "</pre>"
return filehistory
def get_dirs_to_index(folder): LICENSE = None
r = [] GIT_REPO = None
FILEMAP = None
for root, folders, files in os.walk(folder):
if pathlib.Path(os.path.join(root, folder)).is_relative_to(folder.joinpath('permalink')):
continue
[r.append(os.path.join(root, folder)) for folder in folders]
return r
def update_required(src_filename, output_filename): def update_required(src_filepath, output_filepath):
return not os.path.exists(output_filename) or os.path.getmtime(src_filename) > os.path.getmtime(output_filename) """
check if file requires an update,
return boolean
"""
return not output_filepath.exists() or src_filepath.stat().st_mtime > output_filepath.stat().st_mtimeme()
def get_args(): def get_args():
""" Get command line arguments """ """ Get command line arguments """
import argparse
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('notes', type=pathlib.Path) parser.add_argument('notes', type=Path)
parser.add_argument('-o', '--output-dir', type=pathlib.Path, default='web') parser.add_argument('-o', '--output-dir', type=Path, default='web')
parser.add_argument('-t', '--template', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/article.html')) parser.add_argument('-F', '--force', action="store_true",
parser.add_argument('-H', '--template-text-head', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/textarticlehead.html')) help="Generate new output html even if source file was modified before output html")
parser.add_argument('-f', '--template-text-foot', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/textarticlefoot.html'))
parser.add_argument('-i', '--template-index-head', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/indexhead.html'))
parser.add_argument('-I', '--template-index-foot', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/indexfoot.html'))
parser.add_argument('-s', '--stylesheet', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/styles.css'))
parser.add_argument('--home_index', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/home_index.html'))
parser.add_argument('--permalink_index', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/permalink_index.html'))
parser.add_argument('-e', '--extra-index-content', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/extra_index_content.html'))
parser.add_argument('-n', '--index-article-names', action='append', default=['index.md'])
parser.add_argument('-F', '--force', action="store_true", help="Generate new output html even if source file was modified before output html")
parser.add_argument('--fuse', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/fuse.js'))
parser.add_argument('--searchjs', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/search.js'))
parser.add_argument('--indexsearchjs', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/indexsearch.js'))
parser.add_argument('--permalinkjs', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/permalink.js'))
parser.add_argument('--tocsearchjs', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/toc_search.js'))
parser.add_argument('--toc-depth', type=int, default=6, dest='toc_depth')
return parser.parse_args() return parser.parse_args()
def render_markdown_file(input_filepath):
"""
render markdown file to file
write markdown file to args.output_dir in html,
return list of tuple of output filepath, frontmatter post
"""
print(f"render_markdown_file({input_filepath})")
with open(input_filepath, encoding='utf-8') as file_pointer:
content = frontmatter.load(file_pointer).content
properties = FILEMAP.get(input_filepath)
# TODO pandoc no longer handles template due to metadata passing issues, use jinja to fill in the metadata
html = render_markdown(content)
with open(properties['dst_path']['html'], 'w+', encoding='utf-8') as file_pointer:
file_pointer.write(html)
def render_plaintext_file(input_filepath):
"""
render plaintext file to file
copy plaintext file into a html preview, copy raw to output dir
return list of tuple of output filepath, empty dict
"""
with open(input_filepath, encoding='utf-8') as file_pointer:
raw_content = file_pointer.read()
properties = FILEMAP.get(input_filepath)
html = JINJA_TEMPLATE_TEXTARTICLE.render(license = LICENSE, **properties)
with open(properties['dst_path']['raw'], "w+", encoding='utf-8') as file_pointer:
file_pointer.write(raw_content)
with open(properties['dst_path']['html'], "w+", encoding='utf-8') as file_pointer:
file_pointer.write(html)
def render_generic_file(input_filepath):
"""
render generic file to file
copy generic file into to output_dir
return list of tuple of output filepath, empty dict
"""
properties = FILEMAP.get(input_filepath)
output_filepath = properties['dst_path']['raw']
shutil.copyfile(input_filepath, output_filepath)
def render_file(input_filepath):
"""
render any file by detecting type and applying appropriate type
write input_filepath to correct file in args.output_dir in appropriate formats,
return list of tuples of output filepath, frontmatter post
"""
if input_filepath.suffix == '.md':
return render_markdown_file(input_filepath)
if FileMap.is_plaintext(input_filepath):
return render_plaintext_file(input_filepath)
return render_generic_file(input_filepath)
def render_markdown(content):
"""
render markdown to html
"""
post_body = {
'text': content,
'toc-depth': 6,
'highlight-style': 'pygments',
'html-math-method': 'mathml',
'to': 'html',
'files': {
'data/data/abbreviations': '',
},
'standalone': False,
}
headers = {
'Accept': 'application/json'
}
response = requests.post(
PANDOC_SERVER_URL,
headers=headers,
json=post_body,
timeout=PANDOC_TIMEOUT
)
response = response.json()
# TODO look at response['messages'] and log them maybe?
# https://github.com/jgm/pandoc/blob/main/doc/pandoc-server.md#response
return response['output']
def process_home_index(output_dir, search_data, notes_git_head_sha1=None):
"""
create home index.html in output_dir
"""
html = JINJA_TEMPLATE_HOME_INDEX.render(
n2w_commit = N2W_COMMIT,
search_data=search_data,
notes_git_head_sha1=notes_git_head_sha1,
)
with open(output_dir.joinpath('index.html'), 'w+', encoding='utf-8') as file_pointer:
file_pointer.write(html)
def generate_variable_browser(output_dir, posts, variable_name) :
"""
generate a directory that lets you groub by and browse by any given tag. e.g. tags, authors
"""
groups = {}
for key, post in posts.iter():
group_val = post.get(variable_name, None)
if group_val is None:
continue
if group_val not in groups.keys():
groups[group_val] = []
groups[group_val].append(post)
for group_val, index_entries in groups.iter():
post = {
'index_entries': index_entries,
'title': group_val,
}
# TODO finish writing function, write page to disk
def main(args): def main(args):
""" Entry point for script """ """ Entry point for script """
with open(args.template_text_foot) as fp: global LICENSE
TEXT_ARTICLE_TEMPLATE_FOOT = fp.read() global GIT_REPO
global FILEMAP
with open(args.template_text_head) as fp: FILEMAP = FileMap(args.notes, args.output_dir.joinpath('notes'))
TEXT_ARTICLE_TEMPLATE_HEAD = fp.read()
with open(args.template_index_foot) as fp:
INDEX_TEMPLATE_FOOT = fp.read()
with open(args.template_index_head) as fp:
INDEX_TEMPLATE_HEAD = fp.read()
with open(args.extra_index_content) as fp:
EXTRA_INDEX_CONTENT = fp.read()
if args.output_dir.is_file(): if args.output_dir.is_file():
print(f"Output directory ({args.output_dir}) cannot be a file.") print(f"Output directory ({args.output_dir}) cannot be a file.")
args.output_dir.mkdir(parents=True, exist_ok=True) args.output_dir.mkdir(parents=True, exist_ok=True)
notes_license = "This note has no copyright license.", # attempt to get licensing information
print(f"{notes_license=}")
license_path = args.notes.joinpath("LICENSE") license_path = args.notes.joinpath("LICENSE")
if license_path.exists(): if license_path.exists():
with open(license_path) as fp: with open(license_path, encoding='utf-8') as file_pointer:
notes_license = fp.read() LICENSE = file_pointer.read()
markdown_files, plaintext_files, other_files = get_files(args.notes) # create git.Repo object if notes dir is a git repo
# TODO git commit log integration
if '.git' in args.notes.iterdir():
GIT_REPO = git.Repo(args.notes)
all_entries=[] for root_str, subdirectories, files in os.walk(args.notes):
dirs_with_index_article = [] root = Path(root_str)
tag_dict = {} if '.git' in root.parts:
permalink_to_filepath = {}
print(f"{markdown_files=}")
for filename in markdown_files:
print(f"{filename=}")
# calculate output filename
output_filename = args.output_dir.joinpath('notes').joinpath(
pathlib.Path(filename).relative_to(args.notes)
).with_suffix('.html')
if os.path.basename(filename) in args.index_article_names:
output_filename = output_filename.parent.joinpath('index.html')
dirs_with_index_article.append(str(output_filename.parent))
print(f"{output_filename=}")
# extract tags from frontmatter, save to tag_dict
fm = frontmatter.load(filename)
if isinstance(fm.get('tags'), list):
for tag in list(set(fm.get('tags') + get_inherited_tags(filename, args.notes))):
t = {
'path': str(pathlib.Path(output_filename).relative_to(args.output_dir)),
'title': fm.get('title') or pathlib.Path(filename).name
}
if tag in tag_dict.keys():
tag_dict[tag].append(t)
else:
tag_dict[tag] = [t]
# find headers in markdown
with open(filename) as fp:
lines = fp.read().split('\n')
header_lines = []
for line in lines:
if re.match('^#{1,6} \S', line):
header_lines.append(" ".join(line.split(" ")[1:]))
all_entries.append({
'path': '/' + str(pathlib.Path(*pathlib.Path(output_filename).parts[1:])),
'title': fm.get('title') or pathlib.Path(filename).name,
'tags': list(set(fm.get('tags'))),
'headers': header_lines,
'uuid': fm.get('uuid')
})
if 'uuid' in fm.keys():
permalink_to_filepath[fm['uuid']] = all_entries[-1]['path']
# update file if required
if update_required(filename, output_filename) or args.force:
filehistory = git_filehistory(args.notes, filename)
with open(filename) as fp:
article = frontmatter.load(fp)
article['tags'] += get_inherited_tags(filename, args.notes)
article['tags'] = sorted(list(set(article['tags'])))
article['filehistory'] = filehistory
article['licenseFull'] = notes_license
html = pypandoc.convert_text(frontmatter.dumps(article), 'html', format='md', extra_args=[
f'--template={args.template}',
'--mathjax',
'--toc', f'--toc-depth={args.toc_depth}'
])
pathlib.Path(output_filename).parent.mkdir(parents=True, exist_ok=True)
with open(output_filename, 'w+') as fp:
fp.write(html)
print(f"{plaintext_files=}")
for filename in plaintext_files:
filehistory = git_filehistory(args.notes, filename)
title = os.path.basename(filename)
output_filename = str(
args.output_dir.joinpath('notes').joinpath(
pathlib.Path(filename).relative_to(args.notes)
)
) + '.html'
print(f"{output_filename=}")
pathlib.Path(output_filename).parent.mkdir(parents=True, exist_ok=True)
html = re.sub(r'\$title\$', title, TEXT_ARTICLE_TEMPLATE_HEAD)
html = re.sub(r'\$h1title\$', title, html)
html = re.sub(r'\$raw\$', os.path.basename(filename), html)
html = re.sub(r'\$licenseFull\$', notes_license, html)
html = html.replace('$filehistory$', filehistory)
with open(filename) as fp:
html += fp.read().replace("<", "&lt;").replace(">", "&gt;")
html += TEXT_ARTICLE_TEMPLATE_FOOT
with open(output_filename, 'w+') as fp:
fp.write(html)
all_entries.append({
'path': str(pathlib.Path(*pathlib.Path(output_filename).parts[1:])),
'title': title,
'tags': [get_inherited_tags(filename, args.notes)],
'headers': []
})
print(f"{other_files=}")
for filename in other_files:
output_filename = str(
args.output_dir.joinpath('notes').joinpath(
pathlib.Path(filename).relative_to(args.notes)
)
)
title = os.path.basename(filename)
pathlib.Path(output_filename).parent.mkdir(parents=True, exist_ok=True)
all_entries.append({
'path': str(pathlib.Path(*pathlib.Path(output_filename).parts[1:])),
'title': title,
'tags': [get_inherited_tags(filename, args.notes)],
'headers': []
})
shutil.copyfile(filename, output_filename)
tagdir = args.output_dir.joinpath('.tags')
tagdir.mkdir(parents=True, exist_ok=True)
for tag in tag_dict.keys():
html = re.sub(r'\$title\$', f'{tag}', INDEX_TEMPLATE_HEAD)
html = re.sub(r'\$h1title\$', f'tag: {tag}', html)
html = re.sub(r'\$extra_content\$', '', html)
for entry in tag_dict[tag]:
entry['path'] = '/' + entry['path']
html += f"<div class=\"article\"><a href=\"{entry['path']}\">{entry['title']}</a></div>"
html += re.sub('\$data\$', json.dumps(tag_dict[tag]), INDEX_TEMPLATE_FOOT)
with open(tagdir.joinpath(f'{tag}.html'), 'w+') as fp:
fp.write(html)
dirs_to_index = [args.output_dir.name] + get_dirs_to_index(args.output_dir)
print(f"{dirs_to_index=}")
print(f"{dirs_with_index_article=}")
for d in dirs_to_index:
print(f"{d in dirs_with_index_article=} {d=}")
if d in dirs_with_index_article:
continue continue
directory = pathlib.Path(d) root_properties = FILEMAP.get(root)
paths = os.listdir(directory) root_properties['dst_path']['raw'].mkdir(parents=True, exist_ok=True)
#print(f"{paths=}")
indexentries = [] pprint.pprint(root_properties)
print(JINJA_TEMPLATE_DIRECTORY_INDEX)
html = JINJA_TEMPLATE_DIRECTORY_INDEX.render(**root_properties)
with open(root_properties['dst_path']['raw'].joinpath('index.html'), 'w+', encoding='utf-8') as file_pointer:
file_pointer.write(html)
for p in paths: # render each file
path = pathlib.Path(p) for file in files:
#print(f"{path=}") render_file(root.joinpath(file))
if p in [ 'index.html', '.git' ]:
continue
fullpath = directory.joinpath(path)
title = path.name
if path.suffix == '.html':
with open(fullpath) as fp:
soup = bs(fp.read(), 'html.parser')
try: process_home_index(args.output_dir, search_data=FILEMAP.to_list())
title = soup.find('title').get_text() or pathlib.Path(path).name
except AttributeError:
title = pathlib.Path(path).stem
elif fullpath.is_dir():
title = path
elif is_plaintext(fullpath):
# don't add plaintext files to index, since they have a html wrapper
continue
if str(title).strip() == '': # copy styling and js scripts necessary for function
title = path shutil.copytree(CSS_DIR, args.output_dir.joinpath('css'), dirs_exist_ok=True)
shutil.copytree(JS_DIR, args.output_dir.joinpath('js'), dirs_exist_ok=True)
indexentries.append({
'title': str(title),
'path': './' + str(path),
'isdirectory': fullpath.is_dir()
})
indexentries.sort(key=lambda entry: str(entry['title']).lower())
indexentries.sort(key=lambda entry: entry['isdirectory'], reverse=True)
html = re.sub(r'\$title\$', str(directory), INDEX_TEMPLATE_HEAD)
html = re.sub(r'\$h1title\$', str(directory), html)
html = re.sub(r'\$extra_content\$',
EXTRA_INDEX_CONTENT if directory == args.notes else '',
html
)
for entry in indexentries:
html += (
'<li class="article">'
f'<a href="{entry["path"]}"><p>'
f'{entry["title"]}{"/" if entry["isdirectory"] else ""}'
'</p></a>'
'</li>'
)
html += re.sub(r'\$data\$', json.dumps(indexentries), INDEX_TEMPLATE_FOOT)
with open(directory.joinpath('index.html'), 'w+') as fp:
fp.write(html)
shutil.copyfile(args.stylesheet, args.output_dir.joinpath('styles.css'))
shutil.copyfile(args.fuse, args.output_dir.joinpath('fuse.js'))
shutil.copyfile(args.searchjs, args.output_dir.joinpath('search.js'))
shutil.copyfile(args.indexsearchjs, args.output_dir.joinpath('indexsearch.js'))
shutil.copyfile(args.tocsearchjs, args.output_dir.joinpath('toc_search.js'))
shutil.copyfile(args.permalinkjs, args.output_dir.joinpath('permalink.js'))
with open(args.output_dir.joinpath('index.html'), 'w+') as fp:
with open(args.home_index) as fp2:
html = re.sub(r'\$title\$', args.output_dir.parts[0], fp2.read())
html = re.sub(r'\$h1title\$', args.output_dir.parts[0], html)
html = re.sub(r'\$n2w_commit\$', N2W_COMMIT, html)
html = re.sub(r'\$notes_git_head_sha1\$', git_head_sha1(args.notes), html)
html = re.sub(r'\$data\$', json.dumps(all_entries), html)
fp.write(html)
permalink_dir = args.output_dir.joinpath('permalink')
permalink_dir.mkdir(exist_ok=True)
with open(args.permalink_index) as fp:
html = re.sub(r'\$data\$', json.dumps(permalink_to_filepath), fp.read())
with open(permalink_dir.joinpath('index.html'), 'w+') as fp:
fp.write(html)
print(tag_dict)
return 0 return 0
# TODO implement useful logging and debug printing
# TODO build tag/metadata pages
if __name__ == '__main__': if __name__ == '__main__':
try: try:
sys.exit(main(get_args())) sys.exit(main(get_args()))

View File

@ -25,27 +25,43 @@ doing it for me:
0. Install [Pandoc](https://pandoc.org/index.html) and [Pip](https://github.com/pypa/pip), python3-dev, and a C compiler 0. Install [Pandoc](https://pandoc.org/index.html) and [Pip](https://github.com/pypa/pip), python3-dev, and a C compiler
1. Run `make install` as root 1. Run `make install` as root
## Things to Remember Whilst Writing Notes ## Other Things to Know
- notes2web reads the following YAML [frontmatter](https://jekyllrb.com/docs/front-matter/) variable:
- `author` --- The person(s) who wrote the article
- `tags` --- A YAML list of tags which the article relates to - this is used for browsing and also
searching
- `title` --- The title of the article
- `uuid` --- A unique identifier used for permalinks. More below.
- `lecture_slides` --- a list of paths pointing to lecture slides used while taking notes
- `lecture_notes` --- a list of paths pointing to other notes used while taking notes
- notes2web indexes [ATX-style headings](https://pandoc.org/MANUAL.html#atx-style-headings) for - notes2web indexes [ATX-style headings](https://pandoc.org/MANUAL.html#atx-style-headings) for
searching searching
- notes2web attempts to display file history through the `git log` command - notes2web attempts to display file history through the `git log` command
- notes2web looks for the plaintext file `LICENSE` in the root directory of your notes - notes2web looks for the plaintext file `LICENSE` in the root directory of your notes
This is optional but if you would like to add a license you can find one
[here](https://choosealicense.com).
### Permalinks ## Custom Directory Index
To add custom content to a directory index, put it in a file called `index.md` under the directory.
You can set the following frontmatter variables to customise the directory index of a directory:
| variable | default value | description |
|------------------------|-------------------|--------------------------------------------------------------------------------------------|
| `tags` | `[]` | list of tags, used by search and inherited by any notes and subdirectories |
| `uuid` | none | unique id to reference directory, used for permalinking |
| `content_after_search` | `false` | show custom content in `index.md` after search bar and directory index |
| `automatic_index` | `true` | show the automatically generated directory index. required for search bar to function. |
| `search_bar` | `true` | show search bar to search directory items. requires `automatic_index` (enabled by default) |
## Notes Metadata
notes2web reads the following YAML [frontmatter](https://jekyllrb.com/docs/front-matter/) variables for metadata:
| variable | description |
|------------------|---------------------------------------------------------------------------------------|
| `author` | The person(s) who wrote the article |
| `tags` | A YAML list of tags which the article relates to - this is used for browsing and also |
| `title` | The title of the article |
| `uuid` | A unique identifier used for permalinks. |
| `lecture_slides` | a list of paths pointing to lecture slides used while taking notes |
| `lecture_notes` | a list of paths pointing to other notes used while taking notes |
## Permalinks
Permalinks are currently rather basic and requires JavaScript to be enabled on the local computer. Permalinks are currently rather basic and requires JavaScript to be enabled on the local computer.
In order to identify documents between file changes, a unique identifier is used to identify a file. In order to identify documents between file changes, a unique identifier is used to identify a file.
@ -57,21 +73,13 @@ The included `n2w_add_uuid.py` will add a UUID to a markdown file which does not
already. already.
Combine it with `find` to UUIDify all your markdown files (but make a backup first). Combine it with `find` to UUIDify all your markdown files (but make a backup first).
### Inherited Properties ## Custom Styling
Notes can inherit a some properties from their parent folder(s) by creating a `.n2w.yml` file in a To completely replace the existing styling, set the environment variable `CSS_DIR` to another directory with
folder. a file called `styles.css`.
#### Tags To add additional styling, the default styling will attempt to import `styles.css` from the root of the notes
directory.
If you have a folder `uni` with all you university notes, you might want all the files in there to
be tagged `uni`:
`NOTES_PATH/uni/.n2w.yaml`:
```yaml
itags: [ university ]
```
## CLI Usage ## CLI Usage
@ -81,35 +89,7 @@ $ notes2web.py notes_directory
Output of `notes2web.py --help`: Output of `notes2web.py --help`:
``` TODO add cli output
usage: notes2web.py [-h] [-o OUTPUT_DIR] [-t TEMPLATE] [-H TEMPLATE_TEXT_HEAD]
[-f TEMPLATE_TEXT_FOOT] [-i TEMPLATE_INDEX_HEAD]
[-I TEMPLATE_INDEX_FOOT] [-s STYLESHEET]
[--home_index HOME_INDEX] [-e EXTRA_INDEX_CONTENT]
[-n INDEX_ARTICLE_NAMES] [-F] [--fuse FUSE]
[--searchjs SEARCHJS]
notes
positional arguments:
notes
optional arguments:
-h, --help show this help message and exit
-o OUTPUT_DIR, --output-dir OUTPUT_DIR
-t TEMPLATE, --template TEMPLATE
-H TEMPLATE_TEXT_HEAD, --template-text-head TEMPLATE_TEXT_HEAD
-f TEMPLATE_TEXT_FOOT, --template-text-foot TEMPLATE_TEXT_FOOT
-i TEMPLATE_INDEX_HEAD, --template-index-head TEMPLATE_INDEX_HEAD
-I TEMPLATE_INDEX_FOOT, --template-index-foot TEMPLATE_INDEX_FOOT
-s STYLESHEET, --stylesheet STYLESHEET
--home_index HOME_INDEX
-e EXTRA_INDEX_CONTENT, --extra-index-content EXTRA_INDEX_CONTENT
-n INDEX_ARTICLE_NAMES, --index-article-names INDEX_ARTICLE_NAMES
-F, --force Generate new output html even if source file was
modified before output html
--fuse FUSE
--searchjs SEARCHJS
```
The command will generate a website in the `output-dir` directory (`./web` by default). The command will generate a website in the `output-dir` directory (`./web` by default).
It will then generate a list of all note files and put it in `index.html`. It will then generate a list of all note files and put it in `index.html`.

View File

@ -9,3 +9,4 @@ python-magic==0.4.24
PyYAML==5.3.1 PyYAML==5.3.1
regex==2021.11.10 regex==2021.11.10
soupsieve==2.2.1 soupsieve==2.2.1
PyYAML==6.0.1

View File

@ -2,7 +2,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
<meta charset="utf-8"> <meta charset="utf-8">
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Open+Sans&display=swap" /> <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Open+Sans&display=swap" />
<link rel="stylesheet" type="text/css" href="/styles.css" /> <link rel="stylesheet" type="text/css" href="/css/styles.css" />
<script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script> <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
<script> <script>
MathJax = { MathJax = {
@ -12,40 +12,43 @@
} }
</script> </script>
<script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script> <script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
<title>$title$</title> <title>{{ title }}</title>
</head> </head>
<body> <body>
<div id="contentWrapper"> <div id="contentWrapper">
<div id="content"> <div id="content">
<p class="smallText metadata"> <p class="smallText metadata"> title: {{ title }} </p>
title: $title$ {% if lecture_slides %}
</p>
$if(lecture_slides)$
<p class="smallText metadata"> lecture_slides: [ <p class="smallText metadata"> lecture_slides: [
$for(lecture_slides)$ {% for slide in lecture_slides %}
<a href="$lecture_slides$">$lecture_slides$</a>$sep$, <a href="{{ slide }}">{{ slide }}</a>{% if loop.nextitem %},{% endif %}
$endfor$ {% endfor %}
]</p> ]</p>
$endif$ {% endif %}
$if(lecture_notes)$ {% if lecture_notes %}
<p class="smallText metadata"> lecture_notes: [ <p class="smallText metadata"> lecture_notes: [
$for(lecture_notes)$ {% for note in lecture_notes %}
<a href="$lecture_notes$">$lecture_notes$</a>$sep$, <a href="{{ note }}">{{ note }}</a>{% if loop.nextitem %},{% endif %}
$endfor$ {% endfor %}
]</p> ]</p>
$endif$ {% endif %}
{% if uuid %}
<p class="smallText metadata"> uuid: {{ uuid }} (<a href="/permalink?uuid={{ uuid }}">permalink</a>) </p>
{% endif %}
<p class="smallText metadata">
uuid: $uuid$ (<a href="/permalink?uuid=$uuid$">permalink</a>)
</p>
<p class="smallText metadata"> tags: [ <p class="smallText metadata"> tags: [
$for(tags)$ {% for tag in tags %}
<a href="/.tags/$tags$.html">$tags$</a>$sep$, <a href="/.tags/{{ tag }}.html">{{ tag }}</a>{% if loop.nextitem %},{% endif %}
$endfor$ {% endfor %}
]</p> ]</p>
<p class="smallText metadata"> <p class="smallText metadata">
written by $for(author)$$author$$sep$, $endfor$ {% if author is iterable %}
written by {{ author }}
{% else %}
written by {% for auth in author %}{{ auth }}{% if loop.nextitem %}, {% endif %}{% endfor %}
{% endif %}
</p> </p>
<p class="smallText metadata"> <p class="smallText metadata">
syntax highlighting based on <a href="https://pygments.org/">Pygments'</a> default syntax highlighting based on <a href="https://pygments.org/">Pygments'</a> default
@ -54,21 +57,17 @@
<p class="smallText metadata"> <p class="smallText metadata">
page generated by <a href="https://git.alv.cx/alvierahman90/notes2web">notes2web</a> page generated by <a href="https://git.alv.cx/alvierahman90/notes2web">notes2web</a>
</p> </p>
<details id="commitLog"> {% if license %}
<summary class="smallText">
Commit log (file history)
</summary>
$filehistory$
</details>
<details id="license"> <details id="license">
<summary class="smallText"> <summary class="smallText">
License License
</summary> </summary>
<pre>$licenseFull$</pre> <pre>{{ license }}</pre>
</details> </details>
$body$ {% endif %}
{% block body %}
{{ content }}
{% endblock %}
</div> </div>
</div> </div>
<script src="/fuse.js"> </script>
<script src="/toc_search.js"> </script>
</body> </body>

View File

@ -1,4 +0,0 @@
<p>
These are my personal notes. Correctness is not guaranteed.
Browse by tag <a href="/.tags">here</a>.
</p>

View File

@ -2,15 +2,15 @@
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
<meta charset="utf-8"> <meta charset="utf-8">
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Open+Sans&display=swap" /> <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Open+Sans&display=swap" />
<link rel="stylesheet" type="text/css" href="/styles.css" /> <link rel="stylesheet" type="text/css" href="/css/styles.css" />
<title>$title$</title> <title>{{ title }}</title>
</head> </head>
<body> <body>
<div id="content"> <div id="content">
<h1>$h1title$</h1> <h1> {{ h1title }}</>
<p> <p>
These are my personal notes. Correctness is not guaranteed. These are my personal notes. Correctness is not guaranteed.
Browse <a href="/notes">here</a> or by tag <a href="/.tags">here</a>. Browse <a href="/notes">here</a> or by tag <a href="/tags">here</a>.
</p> </p>
<div id="searchWrapper"> <div id="searchWrapper">
@ -20,9 +20,9 @@ Browse <a href="/notes">here</a> or by tag <a href="/.tags">here</a>.
<div id="results"> <div id="results">
</div> </div>
<p class="smallText"> page generated by <a href="https://github.com/alvierahman90/notes2web">notes2web</a> (commit $n2w_commit$) notes commit $notes_git_head_sha1$</p> <p class="smallText"> page generated by <a href="https://github.com/alvierahman90/notes2web">notes2web</a> (commit {{ n2w_commit }}) {% if notes_git_head_sha1 %}notes commit {{ notes_git_head_sha1 }}{% endif %}</p>
</div> </div>
<script src="/fuse.js"> </script> <script src="/js/fuse.js"> </script>
<script> const data = $data$ </script> <script> const search_data = {{ search_data|tojson }} </script>
<script src="/search.js"> </script> <script src="/js/search.js"> </script>
</body> </body>

View File

@ -2,13 +2,43 @@
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
<meta charset="utf-8"> <meta charset="utf-8">
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Open+Sans&display=swap" /> <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Open+Sans&display=swap" />
<link rel="stylesheet" type="text/css" href="/styles.css" /> <link rel="stylesheet" type="text/css" href="/css/styles.css" />
<title>$title$</title> <title>{{ title }}</title>
</head> </head>
<body> <body>
<h1>$h1title$</h1>
<div id="content"> <div id="content">
$body$ <h1>{{ title }}</h1>
{% if not content_after_search %}
{{ content }}
{% endif %}
{% if automatic_index %}
{% if search_bar %}
<div id="searchWrapper">
<input id="search" placeholder="search" autocomplete="off" autofocus>
</div>
<p class="searchSmallText" style="margin-top: 0; text-align: center">
Press (<kbd>Shift</kbd>+)<kbd>Enter</kbd> to open first result (in new tab)
</p>
{% endif %}
<ul id="searchResults" class="buttonlist">
<li class="article"><a href=".."><p>../</p></a></li>
{% for entry in index_entries %}
<li class="article"><a href="{{ entry['path'] }}"><p>{{ entry['title'] }}{% if entry['is_dir'] %}/{% endif %}</p></a></li>
{% endfor %}
</ul>
{% endif %}
{% if content_after_search %}
{{ content }}
{% endif %}
<p style="font-size: 0.7em;"> page generated by <a href="https://github.com/alvierahman90/notes2web">notes2web</a></p> <p style="font-size: 0.7em;"> page generated by <a href="https://github.com/alvierahman90/notes2web">notes2web</a></p>
</div> </div>
<script src="/js/fuse.js"> </script>
<script> const search_data = {{ index_entries }} </script>
<script src="/js/indexsearch.js"> </script>
</body> </body>

View File

@ -1,7 +0,0 @@
</ul>
<p style="font-size: 0.7em;"> page generated by <a href="https://github.com/alvierahman90/notes2web">notes2web</a></p>
</div>
<script src="/fuse.js"> </script>
<script> const data = $data$ </script>
<script src="/indexsearch.js"> </script>
</body>

View File

@ -1,21 +0,0 @@
<head>
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
<meta charset="utf-8">
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Open+Sans&display=swap" />
<link rel="stylesheet" type="text/css" href="/styles.css" />
<title>$title$</title>
</head>
<body>
<div id="content">
<h1>$title$</h1>
$extra_content$
<div id="searchWrapper">
<input id="search" placeholder="search" autocomplete="off" autofocus>
</div>
<p class="searchSmallText" style="margin-top: 0; text-align: center">
Press (<kbd>Shift</kbd>+)<kbd>Enter</kbd> to open first result (in new tab)
</p>
<ul id="searchResults" class="buttonlist">
<li class="article"><a href=".."><p>../</p></a></li>

View File

@ -2,7 +2,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
<meta charset="utf-8"> <meta charset="utf-8">
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Open+Sans&display=swap" /> <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Open+Sans&display=swap" />
<link rel="stylesheet" type="text/css" href="/styles.css" /> <link rel="stylesheet" type="text/css" href="/css/styles.css" />
<title></title> <title></title>
</head> </head>
<body> <body>
@ -15,5 +15,5 @@ Otherwise, click <a id="manual_redirect">here</a>.
<p class="smallText"> page generated by <a href="https://github.com/alvierahman90/notes2web">notes2web</a></p> <p class="smallText"> page generated by <a href="https://github.com/alvierahman90/notes2web">notes2web</a></p>
</div> </div>
<script> const data = $data$ </script> <script> const data = $data$ </script>
<script src="/permalink.js"> </script> <script src="/js/permalink.js"> </script>
</body> </body>

View File

@ -0,0 +1,9 @@
{% extends "article.html" %}
{% block body %}
<p> This file was not rendered by notes2web because it is a plaintext file, not a markdown
file.
You access the raw file <a href="{{ raw_link }}">here</a>.
Below is an unformatted representation of the file:
</p>
<pre>{{ raw_content }}</pre>
{% endblock %}

View File

@ -1,3 +0,0 @@
</pre>
</div>
</body>

View File

@ -1,34 +0,0 @@
<head>
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
<meta charset="utf-8">
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Open+Sans&display=swap" />
<link rel="stylesheet" type="text/css" href="/styles.css" />
<title>$title$</title>
</head>
<body>
<div id="content">
<div id="header">
<p class="smallText">
page generated by <a href="https://git.alv.cx/alvierahman90/notes2web">notes2web</a>
</p>
<details>
<summary class="smallText">
Commit log (file history)
</summary>
$filehistory$
</details>
<details>
<summary class="smallText">
License
</summary>
<pre>$licenseFull$</pre>
</details>
</div>
<h1>$title$</h1>
<p> This file was not rendered by notes2web because it is a plaintext file, not a markdown
file.
You access the raw file <a href="$raw$">here</a>.
Below is an unformatted representation of the file:
</p>
<pre>