styling

Makefile

@@ -1,9 +1,11 @@
 install:
 	cp notes2web.py /usr/local/bin
-	pip install -r requirements.txt
+	pip3 install -r requirements.txt
 	mkdir -p /opt/notes2web
 	cp -r templates /opt/notes2web
 	cp styles.css /opt/notes2web
+	cp fuse.js /opt/notes2web
+	cp search.js /opt/notes2web
 
 uninstall:
 	rm -rf /usr/local/bin/notes2web.py /opt/notes2web

notes2web.py

@@ -2,6 +2,8 @@
 
 
 from bs4 import BeautifulSoup as bs
+import subprocess
+import frontmatter
 import magic
 import sys
 import pathlib
@@ -9,6 +11,8 @@ import pypandoc
 import shutil
 import os
 import re
+import json
+
 
 TEXT_ARTICLE_TEMPLATE_FOOT = None
 TEXT_ARTICLE_TEMPLATE_HEAD = None
@@ -16,6 +20,7 @@ INDEX_TEMPLATE_FOOT = None
 INDEX_TEMPLATE_HEAD = None
 EXTRA_INDEX_CONTENT = None
 
+
 def get_files(folder):
     markdown = []
     plaintext = []
@@ -23,6 +28,8 @@ def get_files(folder):
 
     for root, folders, files in os.walk(folder):
         for filename in files:
+            if '/.git' in root:
+                continue
             name = os.path.join(root, filename)
             if os.path.splitext(name)[1] == '.md':
                 markdown.append(name)
@@ -34,6 +41,30 @@ def get_files(folder):
 
     return markdown, plaintext, other
 
+def git_filehistory(working_dir, filename):
+    print(f"{pathlib.Path(filename).relative_to(working_dir)=}")
+    git_response = subprocess.run(
+            [
+                'git',
+                f"--git-dir={os.path.join(working_dir, '.git')}",
+                "log",
+                "-p",
+                "--",
+                pathlib.Path(filename).relative_to(working_dir)
+            ],
+            stdout=subprocess.PIPE
+    )
+
+    filehistory = f"File history not available: git log returned code {git_response.returncode}."
+    "\nIf this is not a git repository, this is not a problem."
+    if git_response.returncode == 0:
+        filehistory = git_response.stdout.decode('utf-8')
+    if filehistory == "":
+        filehistory = "This file has no history (it may not be part of the git repository)."
+
+    return filehistory
+
+
 def get_dirs(folder):
     r = []
 
@@ -43,6 +74,10 @@ def get_dirs(folder):
     return r
 
 
+def update_required(src_filename, output_filename):
+    return not os.path.exists(output_filename) or os.path.getmtime(src_filename) > os.path.getmtime(output_filename)
+
+
 def get_args():
     """ Get command line arguments """
 
@@ -56,7 +91,12 @@ def get_args():
     parser.add_argument('-i', '--template-index-head', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/indexhead.html'))
     parser.add_argument('-I', '--template-index-foot', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/indexfoot.html'))
     parser.add_argument('-s', '--stylesheet', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/styles.css'))
+    parser.add_argument('--home_index', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/home_index.html'))
     parser.add_argument('-e', '--extra-index-content', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/templates/extra_index_content.html'))
+    parser.add_argument('-n', '--index-article-names', action='append', default=['index.md'])
+    parser.add_argument('-F', '--force', action="store_true", help="Generate new output html even if source file was modified before output html")
+    parser.add_argument('--fuse', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/fuse.js'))
+    parser.add_argument('--searchjs', type=pathlib.Path, default=pathlib.Path('/opt/notes2web/search.js'))
     return parser.parse_args()
 
 
@@ -86,50 +126,131 @@ def main(args):
 
 
     markdown_files, plaintext_files, other_files = get_files(args.notes)
+    all_entries=[]
+
+    print(f"{args.index_article_names=}")
+
+    dirs_with_index_article = []
 
     print(f"{markdown_files=}")
+    tag_dict = {}
     for filename in markdown_files:
         print(f"{filename=}")
-        html = pypandoc.convert_file(filename, 'html', extra_args=[f'--template={args.template}'])
-        output_filename = os.path.splitext(re.sub(f"^{args.notes.name}", args.output_dir.name, filename))[0] + '.html'
-        os.makedirs(os.path.dirname(output_filename), exist_ok=True)
+        print(f"{os.path.basename(filename)=}")
 
-        with open(output_filename, 'w+') as fp:
-            fp.write(html)
+        if os.path.basename(filename) in args.index_article_names:
+            output_filename = os.path.join(
+                    os.path.dirname(re.sub(f"^{args.notes.name}", os.path.join(args.output_dir.name, 'notes', filename))),
+                    'index.html'
+                    )
+            dirs_with_index_article.append(os.path.dirname(re.sub(f"^{args.notes.name}", os.path.join(args.output_dir.name, 'notes'), filename)))
+        else:
+            output_filename = os.path.splitext(re.sub(f"^{args.notes.name}", os.path.join(args.output_dir.name, 'notes'), filename))[0] + '.html'
+
+        fm = frontmatter.load(filename)
+        if isinstance(fm.get('tags'), list):
+            for tag in fm.get('tags'):
+                if tag in tag_dict.keys():
+                    tag_dict[tag].append({
+                        'path': str(pathlib.Path(*pathlib.Path(output_filename).parts[1:])),
+                        'title': fm.get('title')
+                        })
+                else:
+                    tag_dict[tag] = [ {
+                        'path': str(pathlib.Path(*pathlib.Path(output_filename).parts[1:])),
+                        'title': fm.get('title')
+                        } ]
+        with open(filename) as fp:
+            lines = fp.read().split('\n')
+        header_lines = []
+        for line in lines:
+            if re.match('^#{1,6} \S', line):
+                header_lines.append(" ".join(line.split(" ")[1:]))
+
+        print(f"{output_filename=}")
+        all_entries.append({
+            'path': str(pathlib.Path(*pathlib.Path(output_filename).parts[1:])),
+            'title': fm.get('title'),
+            'tags': fm.get('tags'),
+            'headers': header_lines
+        })
+
+        filehistory = git_filehistory(args.notes, filename)
+
+        if update_required(filename, output_filename) or args.force:
+            html = pypandoc.convert_file(filename, 'html', extra_args=[f'--template={args.template}', '-V', f'filehistory={filehistory}'])
+            os.makedirs(os.path.dirname(output_filename), exist_ok=True)
+
+            with open(output_filename, 'w+') as fp:
+                fp.write(html)
 
     print(f"{plaintext_files=}")
     for filename in plaintext_files:
-        output_filename = re.sub(f"^{args.notes.name}", args.output_dir.name, filename) + '.html'
+        filehistory = git_filehistory(args.notes, filename)
+        title = os.path.basename(re.sub(f"^{args.notes.name}", args.output_dir.name, filename))
+        output_filename = re.sub(f"^{args.notes.name}", os.path.join(args.output_dir.name, 'notes'), filename) + '.html'
         os.makedirs(os.path.dirname(output_filename), exist_ok=True)
-        title = os.path.basename(output_filename)
         html = re.sub(r'\$title\$', title, TEXT_ARTICLE_TEMPLATE_HEAD)
+        html = re.sub(r'\$h1title\$', title, html)
         html = re.sub(r'\$raw\$', os.path.basename(filename), html)
+        html = html.replace('$filehistory$', filehistory)
         with open(filename) as fp:
             html += fp.read()
         html += TEXT_ARTICLE_TEMPLATE_FOOT
 
         with open(output_filename, 'w+') as fp:
             fp.write(html)
+        all_entries.append({
+            'path': str(pathlib.Path(*pathlib.Path(output_filename).parts[1:])),
+            'title': title,
+            'tags': [],
+            'headers': []
+        })
 
     print(f"{other_files=}")
     for filename in other_files:
-        output_filename = re.sub(f"^{args.notes.name}", args.output_dir.name, filename)
+        output_filename = re.sub(f"^{args.notes.name}", os.path.join(args.output_dir.name, 'notes'), filename)
         os.makedirs(os.path.dirname(output_filename), exist_ok=True)
+        all_entries.append({
+            'path': str(pathlib.Path(*pathlib.Path(output_filename).parts[1:])),
+            'title': str(pathlib.Path(*pathlib.Path(output_filename).parts[1:])),
+            'tags': [],
+            'headers': []
+        })
         shutil.copyfile(filename, output_filename)
 
+    tagdir = os.path.join(args.output_dir, '.tags')
+    os.makedirs(tagdir, exist_ok=True)
+
+    for tag in tag_dict.keys():
+        html = re.sub(r'\$title\$', f'{tag}', INDEX_TEMPLATE_HEAD)
+        html = re.sub(r'\$h1title\$', f'tag: {tag}', html)
+        html = re.sub(r'\$extra_content\$', '', html)
+
+        for entry in tag_dict[tag]:
+            html += f"<div class=\"article\"><a href=\"/{entry['path']}\">{entry['title']}</a></div>"
+        html += INDEX_TEMPLATE_FOOT
+
+        with open(os.path.join(tagdir, f'{tag}.html'), 'w+') as fp:
+            fp.write(html)
+
+
 
     dirs_to_index = [args.output_dir.name] + get_dirs(args.output_dir)
     print(f"{dirs_to_index=}")
     print(f"{os.path.commonpath(dirs_to_index)=}")
 
     for directory in dirs_to_index:
+        if directory in dirs_with_index_article:
+            continue
         paths = os.listdir(directory)
         print(f"{paths=}")
 
         indexentries = []
         
         for path in paths:
-            if path == 'index.html':
+            print(f"{path=}")
+            if path in [ 'index.html', '.git' ]:
                 continue
 
             fullpath = os.path.join(directory, path)
@@ -141,8 +262,11 @@ def main(args):
                     title = soup.find('title').get_text()
                 except AttributeError:
                     title = path
-            else:
+            elif os.path.isdir(fullpath):
                 title = path
+            else:
+                # don't add plaintext files to index, since they have a html wrapper
+                continue
 
             if title.strip() == '':
                 title = path
@@ -157,6 +281,7 @@ def main(args):
         indexentries.sort(key=lambda entry: entry['isdirectory'], reverse=True)
         
         html = re.sub(r'\$title\$', directory, INDEX_TEMPLATE_HEAD)
+        html = re.sub(r'\$h1title\$', directory, html)
         html = re.sub(r'\$extra_content\$',
                 EXTRA_INDEX_CONTENT if directory == os.path.commonpath(dirs_to_index) else '',
                 html
@@ -170,6 +295,17 @@ def main(args):
             fp.write(html)
 
     shutil.copyfile(args.stylesheet, os.path.join(args.output_dir.name, 'styles.css'))
+    shutil.copyfile(args.fuse, os.path.join(args.output_dir.name, 'fuse.js'))
+    shutil.copyfile(args.searchjs, os.path.join(args.output_dir.name, 'search.js'))
+    with open(os.path.join(args.output_dir.name, 'index.html'), 'w+') as fp:
+        with open(args.home_index) as fp2:
+            html = re.sub(r'\$title\$', args.output_dir.parts[0], fp2.read())
+            html = re.sub(r'\$h1title\$', args.output_dir.parts[0], html)
+
+        html = re.sub(r'\$data\$', json.dumps(all_entries), html)
+
+        fp.write(html)
+    print(tag_dict)
 
     return 0
 

readme.md

@@ -4,13 +4,26 @@ View your notes as a static html site.
 
 ![](./screenshot.png)
 
+
+## Why?
+
+I want to be able to view my notes in a more convenient way.
+I was already writing them in Pandoc markdown and could view them as PDFs but that wasn't quite
+doing it for me:
+
+- It was inconvenient to flick through multiple files of notes to find the right PDF
+- It was annoying to sync to my phone
+- PDFs do not scale so they were hard to read on smaller screens
+- Probably more reasons I can't think of right now
+
+
 ## Install
 
-0. Install [Pandoc](https://pandoc.org/index.html) and [yq](https://github.com/mikefarah/yq)
+0. Install [Pandoc](https://pandoc.org/index.html) and [Pip](https://github.com/pypa/pip)
 
    On arch:
    ```
-   # pacman -S pandoc yq
+   # pacman -S pandoc python-pip
    ```
 
 1. Run `make install` as root
@@ -18,7 +31,7 @@ View your notes as a static html site.
 ## Usage
 
 ```
-$ notes2web.py NOTES_DIRECTORY_1
+$ notes2web.py notes_directory
 ```
 
 Output of `notes2web.py --help`:
@@ -27,7 +40,8 @@ Output of `notes2web.py --help`:
 usage: notes2web.py [-h] [-o OUTPUT_DIR] [-t TEMPLATE] [-H TEMPLATE_TEXT_HEAD]
                     [-f TEMPLATE_TEXT_FOOT] [-i TEMPLATE_INDEX_HEAD]
                     [-I TEMPLATE_INDEX_FOOT] [-s STYLESHEET]
-                    [-e EXTRA_INDEX_CONTENT]
+                    [--home_index HOME_INDEX] [-e EXTRA_INDEX_CONTENT]
+                    [-n INDEX_ARTICLE_NAMES] [-F]
                     notes
 
 positional arguments:
@@ -42,7 +56,11 @@ optional arguments:
   -i TEMPLATE_INDEX_HEAD, --template-index-head TEMPLATE_INDEX_HEAD
   -I TEMPLATE_INDEX_FOOT, --template-index-foot TEMPLATE_INDEX_FOOT
   -s STYLESHEET, --stylesheet STYLESHEET
+  --home_index HOME_INDEX
   -e EXTRA_INDEX_CONTENT, --extra-index-content EXTRA_INDEX_CONTENT
+  -n INDEX_ARTICLE_NAMES, --index-article-names INDEX_ARTICLE_NAMES
+  -F, --force           Generate new output html even if source file was
+                        modified before output html
 ```
 
 The command will generate a website in the `output-dir` directory (`./web` by default).

requirements.txt

@@ -1,3 +1,6 @@
 beautifulsoup4==4.9.3
 pypandoc==1.5
+python-frontmatter==1.0.0
+python-magic==0.4.24
+PyYAML==5.4.1
 soupsieve==2.2.1

search.js

@@ -0,0 +1,73 @@
+const fuse = new Fuse(data, {
+  keys: ['path', 'title', 'tags', 'headers'],
+  includeMatches: true
+})
+
+const searchBar = document.getElementById('search')
+const results = document.getElementById('results')
+
+function callback() {
+  console.log("called")
+  console.log(searchBar.value)
+  results.innerHTML = ''
+  fuse.search(searchBar.value).forEach(r => {
+    console.log(r)
+    wrapper = document.createElement('div')
+    wrapper.className = "article"
+
+    extra_info = document.createElement('p')
+    extra_info.className = "smallText"
+    extra_info.innerHTML = "tags: "
+    if (r.item.tags == null) {
+      extra_info.innerHTML += "none"
+    } else {
+      extra_info.innerHTML += "[" + r.item.tags.join(', ') + ']'
+    }
+    extra_info.innerHTML += ' path: ' + r.item.path
+
+    header_matches_p = document.createElement('p')
+    header_matches_p.className = "smallText"
+    header_matches = []
+
+    r.matches.every(match => {
+      if (header_matches.length > 3) {
+        header_matches.push('...')
+        return false
+      }
+
+      if (match.key === "headers") {
+        display_match = match.value
+        if (match.indices.length >= 1) {
+          match.indices.sort((a, b) => (b[1]-b[0])-(a[1]-a[0]))
+          indexPair = match.indices[0]
+          matching_slice = match.value.slice(indexPair[0], indexPair[1]+1)
+          console.log(matching_slice)
+          console.log(display_match)
+          display_match = display_match.replace(
+            matching_slice,
+            '<span class="matchHighlight">' + matching_slice + '</span>'
+          )
+        }
+        header_matches.push(display_match)
+      }
+      return true
+    })
+
+    header_matches_p.innerHTML += "header_matches: [" + header_matches.join(', ') + ']'
+
+    content = document.createElement('a')
+    content.innerHTML = r.item.title
+    content.href = r.item.path
+
+    wrapper.appendChild(content)
+    wrapper.appendChild(extra_info)
+    if (header_matches.length > 0) {
+      wrapper.appendChild(header_matches_p)
+    }
+
+    results.appendChild(wrapper)
+  })
+}
+
+searchBar.addEventListener('keyup', callback)
+callback()

styles.css

@@ -35,6 +35,7 @@ pre {
 	background-color: #d9d9d9 ;
 	color: #000;
 	padding: 1em;
+	overflow-x: scroll;
 }
 
 details {
@@ -71,3 +72,31 @@ blockquote {
 blockquote * {
 	margin: 0;
 }
+
+#search {
+	padding: 1em;
+	margin: 1em 0 1em 0;
+	font-size: 1em;
+	width: 100%;
+}
+
+#results {
+	overflow-x: scroll;
+}
+
+.smallText {
+	font-size: 0.7em;
+}
+
+.article .smallText {
+	margin: 0
+}
+
+.matchHighlight {
+	background-color: #86c1b9;
+}
+
+#header > * {
+	margin: 0;
+	padding: 0
+}

templates/article.html

@@ -9,6 +9,19 @@
 </head>
 
 <body>
+	<div id="header">
+	<p style="font-size: 0.7em"> tags:
+	$for(tags)$
+	<a href="/.tags/$tags$.html">$tags$</a>$sep$,
+	$endfor$
+	</p>
+	<p class="smallText" id="footer"> written by $author$, generated by <a href="https://git.alv.cx/alvierahman90/notes2web">notes2web</a></p>
+	<details id="commitLog">
+		<summary class="smallText">
+			Commit log (file history)
+		</summary>
+		<pre>$filehistory$</pre>
+	</details>
+	<div>
 	$body$
-	<p style="font-size: 0.7em;"> page generated by <a href="https://github.com/alvierahman90/notes2web">notes2web</a></p>
 </body>

templates/extra_index_content.html

@@ -1 +1,4 @@
-<p>These are my personal notes. Correctness is not guaranteed.</p>
+<p>
+These are my personal notes. Correctness is not guaranteed.
+Browse by tag <a href="/.tags">here</a>.
+</p>

templates/home_index.html

@@ -0,0 +1,24 @@
+<head>
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<meta charset="utf-8"> 
+<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Open+Sans&display=swap" />
+<link rel="stylesheet" type="text/css" href="/styles.css" />
+<title>$title$</title>
+</head>
+<body>
+	<h1>$h1title$</h1>
+<p>
+These are my personal notes. Correctness is not guaranteed.
+Browse <a href="/notes">here</a> or by tag <a href="/.tags">here</a>.
+</p>
+
+<input placeholder="Search" id="search">
+<div id="results">
+</div>
+
+<p style="font-size: 0.7em;"> page generated by <a href="https://github.com/alvierahman90/notes2web">notes2web</a></p>
+
+<script src="/fuse.js"> </script>
+<script> const data = $data$ </script>
+<script src="/search.js"> </script>
+</body>

templates/index.html

@@ -6,7 +6,7 @@
 <title>$title$</title>
 </head>
 <body>
-	<h1>$title$</h1>
+	<h1>$h1title$</h1>
 $body$
 
 <p style="font-size: 0.7em;"> page generated by <a href="https://github.com/alvierahman90/notes2web">notes2web</a></p>

templates/textarticlefoot.html

@@ -1,3 +1,2 @@
 	</pre>
-	<p style="font-size: 0.7em;"> page generated by <a href="https://github.com/alvierahman90/notes2web">notes2web</a></p>
 </body>

templates/textarticlehead.html

@@ -7,6 +7,15 @@
 </head>
 
 <body>
+	<div id="header">
+	<p class="smallText" id="footer"> page generated by <a href="https://git.alv.cx/alvierahman90/notes2web">notes2web</a></p>
+	<details>
+		<summary class="smallText">
+			Commit log (file history)
+		</summary>
+		<pre>$filehistory$</pre>
+	</details>
+	</div>
 	<h1>$title$</h1>
 	<p> This file was not rendered by notes2web because it is a plaintext file, not a markdown
 	file.