Weblog by Lars Steinke

Migrating from Mezzanine content management system to static page generation

Mon, 02 Nov 2026 07:09:05 +0000

As the effort of keeping a complex Python software stack serviced outweighed the outcome, I migrated my little weblog to a static website generator (Hugo).

Content conversion was assisted by agentic engineering (using OpenCode) and the resulting conversion script is provided here for whomever it concerns.

Export the Mezzanine/Django fixtures to JSON (I chose to focus on the blog content solely): python manage.py dumpdata --indent 2 --exclude auth.permission --exclude sessions --exclude admin.LogEntry --exclude django_comments --exclude auth.group --exclude twitter --natural-foreign --natural-primary generic.keyword generic.assignedkeyword pages blog galleries redirects > dump_431_excerpt-202610.json
Covert the HTML content (with tags/categories) from the JSON export (in 01_export) to Markdown for Hugo (in hugo/content), using this Python script (with dependencies = ["beautifulsoup4>=4.13,<5", "markdownify>=1.1,<2"]):

#!/usr/bin/env python3
"""One-time import of the Mezzanine fixture and local media into Hugo."""

from __future__ import annotations

import json
import re
import shutil
from collections import defaultdict
from datetime import datetime
from pathlib import Path, PurePosixPath
from textwrap import dedent
from typing import Any

from bs4 import BeautifulSoup
from markdownify import markdownify


ROOT = Path(__file__).resolve().parents[1]
FIXTURE = ROOT / "01_export/dump_431_excerpt-202610.json"
SOURCE_MEDIA = ROOT / "01_export/blog/media"
SITE = ROOT / "hugo"
CONTENT = SITE / "content"
PUBLISHED_STATUS = 2 # Confirmed against the live site's published posts.
AUTHOR_NAME = "Lars Steinke" # Public display name shown by the live site.


def toml_value(value: Any) -> str:
 if isinstance(value, bool):
 return "true" if value else "false"
 if isinstance(value, str):
 return json.dumps(value, ensure_ascii=False)
 if isinstance(value, (int, float)):
 return str(value)
 if isinstance(value, list):
 return "[" + ", ".join(toml_value(item) for item in value) + "]"
 raise TypeError(f"Unsupported TOML value: {type(value).__name__}")


def write_content(path: Path, fields: dict[str, Any], body: str) -> None:
 path.parent.mkdir(parents=True, exist_ok=True)
 front_matter = "+++\n" + "".join(
 f"{key} = {toml_value(value)}\n" for key, value in fields.items() if value is not None
 )
 path.write_text(front_matter + "+++\n" + body.strip() + "\n", encoding="utf-8")


def normalize_html(body: str) -> str:
 body = (
 body.replace("/static/media/uploads/", "/media/uploads/")
 .replace("/static/img/", "/img/")
 )
 gpg_form = re.compile(
 r'<form\b(?=[^>]*action=["\']https://pgp\.cs\.uu\.nl/mk_path\.cgi["\'])[^>]*>.*?</form>',
 re.IGNORECASE | re.DOTALL,
 )
 return gpg_form.sub("{{< gpg-trust-path >}}", body)


def normalize_blog_links(body: str, post_slugs: set[str]) -> str:
 link = re.compile(r'''(?P<quote>["'])(?P<path>/blog/[^"'?#]+)(?P<tail>[?#][^"']*)?(?P=quote)''')

 def add_trailing_slash(match: re.Match[str]) -> str:
 path = match.group("path")
 slug = path.rstrip("/").rsplit("/", 1)[-1]
 if slug in post_slugs and not path.endswith("/"):
 path += "/"
 return f'{match.group("quote")}{path}{match.group("tail") or ""}{match.group("quote")}'

 return link.sub(add_trailing_slash, body)


def promote_multiline_code(body: str) -> str:
 soup = BeautifulSoup(body, "html.parser")
 for code in list(soup.find_all("code")):
 if code.find("br"):
 code_text = dedent(code.get_text(separator="\n", strip=False)).strip()
 if len([line for line in code_text.splitlines() if line.strip()]) < 2:
 continue
 pre = soup.new_tag("pre")
 block = soup.new_tag("code")
 block.string = code_text
 pre.append(block)
 code.replace_with(pre)
 return str(soup)


def convert_html_to_markdown(body: str, post_slugs: set[str]) -> str:
 prepared = promote_multiline_code(body)
 normalized = normalize_blog_links(normalize_html(prepared), post_slugs)
 return markdownify(normalized, heading_style="ATX", bullets="-").strip()


def date_value(value: str | None) -> str | None:
 if not value:
 return None
 parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
 return parsed.isoformat()


def copy_media() -> None:
 for source, target in (
 (SOURCE_MEDIA / "img", SITE / "static/img"),
 (SOURCE_MEDIA / "uploads", SITE / "static/media/uploads"),
 ):
 if not source.is_dir():
 if target.is_dir():
 continue # Tracked copies are sufficient when ignored source media is absent.
 raise FileNotFoundError(f"Expected source or copied media directory: {source}")
 for item in source.rglob("*"):
 if item.is_file():
 relative = item.relative_to(source)
 destination = target / relative
 destination.parent.mkdir(parents=True, exist_ok=True)
 shutil.copy2(item, destination)

 for required in (SITE / "static/img/lsbw.svg", SITE / "static/img/favicon.ico"):
 if not required.is_file():
 raise FileNotFoundError(f"Missing required site asset: {required}")


def load_records() -> list[dict[str, Any]]:
 with FIXTURE.open(encoding="utf-8") as source:
 rows = json.load(source)
 if not isinstance(rows, list):
 raise ValueError(f"Expected a Django fixture list in {FIXTURE}")
 return rows


def convert() -> None:
 rows = load_records()
 copy_media()
 grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
 for row in rows:
 grouped[row["model"]].append(row)

 categories = {
 row["pk"]: row["fields"] for row in grouped["blog.blogcategory"]
 }
 keywords = {
 row["pk"]: row["fields"] for row in grouped["generic.keyword"]
 }
 category_by_slug = {value["slug"]: value for value in categories.values()}
 tag_title_by_slug = {value["slug"]: value["title"] for value in keywords.values()}
 for category in categories.values():
 tag_title_by_slug.setdefault(category["slug"], category["title"])

 keyword_by_post: dict[str, list[str]] = defaultdict(list)
 for row in grouped["generic.assignedkeyword"]:
 fields = row["fields"]
 content_type = fields.get("content_type")
 if content_type == ["blog", "blogpost"]:
 keyword = keywords.get(fields.get("keyword"))
 if keyword:
 keyword_by_post[str(fields.get("object_pk"))].append(keyword["slug"])

 post_rows = sorted(
 (row for row in grouped["blog.blogpost"] if row["fields"].get("status") == PUBLISHED_STATUS),
 key=lambda row: row["fields"].get("publish_date") or "",
 reverse=True,
 )
 posts_by_pk = {str(row["pk"]): row for row in post_rows}
 published_slugs = {row["fields"]["slug"] for row in post_rows}
 post_slugs: set[str] = set()
 archive_months: set[tuple[int, int]] = set()

 (CONTENT / "blog").mkdir(parents=True, exist_ok=True)
 write_content(CONTENT / "_index.md", {"title": "Blog", "type": "home"}, "")
 write_content(CONTENT / "blog" / "_index.md", {"title": "Blog", "type": "blog"}, "")

 for row in post_rows:
 fields = row["fields"]
 slug = fields.get("slug", "")
 if not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", slug):
 raise ValueError(f"Unexpected blog slug in fixture: {slug!r}")
 if slug in post_slugs:
 raise ValueError(f"Duplicate blog slug in fixture: {slug}")
 post_slugs.add(slug)

 post_tag_slugs: list[str] = []
 for category_pk in fields.get("categories", []):
 category = categories.get(category_pk)
 if not category:
 raise ValueError(f"Missing blog category {category_pk} for post {slug}")
 post_tag_slugs.append(category["slug"])

 post_tag_slugs.extend(keyword_by_post.get(str(row["pk"]), []))
 post_tag_slugs = list(dict.fromkeys(post_tag_slugs))
 for tag_slug in post_tag_slugs:
 if tag_slug not in tag_title_by_slug:
 raise ValueError(f"Missing tag term {tag_slug!r} for post {slug}")
 tags = [tag_title_by_slug[tag_slug] for tag_slug in post_tag_slugs]

 related_slugs = []
 for related_pk in fields.get("related_posts", []):
 related = posts_by_pk.get(str(related_pk))
 if related:
 related_slugs.append(related["fields"]["slug"])

 published = date_value(fields.get("publish_date"))
 if not published:
 raise ValueError(f"Published post {slug} has no publish date")
 published_date = datetime.fromisoformat(published)
 archive_months.add((published_date.year, published_date.month))
 front_matter = {
 "title": fields["title"],
 "slug": slug,
 "date": published,
 "lastmod": date_value(fields.get("updated")),
 "draft": False,
 "type": "blog",
 "description": fields.get("description", ""),
 "tags": tags,
 "author": [AUTHOR_NAME],
 "related_posts": related_slugs,
 }
 write_content(
 CONTENT / "blog" / f"{slug}.md",
 front_matter,
 convert_html_to_markdown(fields.get("content", ""), published_slugs),
 )
 (CONTENT / "blog" / f"{slug}.html").unlink(missing_ok=True)

 # Categories become tags; remove generated category pages and redirect their old routes.
 category_slugs = set(category_by_slug)
 for slug in category_slugs:
 (CONTENT / "blog/category" / slug / "_index.md").unlink(missing_ok=True)

 # Hugo taxonomies generate tag term pages dynamically; remove old imported term files.
 for slug in tag_title_by_slug:
 (CONTENT / "blog/tag" / slug / "_index.md").unlink(missing_ok=True)

 (CONTENT / "blog/author/lss" / "_index.md").unlink(missing_ok=True)

 for year, month in archive_months:
 (CONTENT / "blog/archive" / str(year) / str(month) / "_index.md").unlink(missing_ok=True)
 write_content(
 CONTENT / "blog/archive" / "_index.md",
 {"title": "Archive", "type": "archive-index", "draft": False},
 "",
 )

 # Convert the Mezzanine page tree. The dynamic contact form is intentionally
 # omitted; keep its route as a landing page for the static contact details.
 richtext = {row["pk"]: row["fields"] for row in grouped["pages.richtextpage"]}
 gallery = {row["pk"]: row["fields"] for row in grouped["galleries.gallery"]}
 gallery_images = sorted(
 grouped["galleries.galleryimage"], key=lambda row: row.get("pk", 0)
 )
 for row in grouped["pages.page"]:
 fields = row["fields"]
 if fields.get("status") != PUBLISHED_STATUS:
 continue
 slug = fields["slug"].strip("/")
 if slug == "contact":
 body = (
 "See [About](/contact/about/) for profile and contact links, or "
 "[Legal info](/contact/legal/) for the site's legal details."
 )
 path = CONTENT / "contact" / "_index.md"
 elif fields.get("content_model") == "gallery":
 gallery_fields = gallery.get(row["pk"], {})
 body = convert_html_to_markdown(gallery_fields.get("content", ""), published_slugs)
 body += "\n\n{{< gallery >}}"
 gallery_items = []
 for image_row in gallery_images:
 image = image_row["fields"]
 if image.get("gallery") != row["pk"]:
 continue
 relative = PurePosixPath(image["file"])
 if relative.is_absolute() or ".." in relative.parts:
 raise ValueError(f"Unsafe gallery asset path: {image['file']!r}")
 asset = SITE / "static/media" / relative
 if not asset.is_file():
 raise FileNotFoundError(f"Missing gallery image: {asset}")
 url = "/media/" + relative.as_posix()
 gallery_items.append({"url": url, "caption": image.get("description", "")})
 gallery_data_path = SITE / "data/gallery.json"
 gallery_data_path.parent.mkdir(parents=True, exist_ok=True)
 gallery_data_path.write_text(
 json.dumps({"images": gallery_items}, ensure_ascii=False, indent=2) + "\n",
 encoding="utf-8",
 )
 path = CONTENT / f"{slug}.md"
 else:
 page_content = richtext.get(row["pk"], {}).get("content", "")
 body = convert_html_to_markdown(page_content, published_slugs)
 path = CONTENT / f"{slug}.md"

 if slug != "contact":
 (CONTENT / f"{slug}.html").unlink(missing_ok=True)

 page_fields = {
 "title": fields["title"],
 "type": "gallery" if fields.get("content_model") == "gallery" else "pages",
 "date": date_value(fields.get("publish_date")),
 "description": (
 "Contact information and legal details."
 if slug == "contact"
 else fields.get("description", "")
 ),
 "draft": False,
 }
 write_content(path, page_fields, body)

 data_dir = SITE / "data"
 data_dir.mkdir(parents=True, exist_ok=True)
 (data_dir / "sidebar.json").unlink(missing_ok=True)

 print(
 f"Converted {len(post_rows)} published posts, "
 f"{sum(fields.get('status') == PUBLISHED_STATUS for fields in (r['fields'] for r in grouped['pages.page']))} pages, "
 f"and {sum(1 for r in gallery_images)} gallery images into {SITE}"
 )


if __name__ == "__main__":
 convert()

How to counter vCard address book "bit rot"

Fri, 30 Oct 2026 10:25:00 +0000

After a few years and mobile device generations the state of my address book was rather frustrating: Starting with Nokia Symbian S60 (SyncML) over to early Android (using Backups), on to BlackBerry 10 (Card/CalDAV backend) and ultimately back to Android (using a sync app) has introduced numerous duplications and invisible character additions by the respective address book apps (the Samsung Contacts app being the worst).

In the era of agentic engineering that seemed like a good task for the OpenCode agent harness (in combination with Strata as local LLM provider) to create a fix. A few iterations and testing with over 1000 contact entries yielded very promising results - so here is the repo for anyone who might find the resulting Python script useful as well: https://github.com/BOFH2k/vcf-cleaner

How to compare vCard address book entries

Fri, 06 Apr 2018 10:46:40 +0000

Following a little mishap with the DAVdroid app on my Fire HDX Android tablet, I ended up with a lot of duplicate contacts in my CardDAV account. Upon closer inspection I then noticed, that in the process of copying the contacts from the local contact store (on my Blackberry smartphone) to the CardDAV account, certain contact details had been lost and spurious characters (like spaces and question marks) introduced.

Additionally, I had started editing the contact details for proper display in the Android contact app by Blackberry: The app does unfortunately not display the organization info provided for contacts not connected to a person (like institutions) in the address book view - contrary to that view on BlackBerry 10 before.

How to get Mezzanine up and running on cPanel shared hosting

Tue, 13 Feb 2018 19:15:05 +0000

The follwing articles were of much use when switching this weblog from the no longer maintained Nucleus to Mezzanine, a modern blog and content management system, based on the Django web application framework and Python:

Setting up a Python virtual environment (required to be able to install local packages via pip, the Python package installer):
http://janealam.com/2017/10/06/how-to-install-virtualenv-python-for-flask-app-on-cpanel-base-shared-hosting (archived)
Installing Mezzanine with its dependencies (e.g. psycopg2 python support for the PostgreSQL database to be used as content repository):
https://www.rosslaird.com/2013/06/10/first-steps-with-mezzanine (no longer available)
Running Mezzanine through the Phusion Passenger Application Manager available in cPanel (via its WSGI loader):
https://help.dreamhost.com/hc/en-us/articles/215769548-Passenger-and-Python-WSGI

Manual restore of Android databases fails

Fri, 04 Nov 2011 16:11:37 +0000

As described in my earlier post, it is wise to create regular backups of your apps and data to SD card.

This can be accomplished using adb shell or the local Android terminal:
busybox tar cvf /mnt/sdcard/Backup/Full_`busybox date +%Y%m%d`.tar /system /data
The correspondent command to for example restore your text messages would be:
cd /;busybox tar xvf /mnt/sdcard/Backup/Full_20111101.tar data/data/com.android.providers.telephony/databases/mmssms.db
This worked fine up to Android 2.2, provided that you check file ownership and permissions afterwards, e.g.:

Android firmware updates for the LG GW620

Tue, 21 Jun 2011 14:36:00 +0000

The OpenEtna community provides firmware updates for the GW620 phone by LG, which originally came with Android 1.5.

After using the Korean Android 2.2 (Froyo) update provided by LG for a while, I then took the leap to OpenEVE Android 2.3 (Gingerbread) for usability reasons
(2.2 was pretty laggy sometimes).

Make a backup of your apps and data to SD card (this assumes you start off a rooted Froyo firmware with busybox installed) using adb shell or the local Android terminal:
busybox tar cvf /mnt/sdcard/Backup/Full_20110621.tar /system /data
Reboot your device in fastboot mode (please note that mode might not be available, if you have upgraded to Froyo V20 using the offical LG KDZ updater for WIndows): Remove battery, then hook up USB cable while pressing Menu+Space
Backup your old boot and system images using fastboot and a recovery image: fastboot boot gb-recovery.img
Reset the device settings: fastboot -w
Flash the boot image: fastboot flash boot boot234.img
Flash the system image: fastboot flash system system234.img
Apply ZIP patches from SD card using fastboot to ClockWork Mod recovery image: fastboot boot gb-recovery.img
Apply gapps-gb-20110307-signed.zip & boot-gb234-110509-rndis-224mb.zip

Untar your backup from SD card locally and push back data, as required, e.g.:

adb push contacts2.db /data/data/com.android.providers.contacts/databases 
adb push calendar.db /data/data/com.android.providers.calendar/databases 
adb push mmssms.db /data/data/com.android.providers.telephony/databases 
adb push note_pad.db /data/data/org.openintents.notepad/databases 
adb push shopping.db /data/data/org.openintents.shopping/databases
adb push mytracks.db /data/data/com.google.android.maps.mytracks/databases 
adb push safe /data/data/org.openintents.safe/databases 
adb push database /data/data/com.timsu.astrid/databases

…

Depending on the image used, you might have to change the permissions for the files to be able to overwrite them: chmod 666 ... (and back to chmod 660 ... after the push). Also check that the uid/gid combination still agrees with the app directory and adjust using busybox chown app_xy:app_xy ... (unless you are doing a device shell restore via busybox tar, but even then the uid/gid combination might have changed for the new installation)!
For more details please refer to the OpenEtna forum.
Infos on manual backups of Android databases may be found here.

How to dual-boot Windows physically and virtually

Tue, 30 Nov 2010 09:22:55 +0000

While this is no longer possible with Windows Activation Technologies in Win 7, Windows XP allows using Hardware Profiles for a shared installation that can be run either physically or virtually (e.g. using the KVM virtualization stack built into recent Linux kernels, which is a lot more convenient than VMWare Player with its kernel module dependencies, at the price of 3D graphics acceleration).

The recommended sequence for such a setup:

How to repair an ext3 filesystem after mkdosfs format

Thu, 05 Feb 2009 20:42:44 +0000

After shrinking a large ext3 partition using gparted to make space for an additional partition, I did not take sufficient care to notice the leading partition I newly created with cfdisk subsequently was denoted as sdc2 instead of sdc1 (as one might expect).

Thus woe ensued when I issued a mkdosfs -F 32 /dev/sdc1 command without a closer look at the new partition numbering.

So the interesting question now was, what can be salvaged from an ext3 filesystem with overwritten superblock (as well as a few of the inodes)? Luckily, the erroneous reformatting was FAT32, not ext3, which would have caused the inode block pointers to be zeroed, to quote the ext3 FAQ:
“In order to ensure that ext3 can safely resume an unlink after a crash, it actually zeros out the block pointers in the inode, whereas ext2 just marks these blocks as unused in the block bitmaps and marks the inode as “deleted” and leaves the block pointers alone.”

How to boot a real harddisk in VMWare

Mon, 08 Jan 2007 19:53:00 +0000

The knack is to install the VMWare SCSI driver (included with VMWare Tools) before booting the SATA or PATA harddisk.

Then configure it in the VMX config file as follows:

scsi0:0.present = "TRUE"
scsi0:0.fileName = "win_raw.vmdk"
scsi0:0.deviceType = "rawDisk"
scsi0.present = "TRUE"
scsi0.virtualDev = "buslogic"

Here’s the contents of win_raw.vmdk for my harddrive:

# Disk DescriptorFile
version=1
CID=adf9ef51
parentCID=ffffffff
createType="fullDevice"

# Extent description
RW 625142448 FLAT "/dev/sda" 0

# The Disk Data Base
#DDB
ddb.virtualHWVersion = "3"
ddb.geometry.cylinders = "38913"
ddb.geometry.heads = "255"
ddb.geometry.sectors = "63"
ddb.geometry.biosCylinders = "38913"
ddb.geometry.biosHeads = "255"
ddb.geometry.biosSectors = "63"
ddb.adapterType = "buslogic"
ddb.toolsVersion = "6404"

And how to get those parameters? Exents und geometry can for instance be deduced from hdparm /dev/sda: