From 65894a5f7da0d58eebcdb5664ec116b31056a2ab Mon Sep 17 00:00:00 2001 From: Wolfgang Müller Date: Sun, 10 Nov 2024 13:46:44 +0100 Subject: Migrate to proper Python package layout Up until now we've kept the entirety of later(1)'s functionality in one file. Whilst this is perfectly fine for smaller scripts, the functionality of later(1) has grown to a size where this become less feasible. Since we expect it to grow even further, switch to a "proper" source layout sooner rather than later. This will allow easier extension, testing and packaging later on. Since we want to keep installation simple for end-users still, package later(1) as a zipapp [1] in the Makefile. When installed, this file can be executed just like the single Python script before it. Using this way of installation is not mandatory however, the package layout also supports regular installation with pip and development with poetry. [1] https://docs.python.org/3.12/library/zipapp.html --- .gitignore | 1 + Makefile | 15 ++- later | 281 -------------------------------------------------- poetry.lock | 101 ++++++++++++++++++ pyproject.toml | 45 ++++++++ ruff.toml | 18 ---- src/__main__.py | 3 + src/later/__init__.py | 0 src/later/cache.py | 82 +++++++++++++++ src/later/cli.py | 90 ++++++++++++++++ src/later/entries.py | 82 +++++++++++++++ src/later/logger.py | 12 +++ src/later/main.py | 30 ++++++ src/later/utils.py | 18 ++++ 14 files changed, 476 insertions(+), 302 deletions(-) create mode 100644 .gitignore delete mode 100755 later create mode 100644 poetry.lock create mode 100644 pyproject.toml delete mode 100644 ruff.toml create mode 100644 src/__main__.py create mode 100644 src/later/__init__.py create mode 100644 src/later/cache.py create mode 100644 src/later/cli.py create mode 100644 src/later/entries.py create mode 100644 src/later/logger.py create mode 100644 src/later/main.py create mode 100644 src/later/utils.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..524ef89 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/later.pyz diff --git a/Makefile b/Makefile index d9e9eba..2d947c2 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,16 @@ PREFIX ?= /usr/local -install: later later.1 - install -D -m 755 -t '${DESTDIR}${PREFIX}/bin' later +install: later.pyz later.1 + install -D -m 755 later.pyz '${DESTDIR}${PREFIX}/bin/later' install -D -m 644 -t '${DESTDIR}${PREFIX}/share/man/man1' later.1 -.PHONY: install +later.pyz: + python -m zipapp -p "/usr/bin/env python3" -o later.pyz src/ + +clean: + rm -f later.pyz + +test: + pytest -q + +.PHONY: install clean test diff --git a/later b/later deleted file mode 100755 index 5f7f1dc..0000000 --- a/later +++ /dev/null @@ -1,281 +0,0 @@ -#!/usr/bin/env python3 - -import datetime -import getopt -import hashlib -import json -import os -import re -import sys -from dataclasses import dataclass, field -from datetime import datetime as dt -from typing import Protocol - - -class YTDLPLogger: - def debug(self, msg): - pass - - def warning(self, msg): - pass - - def error(self, msg): - print("later: yt-dlp:", msg, file=sys.stderr) - - -def exit(message=""): - sys.exit(f"later: {message}" if message else 0) - - -def usage(message=""): - lines = [] - if message: - lines.append(f"later: {message}") - lines.append("") - - lines.append("usage: later [options] [list]") - lines.append(" [options] [add] item ...") - lines.append(" [options] del item ...") - lines.append("options: [-u | --update-titles]") - - sys.exit("\n".join(lines)) - - -def get_xdg(directory, fallback): - env = f"XDG_{directory.upper()}_HOME" - if env in os.environ: - return os.environ[env] - return os.path.expanduser(fallback) - - -xdg_cache_home = get_xdg("cache", "~/.cache") -xdg_state_home = get_xdg("state", "~/.local/state") - -watch_later_dir = os.path.join(xdg_state_home, "mpv/watch_later") -later_cache_dir = os.path.join(xdg_cache_home, "later") -title_map_file = os.path.join(later_cache_dir, "titles.json") - - -@dataclass -class WatchLaterEntry: - name: str - path: str - mtime: dt - - def format(self, title_map): - def format_time(time): - now = dt.now() - if time < now - datetime.timedelta(days=7): - return time.strftime("%b %d %Y") - return time.strftime("%b %d %H:%M") - - output = [format_time(self.mtime), self.name] - - if title := title_map.get(self.name): - output.append(f"# {title}") - - return "\t".join(output) - - -class TitleMap: - def __init__(self, path, update=False): - self.map = {} - self.path = path - self.update = update - self.commit_to_disk = update - self.ytdl = None - self.seen = set() - - try: - with open(path) as handle: - self.map = json.load(handle) - except FileNotFoundError: - pass - except json.decoder.JSONDecodeError: - # Clobber the title cache if it was corrupted - self.commit_to_disk = True - except Exception as err: - exit(f"cannot read title cache: {err}") - - def mark_seen(self, key): - self.seen.add(key) - - def get(self, key): - if key in self.map: - return self.map[key] - - if not re.fullmatch(r"https?://.*", key): - return None - - if self.update: - return self.extract(key) - - def extract(self, key): - try: - # Make painstakingly sure that we only do this when absolutely - # necessary: importing yt_dlp is noticeably slow :( - import yt_dlp - except ModuleNotFoundError: - exit("yt-dlp was requested, but yt_dlp python module not found") - - if not self.ytdl: - self.ytdl = yt_dlp.YoutubeDL({"logger": YTDLPLogger()}) - - try: - info = self.ytdl.extract_info(key, download=False) - - # The generic extractor uses the filename part of the url as the - # title. Since we already display the URL, this adds no extra - # information. - if info["extractor"] == "generic": - self.map[key] = "" - else: - self.map[key] = info["title"] - - return self.map.get(key) - - except yt_dlp.utils.DownloadError: - pass - - def maybe_commit(self): - if not self.commit_to_disk: - return - - os.makedirs(later_cache_dir, exist_ok=True) - - seen_entries = { - key: value for key, value in self.map.items() if key in self.seen - } - - try: - with open(self.path, "w") as handle: - json.dump(seen_entries, handle) - except OSError as err: - exit(f"cannot write title cache: {err}") - - -class CommandFunction(Protocol): - def __call__(self, args: "Arguments", title_map: TitleMap) -> None: - pass - - -@dataclass -class Command: - name: str - fun: CommandFunction - implies_list: bool = True - args: bool = True - - -@dataclass -class Arguments: - command: Command - update_titles: bool = False - rest: list[str] = field(default_factory=list) - - -def parse_args(argv, commands, default, default_args): - try: - options, args = getopt.gnu_getopt(argv, "u", "update-titles") - except getopt.GetoptError as e: - usage(e) - - parsed_args = Arguments(command=commands[default]) - - for option in options: - match option: - case ("-u", _) | ("--update-titles", _): - parsed_args.update_titles = True - - if args: - if args[0] in commands: - parsed_args.command = commands[args.pop(0)] - else: - parsed_args.command = commands[default_args] - - parsed_args.rest = args - - if parsed_args.rest and not parsed_args.command.args: - usage(f'unexpected argument for "{parsed_args.command.name}"') - - if not parsed_args.rest and parsed_args.command.args: - usage(f'"{parsed_args.command.name}" requires an argument') - - return parsed_args - - -def entries(title_map): - def get_mtime(entry): - return entry.stat().st_mtime - - with os.scandir(watch_later_dir) as entries: - for entry in sorted(entries, key=get_mtime): - if not entry.is_file(): - continue - - mtime = dt.fromtimestamp(get_mtime(entry)) - - with open(entry.path) as handle: - first = handle.readline().rstrip() - - name = entry.name - if first.startswith("# "): - name = first.strip("# ") - - if name == "redirect entry": - continue - - title_map.mark_seen(name) - yield WatchLaterEntry(name=name, path=entry.path, mtime=mtime) - - -def hexdigest(entry): - return hashlib.md5(entry.encode("utf-8")).hexdigest().upper() - - -def list_entries(args, title_map): - for entry in entries(title_map): - print(entry.format(title_map)) - - -def add_entries(args, title_map): - for entry in args.rest: - path = os.path.join(watch_later_dir, hexdigest(entry)) - - try: - with open(path, "x") as handle: - handle.write(f"# {entry}\n") - except FileExistsError: - pass - except OSError as e: - exit(e) - - -def delete_entries(args, title_map): - for entry in args.rest: - path = os.path.join(watch_later_dir, hexdigest(entry)) - - try: - os.remove(path) - except FileNotFoundError: - pass - except OSError as e: - exit(e) - - -commands = { - "add": Command("add", add_entries), - "del": Command("del", delete_entries), - "list": Command("list", list_entries, implies_list=False, args=False), -} - -args = parse_args(sys.argv[1:], commands, "list", "add") - -title_map = TitleMap(title_map_file, update=args.update_titles) - -args.command.fun(args=args, title_map=title_map) - -if args.command.implies_list: - list_entries(args, title_map) - -title_map.maybe_commit() diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..7794395 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,101 @@ +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.2" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, + {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "packaging" +version = "24.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, + {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, +] + +[[package]] +name = "pluggy" +version = "1.5.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pytest" +version = "8.3.3" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"}, + {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.5,<2" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "tomli" +version = "2.0.2" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38"}, + {file = "tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.8" +content-hash = "269640de7b0e5d44d470b09937cffa18fda6c61c5f997e2bbf72aee448bf314a" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7f5435f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,45 @@ +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +name = "later" +version = "0.1.0" +description = "Manage watch_later entries as saved by mpv(1)" +authors = [ + "Juhani Krekelä ", + "Wolfgang Müller " +] +license = "MIT" +homepage = "https://git.oriole.systems/later" +repository = "https://git.oriole.systems/later" +documentation = "https://git.oriole.systems/later/about" + +[tool.poetry.scripts] +later = 'later.main:main' + +[tool.poetry.dependencies] +python = "^3.8" + +[tool.poetry.group.dev.dependencies] +pytest = "^8.0.0" + +[tool.pytest.ini_options] +pythonpath = ["src"] + +[tool.ruff.lint] +# https://docs.astral.sh/ruff/rules/ +select = [ + "F", # pyflakes + "E", # pycodestyle + "W", # pycodestyle + "I", # isort + "UP", # pyupgrade + "A", # flake8-builtins + "B", # flake8-bugbear + "SIM", # flake8-simplify + "FURB" # refurb +] + +[tool.ruff.lint.flake8-builtins] +builtins-ignorelist = ["exit"] diff --git a/ruff.toml b/ruff.toml deleted file mode 100644 index d00d37e..0000000 --- a/ruff.toml +++ /dev/null @@ -1,18 +0,0 @@ -include = ["later"] - -[lint] -# https://docs.astral.sh/ruff/rules/ -select = [ - "F", # pyflakes - "E", # pycodestyle - "W", # pycodestyle - "I", # isort - "UP", # pyupgrade - "A", # flake8-builtins - "B", # flake8-bugbear - "SIM", # flake8-simplify - "FURB" # refurb -] - -[lint.flake8-builtins] -builtins-ignorelist = ["exit"] diff --git a/src/__main__.py b/src/__main__.py new file mode 100644 index 0000000..5f1043d --- /dev/null +++ b/src/__main__.py @@ -0,0 +1,3 @@ +from later import main + +main.main() diff --git a/src/later/__init__.py b/src/later/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/later/cache.py b/src/later/cache.py new file mode 100644 index 0000000..422b242 --- /dev/null +++ b/src/later/cache.py @@ -0,0 +1,82 @@ +import json +import os +import re + +from later.logger import YTDLPLogger + + +class TitleMap: + def __init__(self, path, update=False): + self.map = {} + self.path = path + self.update = update + self.commit_to_disk = update + self.ytdl = None + self.seen = set() + + try: + with open(path) as handle: + self.map = json.load(handle) + except FileNotFoundError: + pass + except json.decoder.JSONDecodeError: + # Clobber the title cache if it was corrupted + self.commit_to_disk = True + except Exception as err: + exit(f"cannot read title cache: {err}") + + def mark_seen(self, key): + self.seen.add(key) + + def get(self, key): + if key in self.map: + return self.map[key] + + if not re.fullmatch(r"https?://.*", key): + return None + + if self.update: + return self.extract(key) + + def extract(self, key): + try: + # Make painstakingly sure that we only do this when absolutely + # necessary: importing yt_dlp is noticeably slow :( + import yt_dlp + except ModuleNotFoundError: + exit("yt-dlp was requested, but yt_dlp python module not found") + + if not self.ytdl: + self.ytdl = yt_dlp.YoutubeDL({"logger": YTDLPLogger()}) + + try: + info = self.ytdl.extract_info(key, download=False) + + # The generic extractor uses the filename part of the url as the + # title. Since we already display the URL, this adds no extra + # information. + if info["extractor"] == "generic": + self.map[key] = "" + else: + self.map[key] = info["title"] + + return self.map.get(key) + + except yt_dlp.utils.DownloadError: + pass + + def maybe_commit(self): + if not self.commit_to_disk: + return + + os.makedirs(os.path.dirname(self.path), exist_ok=True) + + seen_entries = { + key: value for key, value in self.map.items() if key in self.seen + } + + try: + with open(self.path, "w") as handle: + json.dump(seen_entries, handle) + except OSError as err: + exit(f"cannot write title cache: {err}") diff --git a/src/later/cli.py b/src/later/cli.py new file mode 100644 index 0000000..809632f --- /dev/null +++ b/src/later/cli.py @@ -0,0 +1,90 @@ +import getopt +import sys +from dataclasses import dataclass, field +from typing import Protocol + +from later.entries import EntryManager + + +def usage(message=""): + lines = [] + if message: + lines.append(f"later: {message}") + lines.append("") + + lines.append("usage: later [options] [list]") + lines.append(" [options] [add] item ...") + lines.append(" [options] del item ...") + lines.append("options: [-u | --update-titles]") + + sys.exit("\n".join(lines)) + + +class CommandFunction(Protocol): + def __call__(self, args: "Arguments", entries: EntryManager) -> None: + pass + + +@dataclass +class Command: + name: str + fun: CommandFunction + implies_list: bool = True + args: bool = True + + +@dataclass +class Arguments: + command: Command + update_titles: bool = False + rest: list[str] = field(default_factory=list) + + +def add_entries(args, entries): + for entry in args.rest: + entries.add(entry) + + +def delete_entries(args, entries): + for entry in args.rest: + entries.delete(entry) + + +def list_entries(args, entries): + entries.print() + + +def parse_args(argv, default, default_args): + commands = { + "add": Command("add", add_entries), + "del": Command("del", delete_entries), + "list": Command("list", list_entries, implies_list=False, args=False), + } + + try: + options, args = getopt.gnu_getopt(argv, "u", "update-titles") + except getopt.GetoptError as e: + usage(e) + + parsed_args = Arguments(command=commands[default]) + + for option in options: + match option: + case ("-u", _) | ("--update-titles", _): + parsed_args.update_titles = True + + if args: + if args[0] in commands: + parsed_args.command = commands[args.pop(0)] + else: + parsed_args.command = commands[default_args] + + parsed_args.rest = args + + if parsed_args.rest and not parsed_args.command.args: + usage(f'unexpected argument for "{parsed_args.command.name}"') + + if not parsed_args.rest and parsed_args.command.args: + usage(f'"{parsed_args.command.name}" requires an argument') + + return parsed_args diff --git a/src/later/entries.py b/src/later/entries.py new file mode 100644 index 0000000..ed0df87 --- /dev/null +++ b/src/later/entries.py @@ -0,0 +1,82 @@ +import datetime +import os +from dataclasses import dataclass +from datetime import datetime as dt + +from later.utils import hexdigest + + +@dataclass +class WatchLaterEntry: + name: str + path: str + mtime: dt + + def format(self, title_map): + def format_time(time): + now = dt.now() + if time < now - datetime.timedelta(days=7): + return time.strftime("%b %d %Y") + return time.strftime("%b %d %H:%M") + + output = [format_time(self.mtime), self.name] + + if title := title_map.get(self.name): + output.append(f"# {title}") + + return "\t".join(output) + + +class EntryManager: + def __init__(self, directory, title_map): + self.watch_later_dir = directory + self.title_map = title_map + + def walk_entries(self): + def get_mtime(entry): + return entry.stat().st_mtime + + with os.scandir(self.watch_later_dir) as entries: + for entry in sorted(entries, key=get_mtime): + if not entry.is_file(): + continue + + mtime = dt.fromtimestamp(get_mtime(entry)) + + with open(entry.path) as handle: + first = handle.readline().rstrip() + + name = entry.name + if first.startswith("# "): + name = first.strip("# ") + + if name == "redirect entry": + continue + + self.title_map.mark_seen(name) + yield WatchLaterEntry(name=name, path=entry.path, mtime=mtime) + + def print(self): + for entry in self.walk_entries(): + print(entry.format(self.title_map)) + + def add(self, entry): + path = os.path.join(self.watch_later_dir, hexdigest(entry)) + + try: + with open(path, "x") as handle: + handle.write(f"# {entry}\n") + except FileExistsError: + pass + except OSError as e: + exit(e) + + def delete(self, entry): + path = os.path.join(self.watch_later_dir, hexdigest(entry)) + + try: + os.remove(path) + except FileNotFoundError: + pass + except OSError as e: + exit(e) diff --git a/src/later/logger.py b/src/later/logger.py new file mode 100644 index 0000000..a2c4ae7 --- /dev/null +++ b/src/later/logger.py @@ -0,0 +1,12 @@ +import sys + + +class YTDLPLogger: + def debug(self, msg): + pass + + def warning(self, msg): + pass + + def error(self, msg): + print("later: yt-dlp:", msg, file=sys.stderr) diff --git a/src/later/main.py b/src/later/main.py new file mode 100644 index 0000000..1d07cac --- /dev/null +++ b/src/later/main.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +import os +import sys + +from later.cache import TitleMap +from later.cli import list_entries, parse_args +from later.entries import EntryManager +from later.utils import get_xdg + +xdg_cache_home = get_xdg("cache", "~/.cache") +xdg_state_home = get_xdg("state", "~/.local/state") + +watch_later_dir = os.path.join(xdg_state_home, "mpv/watch_later") +later_cache_dir = os.path.join(xdg_cache_home, "later") +title_map_file = os.path.join(later_cache_dir, "titles.json") + + +def main(): + args = parse_args(sys.argv[1:], "list", "add") + + title_map = TitleMap(title_map_file, update=args.update_titles) + entries = EntryManager(watch_later_dir, title_map) + + args.command.fun(args=args, entries=entries) + + if args.command.implies_list: + list_entries(args, entries) + + title_map.maybe_commit() diff --git a/src/later/utils.py b/src/later/utils.py new file mode 100644 index 0000000..a03b540 --- /dev/null +++ b/src/later/utils.py @@ -0,0 +1,18 @@ +import hashlib +import os +import sys + + +def hexdigest(entry): + return hashlib.md5(entry.encode("utf-8")).hexdigest().upper() + + +def exit(message=""): + sys.exit(f"later: {message}" if message else 0) + + +def get_xdg(directory, fallback): + env = f"XDG_{directory.upper()}_HOME" + if env in os.environ: + return os.environ[env] + return os.path.expanduser(fallback) -- cgit v1.2.3-2-gb3c3