codex
251 lines
8.3 KiB
Python
251 lines
8.3 KiB
Python
# Copyright (c) 2023 Oleh Prypin <oleh@pryp.in>
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
import datetime
|
|
import functools
|
|
import io
|
|
import logging
|
|
import os
|
|
import sys
|
|
import urllib.parse
|
|
from collections.abc import Collection, Mapping, Sequence
|
|
from typing import IO, Any, BinaryIO
|
|
|
|
import yaml
|
|
|
|
from properdocs.config.base import _open_config_file
|
|
from properdocs.utils import cache
|
|
from properdocs.utils import yaml as yaml_util
|
|
|
|
SafeLoader: type[yaml.SafeLoader | yaml.CSafeLoader]
|
|
try:
|
|
from yaml import CSafeLoader as SafeLoader
|
|
except ImportError:
|
|
from yaml import SafeLoader
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class YamlLoaderWithSuppressions(SafeLoader): # type: ignore
|
|
pass
|
|
|
|
|
|
# Prevent errors from trying to access external modules which may not be installed yet.
|
|
YamlLoaderWithSuppressions.add_constructor("!ENV", lambda loader, node: None)
|
|
YamlLoaderWithSuppressions.add_constructor("!relative", lambda loader, node: None)
|
|
YamlLoaderWithSuppressions.add_multi_constructor(
|
|
"tag:yaml.org,2002:python/name:", lambda loader, suffix, node: None
|
|
)
|
|
YamlLoaderWithSuppressions.add_multi_constructor(
|
|
"tag:yaml.org,2002:python/object/apply:", lambda loader, suffix, node: None
|
|
)
|
|
|
|
|
|
DEFAULT_PROJECTS_FILE = "https://raw.githubusercontent.com/properdocs/catalog/main/projects.yaml"
|
|
|
|
BUILTIN_PLUGINS = {"search"}
|
|
_BUILTIN_EXTENSIONS = [
|
|
"abbr",
|
|
"admonition",
|
|
"attr_list",
|
|
"codehilite",
|
|
"def_list",
|
|
"extra",
|
|
"fenced_code",
|
|
"footnotes",
|
|
"md_in_html",
|
|
"meta",
|
|
"nl2br",
|
|
"sane_lists",
|
|
"smarty",
|
|
"tables",
|
|
"toc",
|
|
"wikilinks",
|
|
"legacy_attrs",
|
|
"legacy_em",
|
|
]
|
|
BUILTIN_EXTENSIONS = {
|
|
*_BUILTIN_EXTENSIONS,
|
|
*(f"markdown.extensions.{e}" for e in _BUILTIN_EXTENSIONS),
|
|
}
|
|
|
|
_NotFound = ()
|
|
|
|
|
|
def _dig(cfg, keys: str):
|
|
"""
|
|
Receives a string such as 'foo.bar' and returns `cfg['foo']['bar']`, or `_NotFound`.
|
|
|
|
A list of single-item dicts gets converted to a flat dict. This is intended for `plugins` config.
|
|
"""
|
|
key, _, rest = keys.partition(".")
|
|
try:
|
|
cfg = cfg[key]
|
|
except (KeyError, TypeError):
|
|
return _NotFound
|
|
if isinstance(cfg, list):
|
|
orig_cfg = cfg
|
|
cfg = {}
|
|
for item in reversed(orig_cfg):
|
|
if isinstance(item, dict) and len(item) == 1:
|
|
cfg.update(item)
|
|
elif isinstance(item, str):
|
|
cfg[item] = {}
|
|
if not rest:
|
|
return cfg
|
|
return _dig(cfg, rest)
|
|
|
|
|
|
def _strings(obj) -> Sequence[str]:
|
|
if isinstance(obj, str):
|
|
return (obj,)
|
|
else:
|
|
return tuple(obj)
|
|
|
|
|
|
@functools.cache
|
|
def _entry_points(group: str) -> Mapping[str, Any]:
|
|
if sys.version_info >= (3, 10):
|
|
from importlib.metadata import entry_points
|
|
else:
|
|
from importlib_metadata import entry_points
|
|
|
|
eps = {ep.name: ep for ep in entry_points(group=group)}
|
|
log.debug(f"Available '{group}' entry points: {sorted(eps)}")
|
|
return eps
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class _PluginKind:
|
|
projects_key: str
|
|
entry_points_key: str
|
|
|
|
def __str__(self) -> str:
|
|
return self.projects_key.rpartition("_")[-1]
|
|
|
|
|
|
def get_projects_file(path: str | None = None) -> BinaryIO:
|
|
if path is None:
|
|
path = DEFAULT_PROJECTS_FILE
|
|
if urllib.parse.urlsplit(path).scheme in ("http", "https"):
|
|
content = cache.download_and_cache_url(path, datetime.timedelta(days=1))
|
|
else:
|
|
with open(path, "rb") as f:
|
|
content = f.read()
|
|
return io.BytesIO(content)
|
|
|
|
|
|
def get_deps(
|
|
config_file: IO | os.PathLike | str | None = None,
|
|
projects_file: IO | None = None,
|
|
) -> Collection[str]:
|
|
"""
|
|
Print PyPI package dependencies inferred from a properdocs.yml file based on a reverse mapping of known projects.
|
|
|
|
Args:
|
|
config_file: Non-default properdocs.yml file - content as a buffer, or path.
|
|
projects_file: File/buffer that declares all known ProperDocs-related projects.
|
|
The file is in YAML format and contains `projects: [{mkdocs_theme:, mkdocs_plugin:, markdown_extension:}]
|
|
"""
|
|
if isinstance(config_file, (str, os.PathLike)):
|
|
config_file = os.path.abspath(config_file)
|
|
with _open_config_file(config_file) as opened_config_file:
|
|
cfg = yaml_util.yaml_load(opened_config_file, loader=YamlLoaderWithSuppressions)
|
|
if not isinstance(cfg, dict):
|
|
raise ValueError(
|
|
f"The configuration is invalid. Expected a key-value mapping but received {type(cfg)}"
|
|
)
|
|
|
|
packages_to_install = set()
|
|
|
|
if all(c not in cfg for c in ("site_name", "theme", "plugins", "markdown_extensions")):
|
|
log.warning(f"The file {config_file!r} doesn't seem to be a properdocs.yml config file")
|
|
else:
|
|
if _dig(cfg, "theme.locale") not in (_NotFound, "en"):
|
|
packages_to_install.add("properdocs[i18n]")
|
|
else:
|
|
packages_to_install.add("properdocs")
|
|
|
|
try:
|
|
theme = cfg["theme"]["name"]
|
|
except (KeyError, TypeError):
|
|
theme = cfg.get("theme")
|
|
themes = {theme} if theme else set()
|
|
|
|
plugins = set(_strings(_dig(cfg, "plugins"))) - BUILTIN_PLUGINS
|
|
extensions = set(_strings(_dig(cfg, "markdown_extensions"))) - BUILTIN_EXTENSIONS
|
|
|
|
wanted_plugins = (
|
|
(_PluginKind("properdocs_theme", "properdocs.themes"), themes),
|
|
(_PluginKind("mkdocs_theme", "mkdocs.themes"), themes),
|
|
(_PluginKind("properdocs_plugin", "properdocs.plugins"), plugins),
|
|
(_PluginKind("mkdocs_plugin", "mkdocs.plugins"), plugins),
|
|
(_PluginKind("markdown_extension", "markdown.extensions"), extensions),
|
|
)
|
|
for kind, wanted in (wanted_plugins[0], wanted_plugins[2], wanted_plugins[4]):
|
|
log.debug(f"Wanted {kind}s: {sorted(wanted)}")
|
|
|
|
if projects_file is None:
|
|
projects_file = get_projects_file()
|
|
with projects_file:
|
|
projects = yaml.load(projects_file, Loader=SafeLoader)["projects"]
|
|
|
|
for project in projects:
|
|
for kind, wanted in wanted_plugins:
|
|
available = _strings(project.get(kind.projects_key, ()))
|
|
for entry_name in available:
|
|
if ( # Also check theme-namespaced plugin names against the current theme.
|
|
"/" in entry_name
|
|
and theme is not None
|
|
and kind.projects_key in ("properdocs_plugin", "mkdocs_plugin")
|
|
and entry_name.startswith(f"{theme}/")
|
|
and entry_name[len(theme) + 1 :] in wanted
|
|
and entry_name not in wanted
|
|
):
|
|
entry_name = entry_name[len(theme) + 1 :]
|
|
if entry_name in wanted:
|
|
if "pypi_id" in project:
|
|
install_name = project["pypi_id"]
|
|
elif "github_id" in project:
|
|
install_name = "git+https://github.com/{github_id}".format_map(project)
|
|
else:
|
|
log.error(
|
|
f"Can't find how to install {kind} '{entry_name}' although it was identified as {project}"
|
|
)
|
|
continue
|
|
packages_to_install.add(install_name)
|
|
for extra_key, extra_pkgs in project.get("extra_dependencies", {}).items():
|
|
if _dig(cfg, extra_key) is not _NotFound:
|
|
packages_to_install.update(_strings(extra_pkgs))
|
|
|
|
wanted.remove(entry_name)
|
|
|
|
warnings: dict[str, str] = {}
|
|
|
|
for kind, wanted in wanted_plugins:
|
|
for entry_name in sorted(wanted):
|
|
dist_name = None
|
|
ep = _entry_points(kind.entry_points_key).get(entry_name)
|
|
if ep is not None and ep.dist is not None:
|
|
dist_name = ep.dist.name
|
|
base_warning = (
|
|
f"{str(kind).capitalize()} '{entry_name}' is not provided by any registered project"
|
|
)
|
|
if ep is not None:
|
|
warning = base_warning + " but is installed locally"
|
|
if dist_name:
|
|
warning += f" from '{dist_name}'"
|
|
warnings[base_warning] = warning # Always prefer the lesser warning
|
|
else:
|
|
warnings.setdefault(base_warning, base_warning)
|
|
|
|
for warning in warnings.values():
|
|
if " is installed " in warning:
|
|
log.info(warning)
|
|
else:
|
|
log.warning(warning)
|
|
|
|
return sorted(packages_to_install)
|