173 lines
6.8 KiB
Python
Executable File
173 lines
6.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Collect license files from the packages installed in the build environment.
|
|
|
|
The result is evidence for the concrete image build. It does not replace a
|
|
manual compatibility review of new or changed dependencies.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import importlib.metadata as metadata
|
|
import json
|
|
import re
|
|
import shutil
|
|
from pathlib import Path
|
|
from typing import Iterable
|
|
|
|
LICENSE_PREFIXES = ("license", "licence", "copying", "notice", "copyright", "authors")
|
|
|
|
|
|
def safe_name(value: str) -> str:
|
|
value = re.sub(r"[^A-Za-z0-9._+-]+", "-", value.strip())
|
|
return value.strip("-.") or "unknown"
|
|
|
|
|
|
def is_license_name(name: str) -> bool:
|
|
lower = name.lower()
|
|
return any(lower.startswith(prefix) for prefix in LICENSE_PREFIXES)
|
|
|
|
|
|
def unique_target(directory: Path, relative: Path) -> Path:
|
|
candidate = directory / relative
|
|
candidate.parent.mkdir(parents=True, exist_ok=True)
|
|
if not candidate.exists():
|
|
return candidate
|
|
stem, suffix = candidate.stem, candidate.suffix
|
|
index = 2
|
|
while True:
|
|
numbered = candidate.with_name(f"{stem}-{index}{suffix}")
|
|
if not numbered.exists():
|
|
return numbered
|
|
index += 1
|
|
|
|
|
|
def one_line(value: object) -> str:
|
|
return " ".join(str(value or "").split())
|
|
|
|
|
|
def collect_python(output: Path) -> list[dict[str, str]]:
|
|
rows: list[dict[str, str]] = []
|
|
for dist in sorted(metadata.distributions(), key=lambda item: (item.metadata.get("Name") or "").lower()):
|
|
name = dist.metadata.get("Name") or "unknown"
|
|
version = dist.version or "unknown"
|
|
destination = output / "python" / f"{safe_name(name)}-{safe_name(version)}"
|
|
copied: list[str] = []
|
|
for item in dist.files or ():
|
|
if not is_license_name(Path(str(item)).name):
|
|
continue
|
|
source = Path(dist.locate_file(item))
|
|
if not source.is_file():
|
|
continue
|
|
relative = Path(*Path(str(item)).parts[-2:]) if len(Path(str(item)).parts) > 1 else Path(source.name)
|
|
target = unique_target(destination, relative)
|
|
shutil.copy2(source, target)
|
|
copied.append(target.relative_to(output).as_posix())
|
|
classifiers = [value.removeprefix("License :: ") for value in dist.metadata.get_all("Classifier", []) if value.startswith("License :: ")]
|
|
rows.append({
|
|
"ecosystem": "Python",
|
|
"name": name,
|
|
"version": version,
|
|
"license": one_line(dist.metadata.get("License") or "; ".join(classifiers)),
|
|
"homepage": one_line(dist.metadata.get("Home-page") or dist.metadata.get("Project-URL")),
|
|
"files": ", ".join(copied),
|
|
})
|
|
return rows
|
|
|
|
|
|
def node_package_dirs(node_modules: Path) -> Iterable[Path]:
|
|
if not node_modules.is_dir():
|
|
return
|
|
for entry in sorted(node_modules.iterdir(), key=lambda path: path.name.lower()):
|
|
if entry.name.startswith(".") or entry.name == ".bin" or not entry.is_dir():
|
|
continue
|
|
if entry.name.startswith("@"):
|
|
for scoped in sorted(entry.iterdir(), key=lambda path: path.name.lower()):
|
|
if scoped.is_dir() and (scoped / "package.json").is_file():
|
|
yield scoped
|
|
elif (entry / "package.json").is_file():
|
|
yield entry
|
|
|
|
|
|
def license_value(value: object) -> str:
|
|
if isinstance(value, str):
|
|
return value
|
|
if isinstance(value, dict):
|
|
return one_line(value.get("type") or value.get("name"))
|
|
if isinstance(value, list):
|
|
return "; ".join(filter(None, (license_value(item) for item in value)))
|
|
return ""
|
|
|
|
|
|
def collect_node(output: Path, node_modules: Path) -> list[dict[str, str]]:
|
|
rows: list[dict[str, str]] = []
|
|
for package_dir in node_package_dirs(node_modules):
|
|
try:
|
|
package = json.loads((package_dir / "package.json").read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
print(f"WARNING: cannot read {package_dir / 'package.json'}: {exc}")
|
|
continue
|
|
name = str(package.get("name") or package_dir.name)
|
|
version = str(package.get("version") or "unknown")
|
|
destination = output / "node" / f"{safe_name(name)}-{safe_name(version)}"
|
|
copied: list[str] = []
|
|
for source in sorted(package_dir.iterdir(), key=lambda path: path.name.lower()):
|
|
if source.is_file() and is_license_name(source.name):
|
|
target = unique_target(destination, Path(source.name))
|
|
shutil.copy2(source, target)
|
|
copied.append(target.relative_to(output).as_posix())
|
|
repository = package.get("repository")
|
|
if isinstance(repository, dict):
|
|
repository = repository.get("url")
|
|
rows.append({
|
|
"ecosystem": "Node.js",
|
|
"name": name,
|
|
"version": version,
|
|
"license": license_value(package.get("license") or package.get("licenses")),
|
|
"homepage": one_line(package.get("homepage") or repository),
|
|
"files": ", ".join(copied),
|
|
})
|
|
return rows
|
|
|
|
|
|
def write_index(output: Path, rows: list[dict[str, str]]) -> None:
|
|
lines = [
|
|
"# Installed dependency licenses",
|
|
"",
|
|
"Generated from the packages present during this build. Empty metadata fields must be checked against the copied original license files and upstream source.",
|
|
"",
|
|
"| Ecosystem | Package | Version | Declared license | Copied files |",
|
|
"|---|---|---:|---|---|",
|
|
]
|
|
for row in sorted(rows, key=lambda item: (item["ecosystem"], item["name"].lower(), item["version"])):
|
|
escaped = {key: value.replace("|", "\\|") for key, value in row.items()}
|
|
lines.append(f"| {escaped['ecosystem']} | {escaped['name']} | {escaped['version']} | {escaped['license']} | {escaped['files']} |")
|
|
(output / "INDEX.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
(output / "INDEX.json").write_text(json.dumps(rows, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
parser.add_argument("--node-modules", type=Path)
|
|
args = parser.parse_args()
|
|
|
|
output = args.output.resolve()
|
|
if output.exists():
|
|
shutil.rmtree(output)
|
|
output.mkdir(parents=True)
|
|
|
|
rows = collect_python(output)
|
|
if args.node_modules:
|
|
rows.extend(collect_node(output, args.node_modules.resolve()))
|
|
write_index(output, rows)
|
|
|
|
missing = [row for row in rows if not row["license"] or not row["files"]]
|
|
print(f"Collected metadata for {len(rows)} installed packages in {output}")
|
|
if missing:
|
|
print(f"WARNING: {len(missing)} package(s) have missing license metadata or no copied license file; review INDEX.md")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|