378 lines
13 KiB
Python
378 lines
13 KiB
Python
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import xml.etree.ElementTree as ET
|
||
|
|
from typing import List, Dict, Any, Optional
|
||
|
|
|
||
|
|
from .toml_compat import loads as toml_loads
|
||
|
|
|
||
|
|
|
||
|
|
def _xml_text(elem: ET.Element, tag: str) -> Optional[str]:
|
||
|
|
def ns_strip(s: str) -> str:
|
||
|
|
return s.split("}", 1)[-1] if "}" in s else s
|
||
|
|
|
||
|
|
for child in elem:
|
||
|
|
if ns_strip(child.tag) == tag and child.text:
|
||
|
|
return child.text.strip()
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
def parse_requirements_txt(text: str) -> List[Dict[str, Any]]:
|
||
|
|
deps: List[Dict[str, Any]] = []
|
||
|
|
for line in text.splitlines():
|
||
|
|
line = line.strip()
|
||
|
|
if not line or line.startswith("#"):
|
||
|
|
continue
|
||
|
|
if "://" in line or line.startswith("git+"):
|
||
|
|
deps.append({"ecosystem": "pypi", "name": line, "spec": None, "note": "non-standard", "scope": "requirements"})
|
||
|
|
continue
|
||
|
|
|
||
|
|
m = re.match(r"^([A-Za-z0-9_.-]+)\s*([<>=!~]=?)\s*([^\s;]+)", line)
|
||
|
|
if m:
|
||
|
|
name, op, ver = m.group(1), m.group(2), m.group(3)
|
||
|
|
deps.append({"ecosystem": "pypi", "name": name, "spec": f"{op}{ver}", "scope": "requirements"})
|
||
|
|
else:
|
||
|
|
deps.append({"ecosystem": "pypi", "name": line, "spec": None, "scope": "requirements"})
|
||
|
|
return deps
|
||
|
|
|
||
|
|
|
||
|
|
def parse_pipfile(text: str) -> List[Dict[str, Any]]:
|
||
|
|
deps: List[Dict[str, Any]] = []
|
||
|
|
data = toml_loads(text)
|
||
|
|
|
||
|
|
for section, scope in (("packages", "dependencies"), ("dev-packages", "devDependencies")):
|
||
|
|
block = data.get(section, {}) or {}
|
||
|
|
for name, spec in block.items():
|
||
|
|
if isinstance(spec, str):
|
||
|
|
deps.append({"ecosystem": "pypi", "name": name, "spec": spec, "scope": scope})
|
||
|
|
elif isinstance(spec, dict):
|
||
|
|
deps.append({"ecosystem": "pypi", "name": name, "spec": spec.get("version"), "scope": scope})
|
||
|
|
else:
|
||
|
|
deps.append({"ecosystem": "pypi", "name": name, "spec": None, "scope": scope})
|
||
|
|
return deps
|
||
|
|
|
||
|
|
|
||
|
|
def parse_pyproject_toml(text: str) -> List[Dict[str, Any]]:
|
||
|
|
deps: List[Dict[str, Any]] = []
|
||
|
|
data = toml_loads(text)
|
||
|
|
|
||
|
|
# PEP 621
|
||
|
|
proj = data.get("project", {}) or {}
|
||
|
|
for item in proj.get("dependencies", []) or []:
|
||
|
|
if isinstance(item, str):
|
||
|
|
deps.append({"ecosystem": "pypi", "name": item, "spec": None, "scope": "project.dependencies"})
|
||
|
|
|
||
|
|
opt = proj.get("optional-dependencies", {}) or {}
|
||
|
|
for group, items in opt.items():
|
||
|
|
for item in items or []:
|
||
|
|
if isinstance(item, str):
|
||
|
|
deps.append({"ecosystem": "pypi", "name": item, "spec": None, "scope": f"optional:{group}"})
|
||
|
|
|
||
|
|
# Poetry
|
||
|
|
tool = data.get("tool", {}) or {}
|
||
|
|
poetry = (tool.get("poetry", {}) or {})
|
||
|
|
|
||
|
|
for section, scope in (("dependencies", "poetry.dependencies"), ("dev-dependencies", "poetry.dev-dependencies")):
|
||
|
|
block = poetry.get(section, {}) or {}
|
||
|
|
for name, spec in block.items():
|
||
|
|
if name.lower() == "python":
|
||
|
|
continue
|
||
|
|
if isinstance(spec, str):
|
||
|
|
deps.append({"ecosystem": "pypi", "name": name, "spec": spec, "scope": scope})
|
||
|
|
elif isinstance(spec, dict):
|
||
|
|
deps.append({"ecosystem": "pypi", "name": name, "spec": spec.get("version"), "scope": scope})
|
||
|
|
else:
|
||
|
|
deps.append({"ecosystem": "pypi", "name": name, "spec": None, "scope": scope})
|
||
|
|
|
||
|
|
group = poetry.get("group", {}) or {}
|
||
|
|
for gname, gdata in group.items():
|
||
|
|
block = (gdata or {}).get("dependencies", {}) or {}
|
||
|
|
for name, spec in block.items():
|
||
|
|
if name.lower() == "python":
|
||
|
|
continue
|
||
|
|
if isinstance(spec, str):
|
||
|
|
deps.append({"ecosystem": "pypi", "name": name, "spec": spec, "scope": f"poetry.group.{gname}"})
|
||
|
|
elif isinstance(spec, dict):
|
||
|
|
deps.append({"ecosystem": "pypi", "name": name, "spec": spec.get("version"), "scope": f"poetry.group.{gname}"})
|
||
|
|
else:
|
||
|
|
deps.append({"ecosystem": "pypi", "name": name, "spec": None, "scope": f"poetry.group.{gname}"})
|
||
|
|
|
||
|
|
return deps
|
||
|
|
|
||
|
|
|
||
|
|
def parse_poetry_lock(text: str) -> List[Dict[str, Any]]:
|
||
|
|
deps: List[Dict[str, Any]] = []
|
||
|
|
data = toml_loads(text)
|
||
|
|
pkgs = data.get("package", []) or []
|
||
|
|
for p in pkgs:
|
||
|
|
name = p.get("name")
|
||
|
|
ver = p.get("version")
|
||
|
|
if name and ver:
|
||
|
|
deps.append({"ecosystem": "pypi", "name": name, "spec": ver, "scope": "lock"})
|
||
|
|
return deps
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
def parse_package_json(text: str) -> List[Dict[str, Any]]:
|
||
|
|
data = json.loads(text)
|
||
|
|
deps: List[Dict[str, Any]] = []
|
||
|
|
for section in ("dependencies", "devDependencies", "optionalDependencies", "peerDependencies"):
|
||
|
|
block = data.get(section, {}) or {}
|
||
|
|
for name, spec in block.items():
|
||
|
|
deps.append({"ecosystem": "npm", "name": name, "spec": spec, "scope": section})
|
||
|
|
return deps
|
||
|
|
|
||
|
|
|
||
|
|
def parse_package_lock_json(text: str) -> List[Dict[str, Any]]:
|
||
|
|
data = json.loads(text)
|
||
|
|
deps: List[Dict[str, Any]] = []
|
||
|
|
|
||
|
|
if isinstance(data.get("packages"), dict):
|
||
|
|
for k, v in data["packages"].items():
|
||
|
|
if not k or not k.startswith("node_modules/"):
|
||
|
|
continue
|
||
|
|
name = k[len("node_modules/"):]
|
||
|
|
ver = (v or {}).get("version")
|
||
|
|
if name and ver:
|
||
|
|
deps.append({"ecosystem": "npm", "name": name, "spec": ver, "scope": "lock"})
|
||
|
|
return deps
|
||
|
|
|
||
|
|
if isinstance(data.get("dependencies"), dict):
|
||
|
|
for name, v in data["dependencies"].items():
|
||
|
|
ver = (v or {}).get("version")
|
||
|
|
if name and ver:
|
||
|
|
deps.append({"ecosystem": "npm", "name": name, "spec": ver, "scope": "lock"})
|
||
|
|
return deps
|
||
|
|
|
||
|
|
return deps
|
||
|
|
|
||
|
|
|
||
|
|
def parse_yarn_lock(text: str) -> List[Dict[str, Any]]:
|
||
|
|
deps: List[Dict[str, Any]] = []
|
||
|
|
current_key: Optional[str] = None
|
||
|
|
|
||
|
|
for raw in text.splitlines():
|
||
|
|
line = raw.rstrip()
|
||
|
|
if not line:
|
||
|
|
continue
|
||
|
|
|
||
|
|
if not line.startswith(" ") and line.endswith(":"):
|
||
|
|
key = line[:-1].strip().strip('"').strip("'")
|
||
|
|
first = key.split(",")[0].strip().strip('"').strip("'")
|
||
|
|
current_key = first
|
||
|
|
continue
|
||
|
|
|
||
|
|
if current_key and line.strip().startswith("version "):
|
||
|
|
m = re.match(r'^\s*version\s+"([^"]+)"\s*$', line)
|
||
|
|
if m:
|
||
|
|
ver = m.group(1)
|
||
|
|
last_at = current_key.rfind("@")
|
||
|
|
if last_at > 0:
|
||
|
|
name = current_key[:last_at]
|
||
|
|
deps.append({"ecosystem": "npm", "name": name, "spec": ver, "scope": "lock"})
|
||
|
|
current_key = None
|
||
|
|
|
||
|
|
return deps
|
||
|
|
|
||
|
|
|
||
|
|
def parse_pnpm_lock_yaml(text: str) -> List[Dict[str, Any]]:
|
||
|
|
deps: List[Dict[str, Any]] = []
|
||
|
|
rx = re.compile(r'^\s*/(.+?)@([0-9][^:\s]+):\s*$', re.MULTILINE)
|
||
|
|
for m in rx.finditer(text):
|
||
|
|
name = m.group(1).strip()
|
||
|
|
ver = m.group(2).strip()
|
||
|
|
if name and ver:
|
||
|
|
deps.append({"ecosystem": "npm", "name": name, "spec": ver, "scope": "lock"})
|
||
|
|
return deps
|
||
|
|
|
||
|
|
|
||
|
|
def parse_go_mod(text: str) -> List[Dict[str, Any]]:
|
||
|
|
deps: List[Dict[str, Any]] = []
|
||
|
|
in_require = False
|
||
|
|
for raw in text.splitlines():
|
||
|
|
line = raw.strip()
|
||
|
|
if not line or line.startswith("//"):
|
||
|
|
continue
|
||
|
|
|
||
|
|
if line.startswith("require ("):
|
||
|
|
in_require = True
|
||
|
|
continue
|
||
|
|
if in_require and line == ")":
|
||
|
|
in_require = False
|
||
|
|
continue
|
||
|
|
|
||
|
|
if line.startswith("require "):
|
||
|
|
line = line[len("require "):].strip()
|
||
|
|
|
||
|
|
parts = line.split()
|
||
|
|
if len(parts) >= 2:
|
||
|
|
name, ver = parts[0], parts[1]
|
||
|
|
if name == "go":
|
||
|
|
continue
|
||
|
|
deps.append({"ecosystem": "go", "name": name, "spec": ver, "scope": "require"})
|
||
|
|
|
||
|
|
return deps
|
||
|
|
|
||
|
|
|
||
|
|
def parse_go_sum(text: str) -> List[Dict[str, Any]]:
|
||
|
|
deps: List[Dict[str, Any]] = []
|
||
|
|
seen = set()
|
||
|
|
for raw in text.splitlines():
|
||
|
|
line = raw.strip()
|
||
|
|
if not line:
|
||
|
|
continue
|
||
|
|
parts = line.split()
|
||
|
|
if len(parts) >= 2:
|
||
|
|
mod, ver = parts[0], parts[1]
|
||
|
|
ver = ver.replace("/go.mod", "")
|
||
|
|
key = (mod, ver)
|
||
|
|
if key in seen:
|
||
|
|
continue
|
||
|
|
seen.add(key)
|
||
|
|
deps.append({"ecosystem": "go", "name": mod, "spec": ver, "scope": "sum"})
|
||
|
|
return deps
|
||
|
|
|
||
|
|
|
||
|
|
def parse_cargo_toml(text: str) -> List[Dict[str, Any]]:
|
||
|
|
deps: List[Dict[str, Any]] = []
|
||
|
|
data = toml_loads(text)
|
||
|
|
|
||
|
|
for section, scope in (("dependencies", "dependencies"), ("dev-dependencies", "dev-dependencies"), ("build-dependencies", "build-dependencies")):
|
||
|
|
block = data.get(section, {}) or {}
|
||
|
|
for name, spec in block.items():
|
||
|
|
if isinstance(spec, str):
|
||
|
|
deps.append({"ecosystem": "cargo", "name": name, "spec": spec, "scope": scope})
|
||
|
|
elif isinstance(spec, dict):
|
||
|
|
deps.append({"ecosystem": "cargo", "name": name, "spec": spec.get("version"), "scope": scope})
|
||
|
|
else:
|
||
|
|
deps.append({"ecosystem": "cargo", "name": name, "spec": None, "scope": scope})
|
||
|
|
return deps
|
||
|
|
|
||
|
|
|
||
|
|
def parse_cargo_lock(text: str) -> List[Dict[str, Any]]:
|
||
|
|
deps: List[Dict[str, Any]] = []
|
||
|
|
data = toml_loads(text)
|
||
|
|
pkgs = data.get("package", []) or []
|
||
|
|
for p in pkgs:
|
||
|
|
name = p.get("name")
|
||
|
|
ver = p.get("version")
|
||
|
|
if name and ver:
|
||
|
|
deps.append({"ecosystem": "cargo", "name": name, "spec": ver, "scope": "lock"})
|
||
|
|
return deps
|
||
|
|
|
||
|
|
|
||
|
|
def parse_pom_xml(text: str) -> List[Dict[str, Any]]:
|
||
|
|
deps: List[Dict[str, Any]] = []
|
||
|
|
try:
|
||
|
|
root = ET.fromstring(text)
|
||
|
|
except ET.ParseError:
|
||
|
|
return deps
|
||
|
|
|
||
|
|
for dep in root.iter():
|
||
|
|
if not dep.tag.endswith("dependency"):
|
||
|
|
continue
|
||
|
|
gid = _xml_text(dep, "groupId")
|
||
|
|
aid = _xml_text(dep, "artifactId")
|
||
|
|
ver = _xml_text(dep, "version")
|
||
|
|
scope = _xml_text(dep, "scope") or "compile"
|
||
|
|
if gid and aid:
|
||
|
|
deps.append({"ecosystem": "maven", "name": f"{gid}:{aid}", "spec": ver, "scope": scope})
|
||
|
|
return deps
|
||
|
|
|
||
|
|
|
||
|
|
def parse_gradle_build(text: str) -> List[Dict[str, Any]]:
|
||
|
|
deps: List[Dict[str, Any]] = []
|
||
|
|
rx = re.compile(
|
||
|
|
r"^\s*(implementation|api|compileOnly|runtimeOnly|testImplementation|testCompileOnly)\s*\(?\s*['\"]([^'\"]+)['\"]\s*\)?\s*$",
|
||
|
|
re.MULTILINE,
|
||
|
|
)
|
||
|
|
for m in rx.finditer(text):
|
||
|
|
scope = m.group(1)
|
||
|
|
gav = m.group(2).strip()
|
||
|
|
parts = gav.split(":")
|
||
|
|
if len(parts) >= 2:
|
||
|
|
name = f"{parts[0]}:{parts[1]}"
|
||
|
|
ver = parts[2] if len(parts) >= 3 else None
|
||
|
|
deps.append({"ecosystem": "gradle", "name": name, "spec": ver, "scope": scope})
|
||
|
|
return deps
|
||
|
|
|
||
|
|
|
||
|
|
def parse_csproj(text: str) -> List[Dict[str, Any]]:
|
||
|
|
deps: List[Dict[str, Any]] = []
|
||
|
|
try:
|
||
|
|
root = ET.fromstring(text)
|
||
|
|
except ET.ParseError:
|
||
|
|
return deps
|
||
|
|
|
||
|
|
for elem in root.iter():
|
||
|
|
if not elem.tag.endswith("PackageReference"):
|
||
|
|
continue
|
||
|
|
name = elem.attrib.get("Include") or elem.attrib.get("Update")
|
||
|
|
ver = elem.attrib.get("Version")
|
||
|
|
if ver is None:
|
||
|
|
for ch in elem:
|
||
|
|
if ch.tag.endswith("Version") and ch.text:
|
||
|
|
ver = ch.text.strip()
|
||
|
|
break
|
||
|
|
if name:
|
||
|
|
deps.append({"ecosystem": "nuget", "name": name, "spec": ver, "scope": "csproj"})
|
||
|
|
return deps
|
||
|
|
|
||
|
|
|
||
|
|
def parse_packages_config(text: str) -> List[Dict[str, Any]]:
|
||
|
|
deps: List[Dict[str, Any]] = []
|
||
|
|
try:
|
||
|
|
root = ET.fromstring(text)
|
||
|
|
except ET.ParseError:
|
||
|
|
return deps
|
||
|
|
|
||
|
|
for pkg in root.findall(".//package"):
|
||
|
|
name = pkg.attrib.get("id")
|
||
|
|
ver = pkg.attrib.get("version")
|
||
|
|
if name:
|
||
|
|
deps.append({"ecosystem": "nuget", "name": name, "spec": ver, "scope": "packages.config"})
|
||
|
|
return deps
|
||
|
|
|
||
|
|
|
||
|
|
def parse_manifest_by_name(filename: str, text: str) -> List[Dict[str, Any]]:
|
||
|
|
|
||
|
|
base = os.path.basename(filename)
|
||
|
|
|
||
|
|
if base.startswith("requirements") and base.endswith(".txt"):
|
||
|
|
return parse_requirements_txt(text)
|
||
|
|
if base == "Pipfile":
|
||
|
|
return parse_pipfile(text)
|
||
|
|
if base == "pyproject.toml":
|
||
|
|
return parse_pyproject_toml(text)
|
||
|
|
if base == "poetry.lock":
|
||
|
|
return parse_poetry_lock(text)
|
||
|
|
if base == "package.json":
|
||
|
|
return parse_package_json(text)
|
||
|
|
if base == "package-lock.json":
|
||
|
|
return parse_package_lock_json(text)
|
||
|
|
if base == "yarn.lock":
|
||
|
|
return parse_yarn_lock(text)
|
||
|
|
if base == "pnpm-lock.yaml":
|
||
|
|
return parse_pnpm_lock_yaml(text)
|
||
|
|
if base == "go.mod":
|
||
|
|
return parse_go_mod(text)
|
||
|
|
if base == "go.sum":
|
||
|
|
return parse_go_sum(text)
|
||
|
|
if base == "Cargo.toml":
|
||
|
|
return parse_cargo_toml(text)
|
||
|
|
if base == "Cargo.lock":
|
||
|
|
return parse_cargo_lock(text)
|
||
|
|
if base == "pom.xml":
|
||
|
|
return parse_pom_xml(text)
|
||
|
|
if base in ("build.gradle", "build.gradle.kts"):
|
||
|
|
return parse_gradle_build(text)
|
||
|
|
if base.endswith(".csproj") or base.endswith(".fsproj"):
|
||
|
|
return parse_csproj(text)
|
||
|
|
if base == "packages.config":
|
||
|
|
return parse_packages_config(text)
|
||
|
|
|
||
|
|
return []
|