+full refactor
+feat: configuration, progress bar, OSV
This commit is contained in:
@@ -0,0 +1,377 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
from .toml_compat import loads as toml_loads
|
||||
|
||||
|
||||
def _xml_text(elem: ET.Element, tag: str) -> Optional[str]:
|
||||
def ns_strip(s: str) -> str:
|
||||
return s.split("}", 1)[-1] if "}" in s else s
|
||||
|
||||
for child in elem:
|
||||
if ns_strip(child.tag) == tag and child.text:
|
||||
return child.text.strip()
|
||||
return None
|
||||
|
||||
|
||||
|
||||
def parse_requirements_txt(text: str) -> List[Dict[str, Any]]:
|
||||
deps: List[Dict[str, Any]] = []
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if "://" in line or line.startswith("git+"):
|
||||
deps.append({"ecosystem": "pypi", "name": line, "spec": None, "note": "non-standard", "scope": "requirements"})
|
||||
continue
|
||||
|
||||
m = re.match(r"^([A-Za-z0-9_.-]+)\s*([<>=!~]=?)\s*([^\s;]+)", line)
|
||||
if m:
|
||||
name, op, ver = m.group(1), m.group(2), m.group(3)
|
||||
deps.append({"ecosystem": "pypi", "name": name, "spec": f"{op}{ver}", "scope": "requirements"})
|
||||
else:
|
||||
deps.append({"ecosystem": "pypi", "name": line, "spec": None, "scope": "requirements"})
|
||||
return deps
|
||||
|
||||
|
||||
def parse_pipfile(text: str) -> List[Dict[str, Any]]:
|
||||
deps: List[Dict[str, Any]] = []
|
||||
data = toml_loads(text)
|
||||
|
||||
for section, scope in (("packages", "dependencies"), ("dev-packages", "devDependencies")):
|
||||
block = data.get(section, {}) or {}
|
||||
for name, spec in block.items():
|
||||
if isinstance(spec, str):
|
||||
deps.append({"ecosystem": "pypi", "name": name, "spec": spec, "scope": scope})
|
||||
elif isinstance(spec, dict):
|
||||
deps.append({"ecosystem": "pypi", "name": name, "spec": spec.get("version"), "scope": scope})
|
||||
else:
|
||||
deps.append({"ecosystem": "pypi", "name": name, "spec": None, "scope": scope})
|
||||
return deps
|
||||
|
||||
|
||||
def parse_pyproject_toml(text: str) -> List[Dict[str, Any]]:
|
||||
deps: List[Dict[str, Any]] = []
|
||||
data = toml_loads(text)
|
||||
|
||||
# PEP 621
|
||||
proj = data.get("project", {}) or {}
|
||||
for item in proj.get("dependencies", []) or []:
|
||||
if isinstance(item, str):
|
||||
deps.append({"ecosystem": "pypi", "name": item, "spec": None, "scope": "project.dependencies"})
|
||||
|
||||
opt = proj.get("optional-dependencies", {}) or {}
|
||||
for group, items in opt.items():
|
||||
for item in items or []:
|
||||
if isinstance(item, str):
|
||||
deps.append({"ecosystem": "pypi", "name": item, "spec": None, "scope": f"optional:{group}"})
|
||||
|
||||
# Poetry
|
||||
tool = data.get("tool", {}) or {}
|
||||
poetry = (tool.get("poetry", {}) or {})
|
||||
|
||||
for section, scope in (("dependencies", "poetry.dependencies"), ("dev-dependencies", "poetry.dev-dependencies")):
|
||||
block = poetry.get(section, {}) or {}
|
||||
for name, spec in block.items():
|
||||
if name.lower() == "python":
|
||||
continue
|
||||
if isinstance(spec, str):
|
||||
deps.append({"ecosystem": "pypi", "name": name, "spec": spec, "scope": scope})
|
||||
elif isinstance(spec, dict):
|
||||
deps.append({"ecosystem": "pypi", "name": name, "spec": spec.get("version"), "scope": scope})
|
||||
else:
|
||||
deps.append({"ecosystem": "pypi", "name": name, "spec": None, "scope": scope})
|
||||
|
||||
group = poetry.get("group", {}) or {}
|
||||
for gname, gdata in group.items():
|
||||
block = (gdata or {}).get("dependencies", {}) or {}
|
||||
for name, spec in block.items():
|
||||
if name.lower() == "python":
|
||||
continue
|
||||
if isinstance(spec, str):
|
||||
deps.append({"ecosystem": "pypi", "name": name, "spec": spec, "scope": f"poetry.group.{gname}"})
|
||||
elif isinstance(spec, dict):
|
||||
deps.append({"ecosystem": "pypi", "name": name, "spec": spec.get("version"), "scope": f"poetry.group.{gname}"})
|
||||
else:
|
||||
deps.append({"ecosystem": "pypi", "name": name, "spec": None, "scope": f"poetry.group.{gname}"})
|
||||
|
||||
return deps
|
||||
|
||||
|
||||
def parse_poetry_lock(text: str) -> List[Dict[str, Any]]:
|
||||
deps: List[Dict[str, Any]] = []
|
||||
data = toml_loads(text)
|
||||
pkgs = data.get("package", []) or []
|
||||
for p in pkgs:
|
||||
name = p.get("name")
|
||||
ver = p.get("version")
|
||||
if name and ver:
|
||||
deps.append({"ecosystem": "pypi", "name": name, "spec": ver, "scope": "lock"})
|
||||
return deps
|
||||
|
||||
|
||||
|
||||
def parse_package_json(text: str) -> List[Dict[str, Any]]:
|
||||
data = json.loads(text)
|
||||
deps: List[Dict[str, Any]] = []
|
||||
for section in ("dependencies", "devDependencies", "optionalDependencies", "peerDependencies"):
|
||||
block = data.get(section, {}) or {}
|
||||
for name, spec in block.items():
|
||||
deps.append({"ecosystem": "npm", "name": name, "spec": spec, "scope": section})
|
||||
return deps
|
||||
|
||||
|
||||
def parse_package_lock_json(text: str) -> List[Dict[str, Any]]:
|
||||
data = json.loads(text)
|
||||
deps: List[Dict[str, Any]] = []
|
||||
|
||||
if isinstance(data.get("packages"), dict):
|
||||
for k, v in data["packages"].items():
|
||||
if not k or not k.startswith("node_modules/"):
|
||||
continue
|
||||
name = k[len("node_modules/"):]
|
||||
ver = (v or {}).get("version")
|
||||
if name and ver:
|
||||
deps.append({"ecosystem": "npm", "name": name, "spec": ver, "scope": "lock"})
|
||||
return deps
|
||||
|
||||
if isinstance(data.get("dependencies"), dict):
|
||||
for name, v in data["dependencies"].items():
|
||||
ver = (v or {}).get("version")
|
||||
if name and ver:
|
||||
deps.append({"ecosystem": "npm", "name": name, "spec": ver, "scope": "lock"})
|
||||
return deps
|
||||
|
||||
return deps
|
||||
|
||||
|
||||
def parse_yarn_lock(text: str) -> List[Dict[str, Any]]:
|
||||
deps: List[Dict[str, Any]] = []
|
||||
current_key: Optional[str] = None
|
||||
|
||||
for raw in text.splitlines():
|
||||
line = raw.rstrip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
if not line.startswith(" ") and line.endswith(":"):
|
||||
key = line[:-1].strip().strip('"').strip("'")
|
||||
first = key.split(",")[0].strip().strip('"').strip("'")
|
||||
current_key = first
|
||||
continue
|
||||
|
||||
if current_key and line.strip().startswith("version "):
|
||||
m = re.match(r'^\s*version\s+"([^"]+)"\s*$', line)
|
||||
if m:
|
||||
ver = m.group(1)
|
||||
last_at = current_key.rfind("@")
|
||||
if last_at > 0:
|
||||
name = current_key[:last_at]
|
||||
deps.append({"ecosystem": "npm", "name": name, "spec": ver, "scope": "lock"})
|
||||
current_key = None
|
||||
|
||||
return deps
|
||||
|
||||
|
||||
def parse_pnpm_lock_yaml(text: str) -> List[Dict[str, Any]]:
|
||||
deps: List[Dict[str, Any]] = []
|
||||
rx = re.compile(r'^\s*/(.+?)@([0-9][^:\s]+):\s*$', re.MULTILINE)
|
||||
for m in rx.finditer(text):
|
||||
name = m.group(1).strip()
|
||||
ver = m.group(2).strip()
|
||||
if name and ver:
|
||||
deps.append({"ecosystem": "npm", "name": name, "spec": ver, "scope": "lock"})
|
||||
return deps
|
||||
|
||||
|
||||
def parse_go_mod(text: str) -> List[Dict[str, Any]]:
|
||||
deps: List[Dict[str, Any]] = []
|
||||
in_require = False
|
||||
for raw in text.splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("//"):
|
||||
continue
|
||||
|
||||
if line.startswith("require ("):
|
||||
in_require = True
|
||||
continue
|
||||
if in_require and line == ")":
|
||||
in_require = False
|
||||
continue
|
||||
|
||||
if line.startswith("require "):
|
||||
line = line[len("require "):].strip()
|
||||
|
||||
parts = line.split()
|
||||
if len(parts) >= 2:
|
||||
name, ver = parts[0], parts[1]
|
||||
if name == "go":
|
||||
continue
|
||||
deps.append({"ecosystem": "go", "name": name, "spec": ver, "scope": "require"})
|
||||
|
||||
return deps
|
||||
|
||||
|
||||
def parse_go_sum(text: str) -> List[Dict[str, Any]]:
|
||||
deps: List[Dict[str, Any]] = []
|
||||
seen = set()
|
||||
for raw in text.splitlines():
|
||||
line = raw.strip()
|
||||
if not line:
|
||||
continue
|
||||
parts = line.split()
|
||||
if len(parts) >= 2:
|
||||
mod, ver = parts[0], parts[1]
|
||||
ver = ver.replace("/go.mod", "")
|
||||
key = (mod, ver)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
deps.append({"ecosystem": "go", "name": mod, "spec": ver, "scope": "sum"})
|
||||
return deps
|
||||
|
||||
|
||||
def parse_cargo_toml(text: str) -> List[Dict[str, Any]]:
|
||||
deps: List[Dict[str, Any]] = []
|
||||
data = toml_loads(text)
|
||||
|
||||
for section, scope in (("dependencies", "dependencies"), ("dev-dependencies", "dev-dependencies"), ("build-dependencies", "build-dependencies")):
|
||||
block = data.get(section, {}) or {}
|
||||
for name, spec in block.items():
|
||||
if isinstance(spec, str):
|
||||
deps.append({"ecosystem": "cargo", "name": name, "spec": spec, "scope": scope})
|
||||
elif isinstance(spec, dict):
|
||||
deps.append({"ecosystem": "cargo", "name": name, "spec": spec.get("version"), "scope": scope})
|
||||
else:
|
||||
deps.append({"ecosystem": "cargo", "name": name, "spec": None, "scope": scope})
|
||||
return deps
|
||||
|
||||
|
||||
def parse_cargo_lock(text: str) -> List[Dict[str, Any]]:
|
||||
deps: List[Dict[str, Any]] = []
|
||||
data = toml_loads(text)
|
||||
pkgs = data.get("package", []) or []
|
||||
for p in pkgs:
|
||||
name = p.get("name")
|
||||
ver = p.get("version")
|
||||
if name and ver:
|
||||
deps.append({"ecosystem": "cargo", "name": name, "spec": ver, "scope": "lock"})
|
||||
return deps
|
||||
|
||||
|
||||
def parse_pom_xml(text: str) -> List[Dict[str, Any]]:
|
||||
deps: List[Dict[str, Any]] = []
|
||||
try:
|
||||
root = ET.fromstring(text)
|
||||
except ET.ParseError:
|
||||
return deps
|
||||
|
||||
for dep in root.iter():
|
||||
if not dep.tag.endswith("dependency"):
|
||||
continue
|
||||
gid = _xml_text(dep, "groupId")
|
||||
aid = _xml_text(dep, "artifactId")
|
||||
ver = _xml_text(dep, "version")
|
||||
scope = _xml_text(dep, "scope") or "compile"
|
||||
if gid and aid:
|
||||
deps.append({"ecosystem": "maven", "name": f"{gid}:{aid}", "spec": ver, "scope": scope})
|
||||
return deps
|
||||
|
||||
|
||||
def parse_gradle_build(text: str) -> List[Dict[str, Any]]:
|
||||
deps: List[Dict[str, Any]] = []
|
||||
rx = re.compile(
|
||||
r"^\s*(implementation|api|compileOnly|runtimeOnly|testImplementation|testCompileOnly)\s*\(?\s*['\"]([^'\"]+)['\"]\s*\)?\s*$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
for m in rx.finditer(text):
|
||||
scope = m.group(1)
|
||||
gav = m.group(2).strip()
|
||||
parts = gav.split(":")
|
||||
if len(parts) >= 2:
|
||||
name = f"{parts[0]}:{parts[1]}"
|
||||
ver = parts[2] if len(parts) >= 3 else None
|
||||
deps.append({"ecosystem": "gradle", "name": name, "spec": ver, "scope": scope})
|
||||
return deps
|
||||
|
||||
|
||||
def parse_csproj(text: str) -> List[Dict[str, Any]]:
|
||||
deps: List[Dict[str, Any]] = []
|
||||
try:
|
||||
root = ET.fromstring(text)
|
||||
except ET.ParseError:
|
||||
return deps
|
||||
|
||||
for elem in root.iter():
|
||||
if not elem.tag.endswith("PackageReference"):
|
||||
continue
|
||||
name = elem.attrib.get("Include") or elem.attrib.get("Update")
|
||||
ver = elem.attrib.get("Version")
|
||||
if ver is None:
|
||||
for ch in elem:
|
||||
if ch.tag.endswith("Version") and ch.text:
|
||||
ver = ch.text.strip()
|
||||
break
|
||||
if name:
|
||||
deps.append({"ecosystem": "nuget", "name": name, "spec": ver, "scope": "csproj"})
|
||||
return deps
|
||||
|
||||
|
||||
def parse_packages_config(text: str) -> List[Dict[str, Any]]:
|
||||
deps: List[Dict[str, Any]] = []
|
||||
try:
|
||||
root = ET.fromstring(text)
|
||||
except ET.ParseError:
|
||||
return deps
|
||||
|
||||
for pkg in root.findall(".//package"):
|
||||
name = pkg.attrib.get("id")
|
||||
ver = pkg.attrib.get("version")
|
||||
if name:
|
||||
deps.append({"ecosystem": "nuget", "name": name, "spec": ver, "scope": "packages.config"})
|
||||
return deps
|
||||
|
||||
|
||||
def parse_manifest_by_name(filename: str, text: str) -> List[Dict[str, Any]]:
|
||||
|
||||
base = os.path.basename(filename)
|
||||
|
||||
if base.startswith("requirements") and base.endswith(".txt"):
|
||||
return parse_requirements_txt(text)
|
||||
if base == "Pipfile":
|
||||
return parse_pipfile(text)
|
||||
if base == "pyproject.toml":
|
||||
return parse_pyproject_toml(text)
|
||||
if base == "poetry.lock":
|
||||
return parse_poetry_lock(text)
|
||||
if base == "package.json":
|
||||
return parse_package_json(text)
|
||||
if base == "package-lock.json":
|
||||
return parse_package_lock_json(text)
|
||||
if base == "yarn.lock":
|
||||
return parse_yarn_lock(text)
|
||||
if base == "pnpm-lock.yaml":
|
||||
return parse_pnpm_lock_yaml(text)
|
||||
if base == "go.mod":
|
||||
return parse_go_mod(text)
|
||||
if base == "go.sum":
|
||||
return parse_go_sum(text)
|
||||
if base == "Cargo.toml":
|
||||
return parse_cargo_toml(text)
|
||||
if base == "Cargo.lock":
|
||||
return parse_cargo_lock(text)
|
||||
if base == "pom.xml":
|
||||
return parse_pom_xml(text)
|
||||
if base in ("build.gradle", "build.gradle.kts"):
|
||||
return parse_gradle_build(text)
|
||||
if base.endswith(".csproj") or base.endswith(".fsproj"):
|
||||
return parse_csproj(text)
|
||||
if base == "packages.config":
|
||||
return parse_packages_config(text)
|
||||
|
||||
return []
|
||||
Reference in New Issue
Block a user