From 832c0bc7d90648264e14ac0b2df82ea8704ea7fa Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Wed, 23 Apr 2025 18:41:21 -0500 Subject: [PATCH 01/63] Move pypi flattent to pypi --- src/github_helper/api/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 8b521be..e4f5ae5 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -334,7 +334,7 @@ async def get_project_configs(self, repo): sadness = int(not projects) return projects, sadness - async def get_pypi(self, repo, *, testing=False): + async def get_pypi(self, repo, *, testing=False, flatten=False): """Get all pypi releases for a particular project.""" project_configs, sadness = await self.get_project_configs(repo) project_names = set() @@ -371,16 +371,20 @@ async def fetch_json(name): for name in project_names: releases[name] = await fetch_json(name) sadness = int(not releases) - _logger.debug2(releases) + if flatten: + releases = [release for project in releases.values() for release in project] return releases, sadness async def audit_pypi(self, repo, count=7, *, testing=False): """Get all pypi releases for a project and audit it.""" - releases, sadness = await self.get_pypi(repo, testing=testing) + releases, sadness = await self.get_pypi( + repo, + testing=testing, + flatten=True, + ) if sadness: return None, sadness - releases = [release for project in releases.values() for release in project] for release in releases: release["audit"] = _compare_versions.ReleaseAudit( release, @@ -416,7 +420,6 @@ async def get_releases(self, repo): _log_one_json(obj) releases = releases_jq.input_value(obj).first() sadness = int(not releases) - _logger.debug2(releases) return releases, sadness async def audit_releases(self, repo, count=7): From 0d73ef4468cc5925edb0094fcddd18513ac69a03 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Wed, 23 Apr 2025 18:41:44 -0500 Subject: [PATCH 02/63] Add pypi into audit-versions --- src/github_helper/api/__init__.py | 37 +++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index e4f5ae5..6263cbc 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -1,6 +1,7 @@ """A CLI dashboard for github status.""" import asyncio +import colored import re import tomllib import warnings @@ -447,26 +448,48 @@ async def audit_versions(self, repo): "repo" and owner is assumed to be the current user. """ - tags, sadness = await self.get_remote_tags(repo) - releases, sadness = await self.get_releases(repo) + # puede mezclar proyectos acá + # todavia no probamos con mas de un projection en repositorio + async with asyncio.TaskGroup() as tg: + tags_task = tg.create_task(self.get_remote_tags(repo)) + releases_task = tg.create_task(self.get_releases(repo)) + pypi_task = tg.create_task(self.get_pypi(repo, flatten=True)) + test_pypi_task = tg.create_task( + self.get_pypi(repo, testing=True, flatten=True), + ) + # que hacemos con sadness? + tags, _ = await tags_task + releases, _ = await releases_task + pypi, _ = await pypi_task + test_pypi, _ = await test_pypi_task filtered_tags = _compare_versions.filter_versions(tags, "tag") filtered_releases = _compare_versions.filter_versions(releases, "tag") + filtered_pypi = _compare_versions.filter_versions(pypi, "tag") + filtered_test_pypi = _compare_versions.filter_versions( + test_pypi, + "tag", + ) - versions = filtered_tags | filtered_releases - + versions = ( + filtered_tags | filtered_releases | filtered_pypi | filtered_test_pypi + ) + yes = f"{colored.Fore.green}True{colored.Style.reset}" + no = f"{colored.Fore.red}False{colored.Style.reset}" result = _compare_versions.order_versions( [ { "version": v, - "tags": v in filtered_tags, - "releases": v in filtered_releases, + "tags": yes if v in filtered_tags else no, + "releases": yes if v in filtered_releases else no, + "pypi": yes if v in filtered_pypi else no, + "test.pypi": yes if v in filtered_test_pypi else no, } for v in versions ], "version", ) - return result, sadness + return result, 0 # TODO: no sadness for audit? async def _get_ruleset(self, owner, repo, ruleset_id): """Return releset for a user by Id.""" From 154b78d6f5d10d9477efbb8903711cdea92ada40 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Wed, 23 Apr 2025 18:45:50 -0500 Subject: [PATCH 03/63] Append pypi version with v --- src/github_helper/api/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 6263cbc..41ff708 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -1,13 +1,13 @@ """A CLI dashboard for github status.""" import asyncio -import colored import re import tomllib import warnings from pathlib import Path import aiohttp +import colored import jq # type: ignore [import-not-found] import logistro import orjson @@ -354,7 +354,7 @@ async def fetch_json(name): jq_dir = ( r".releases // {} | " r"to_entries | map(" - r"{tag: .key, files:" + r'{tag: "v\(.key)", files:' r"[ .value[] | select(.yank != true) | .filename ]" r"})" ) @@ -449,6 +449,7 @@ async def audit_versions(self, repo): """ # puede mezclar proyectos acá + # Ignoramos nombre de proyecto # todavia no probamos con mas de un projection en repositorio async with asyncio.TaskGroup() as tg: tags_task = tg.create_task(self.get_remote_tags(repo)) From 9ba76a133fc14d23abf4af283c35e0eaec199c2f Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Thu, 24 Apr 2025 10:36:08 -0500 Subject: [PATCH 04/63] Move tests out of package directory --- {src/integration_test => integration_test}/test.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {src/integration_test => integration_test}/test.sh (100%) diff --git a/src/integration_test/test.sh b/integration_test/test.sh similarity index 100% rename from src/integration_test/test.sh rename to integration_test/test.sh From e7ae2d8a70a4ea0e8d305ab96e73f11a6e279a91 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Fri, 25 Apr 2025 12:15:43 -0500 Subject: [PATCH 05/63] Fix versions to use audit --- src/github_helper/api/__init__.py | 93 ++++++++++++++-------- src/github_helper/api/_compare_versions.py | 43 +++++++++- 2 files changed, 99 insertions(+), 37 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 41ff708..8bb3cc0 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -286,19 +286,6 @@ async def query_version(repo): sadness = int(not repos) return repos, sadness - async def get_remote_tags(self, repo): - """Return tags for a repo.""" - _ = await self.get_user() - tags_jq = jq.compile("map({tag: .name})") - owner, repo = self._split_full_name(full_name=repo) - endpoint = f"repos/{owner}/{repo}/tags" - _logger.debug(f"Calling API: {endpoint}") - retval, out, err = await srv.gh_api(endpoint) - srv.check_retval(retval, err, endpoint=endpoint) - tags = tags_jq.input_value(orjson.loads(out)).first() - sadness = int(not tags) - return tags, sadness - async def get_project_configs(self, repo): """Find all projects in a repo.""" owner, repo = self._split_full_name(full_name=repo) @@ -335,8 +322,23 @@ async def get_project_configs(self, repo): sadness = int(not projects) return projects, sadness + async def get_remote_tags(self, repo): + """Return tags ("tag":"name") for a repo.""" + _ = await self.get_user() + tags_jq = jq.compile("map({tag: .name})") + owner, repo = self._split_full_name(full_name=repo) + endpoint = f"repos/{owner}/{repo}/tags" + _logger.debug(f"Calling API: {endpoint}") + retval, out, err = await srv.gh_api(endpoint) + srv.check_retval(retval, err, endpoint=endpoint) + tags = tags_jq.input_value(orjson.loads(out)).first() + sadness = int(not tags) + return tags, sadness + + # probably need to handle specific projects + # project metadata usually has github repo async def get_pypi(self, repo, *, testing=False, flatten=False): - """Get all pypi releases for a particular project.""" + """Get all pypi releases for ALL projects in a repo.""" project_configs, sadness = await self.get_project_configs(repo) project_names = set() if "py" in project_configs: @@ -350,12 +352,11 @@ async def fetch_json(name): url = f"https://{prefix}pypi.org/pypi/{name}/json" _logger.debug(url) try: - # TODO: probably need to check that project exists first jq_dir = ( r".releases // {} | " r"to_entries | map(" r'{tag: "v\(.key)", files:' - r"[ .value[] | select(.yank != true) | .filename ]" + r"[ .value[] | select(.yanked != true) | .filename ]" r"})" ) pypi_jq = jq.compile(jq_dir) @@ -373,6 +374,7 @@ async def fetch_json(name): releases[name] = await fetch_json(name) sadness = int(not releases) if flatten: + # two objects may have same tag releases = [release for project in releases.values() for release in project] return releases, sadness @@ -418,6 +420,7 @@ async def get_releases(self, repo): retval, out, err = await srv.gh_api(endpoint) srv.check_retval(retval, err, endpoint=endpoint) obj = orjson.loads(out) + _logger.debug2("get_releases") _log_one_json(obj) releases = releases_jq.input_value(obj).first() sadness = int(not releases) @@ -453,44 +456,64 @@ async def audit_versions(self, repo): # todavia no probamos con mas de un projection en repositorio async with asyncio.TaskGroup() as tg: tags_task = tg.create_task(self.get_remote_tags(repo)) - releases_task = tg.create_task(self.get_releases(repo)) - pypi_task = tg.create_task(self.get_pypi(repo, flatten=True)) - test_pypi_task = tg.create_task( - self.get_pypi(repo, testing=True, flatten=True), - ) + releases_task = tg.create_task(self.audit_releases(repo)) + pypi_task = tg.create_task(self.audit_pypi(repo)) + test_pypi_task = tg.create_task(self.audit_pypi(repo, testing=True)) # que hacemos con sadness? tags, _ = await tags_task releases, _ = await releases_task pypi, _ = await pypi_task test_pypi, _ = await test_pypi_task - filtered_tags = _compare_versions.filter_versions(tags, "tag") - filtered_releases = _compare_versions.filter_versions(releases, "tag") - filtered_pypi = _compare_versions.filter_versions(pypi, "tag") - filtered_test_pypi = _compare_versions.filter_versions( - test_pypi, - "tag", + c_tags = _compare_versions.conform_versions( + _compare_versions.filter_versions(tags), ) + c_releases = _compare_versions.conform_versions(releases) + c_pypi = _compare_versions.conform_versions(pypi) + c_test_pypi = _compare_versions.conform_versions(test_pypi) - versions = ( - filtered_tags | filtered_releases | filtered_pypi | filtered_test_pypi + all_versions = ( + c_tags.keys() | c_releases.keys() | c_pypi.keys() | c_test_pypi.keys() ) + yes = f"{colored.Fore.green}True{colored.Style.reset}" no = f"{colored.Fore.red}False{colored.Style.reset}" + + def empty(x="Empty"): + return f"{colored.Fore.yellow}{x}{colored.Style.reset}" + result = _compare_versions.order_versions( [ { "version": v, - "tags": yes if v in filtered_tags else no, - "releases": yes if v in filtered_releases else no, - "pypi": yes if v in filtered_pypi else no, - "test.pypi": yes if v in filtered_test_pypi else no, + "tags": (no if v not in c_tags else yes), + "releases": ( + no + if v not in c_releases + else empty(empty) + if c_releases[v]["empty"] + else yes + ), + "pypi": ( + no + if v not in c_pypi + else empty("yanked") + if c_pypi[v]["empty"] + else yes + ), + "test.pypi": ( + no + if v not in c_test_pypi + else empty("yanked") + if c_test_pypi[v]["empty"] + else yes + ), } - for v in versions + for v in all_versions ], "version", ) - return result, 0 # TODO: no sadness for audit? + return result, 0 # maybe no sadness for audit? async def _get_ruleset(self, owner, repo, ruleset_id): """Return releset for a user by Id.""" diff --git a/src/github_helper/api/_compare_versions.py b/src/github_helper/api/_compare_versions.py index 5ce6643..3e0d02d 100644 --- a/src/github_helper/api/_compare_versions.py +++ b/src/github_helper/api/_compare_versions.py @@ -42,6 +42,12 @@ def __getattr__(self, name): colored.Fore = colored.Back = colored.Style = NoColor() +# maybe combine these functions into a "normal versions" +# we talk a list of dictionaries, all have to have a "tag" key (version) +# and what would we mark? +# a) we'd sort. +# b) we'd regulate how the tag is expressed (v or no) +# c) we'd determine if its empty or malformed def order_versions(versions: list[dict], key: str): if not versions: return versions @@ -52,12 +58,45 @@ def order_versions(versions: list[dict], key: str): ) -def filter_versions(versions: list[dict], key: str): +def conform_versions(versions: list[dict]): + versions_dict = {} + for v in versions: + v["conformant"] = None + try: + version.Version(v["tag"]) + v["conformant"] = "Python" + except version.InvalidVersion: + pass + try: + semver.Version.parse(v["tag"]) + v["conformant"] = "SemVer (not Python)" + except ValueError: + pass + if v["conformant"]: + if v["tag"].startswith("V"): + v["tag"][0] = "v" + elif not v["tag"].startswith("v"): + v["tag"] = f"v{v['tag']}" + # always true for tag, use audit + _logger.debug2(f"Files in {v['tag']}: {len(v.get('files', []))}") + v["empty"] = not bool(v.get("files")) + _logger.debug2(f"Empty {v['empty']} from {v.get('files')}") + if v["tag"] in versions_dict: + cur = versions_dict[v["tag"]] + cur["empty"] = v["empty"] + cur["files"].extend(v["files"]) + else: + versions_dict[v["tag"]] = v + return versions_dict + + +def filter_versions(versions: list[dict]): + # this needs to return a list of dictionaries _regex = re.compile( r"^" + version.VERSION_PATTERN + r"$", re.VERBOSE, ) - return {v[key] for v in versions if _regex.match(v[key])} + return [v for v in versions if _regex.match(v.get("tag"))] bdist_template = { From 0091dba0204066189525c15dae945d561dfcf0a6 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Fri, 25 Apr 2025 14:08:12 -0500 Subject: [PATCH 06/63] Improve version-tag check --- src/github_helper/api/__init__.py | 22 +++++++----- src/github_helper/api/_compare_versions.py | 42 +++++++++++----------- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 8bb3cc0..6319635 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -476,8 +476,11 @@ async def audit_versions(self, repo): c_tags.keys() | c_releases.keys() | c_pypi.keys() | c_test_pypi.keys() ) - yes = f"{colored.Fore.green}True{colored.Style.reset}" - no = f"{colored.Fore.red}False{colored.Style.reset}" + def yes(x="True"): + return f"{colored.Fore.green}{x}{colored.Style.reset}" + + def no(x="False"): + return f"{colored.Fore.red}{x}{colored.Style.reset}" def empty(x="Empty"): return f"{colored.Fore.yellow}{x}{colored.Style.reset}" @@ -486,28 +489,29 @@ def empty(x="Empty"): [ { "version": v, - "tags": (no if v not in c_tags else yes), + "tags": (no() if v not in c_tags else yes()), "releases": ( - no + no() if v not in c_releases else empty(empty) if c_releases[v]["empty"] - else yes + else yes() ), "pypi": ( - no + no() if v not in c_pypi else empty("yanked") if c_pypi[v]["empty"] - else yes + else yes() ), "test.pypi": ( - no + no() if v not in c_test_pypi else empty("yanked") if c_test_pypi[v]["empty"] - else yes + else yes() ), + "tag valid": (_compare_versions.check_conformant(v)[1] or no()), } for v in all_versions ], diff --git a/src/github_helper/api/_compare_versions.py b/src/github_helper/api/_compare_versions.py index 3e0d02d..8ce9ab8 100644 --- a/src/github_helper/api/_compare_versions.py +++ b/src/github_helper/api/_compare_versions.py @@ -58,20 +58,30 @@ def order_versions(versions: list[dict], key: str): ) +def check_conformant(name): + try: + parsed = version.Version(name) + # we have to do this reverse check + # because python is flexible/tolerant with bad versions + if str(parsed) != name[1:] if name.startswith("v") else name: + raise version.InvalidVersion # noqa: TRY301 + except version.InvalidVersion: + pass + else: + return parsed, "Python" + try: + parsed = semver.Version.parse(name) + except ValueError: + pass + else: + return parsed, "SemVer" + return None, None + + def conform_versions(versions: list[dict]): versions_dict = {} for v in versions: - v["conformant"] = None - try: - version.Version(v["tag"]) - v["conformant"] = "Python" - except version.InvalidVersion: - pass - try: - semver.Version.parse(v["tag"]) - v["conformant"] = "SemVer (not Python)" - except ValueError: - pass + _, v["conformant"] = check_conformant(v["tag"]) if v["conformant"]: if v["tag"].startswith("V"): v["tag"][0] = "v" @@ -158,15 +168,7 @@ def __init__(self, release, *, prerelease_respect=False): [self.summarize_file(file) for file in release["files"]] def explode_versions(self): - v = None - try: - v = version.Version(self.tag) - except version.InvalidVersion: - pass - try: - v = semver.Version.parse(self.tag) - except ValueError: - pass + v, _ = check_conformant(self.tag) if not v: return None ret = { From 9811319c9fd515ff1c8b4dd6ae875c974717c2bb Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Fri, 25 Apr 2025 19:46:27 -0500 Subject: [PATCH 07/63] Mention non-comformant python in audit --- src/github_helper/api/_compare_versions.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/github_helper/api/_compare_versions.py b/src/github_helper/api/_compare_versions.py index 8ce9ab8..87dee40 100644 --- a/src/github_helper/api/_compare_versions.py +++ b/src/github_helper/api/_compare_versions.py @@ -64,7 +64,13 @@ def check_conformant(name): # we have to do this reverse check # because python is flexible/tolerant with bad versions if str(parsed) != name[1:] if name.startswith("v") else name: - raise version.InvalidVersion # noqa: TRY301 + old_parsed = parsed + try: + parsed = semver.Version.parse(name) + except ValueError: + return old_parsed, "Malformed Python" + else: + return parsed, "SemVer" except version.InvalidVersion: pass else: From 6e1e87292724e862d47eea25729cf1022b6c5ee4 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sat, 26 Apr 2025 15:14:37 -0500 Subject: [PATCH 08/63] Refactor and fix to print out basic version audit --- src/github_helper/api/__init__.py | 28 +++++++-- src/github_helper/api/_compare_versions.py | 68 ++++++++++++++++++++-- 2 files changed, 85 insertions(+), 11 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 6319635..151cbd8 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -322,7 +322,7 @@ async def get_project_configs(self, repo): sadness = int(not projects) return projects, sadness - async def get_remote_tags(self, repo): + async def get_remote_tags(self, repo, count=None, *, order_by_version=False): """Return tags ("tag":"name") for a repo.""" _ = await self.get_user() tags_jq = jq.compile("map({tag: .name})") @@ -333,7 +333,12 @@ async def get_remote_tags(self, repo): srv.check_retval(retval, err, endpoint=endpoint) tags = tags_jq.input_value(orjson.loads(out)).first() sadness = int(not tags) - return tags, sadness + if order_by_version: + tags = _compare_versions.order_versions( + _compare_versions.filter_versions(tags), + "tag", + ) + return tags[:count], sadness # probably need to handle specific projects # project metadata usually has github repo @@ -442,23 +447,28 @@ async def audit_releases(self, repo, count=7): sadness, ) - async def audit_versions(self, repo): + async def audit_versions(self, repo, count=15): """ Verify that version of a repository have differences. Args: repo: the name of the repo to verify. Can be "owner/repo" or just "repo" and owner is assumed to be the current user. + count: the number of versions to look at """ # puede mezclar proyectos acá # Ignoramos nombre de proyecto # todavia no probamos con mas de un projection en repositorio async with asyncio.TaskGroup() as tg: - tags_task = tg.create_task(self.get_remote_tags(repo)) + tags_task = tg.create_task( + self.get_remote_tags(repo, order_by_version=True), + ) releases_task = tg.create_task(self.audit_releases(repo)) pypi_task = tg.create_task(self.audit_pypi(repo)) - test_pypi_task = tg.create_task(self.audit_pypi(repo, testing=True)) + test_pypi_task = tg.create_task( + self.audit_pypi(repo, testing=True), + ) # que hacemos con sadness? tags, _ = await tags_task releases, _ = await releases_task @@ -512,9 +522,15 @@ def empty(x="Empty"): else yes() ), "tag valid": (_compare_versions.check_conformant(v)[1] or no()), + "incongruency": _compare_versions.compare_audits( + v, + releases=c_releases, + pypi=c_pypi, + test_pypi=c_test_pypi, + ), } for v in all_versions - ], + ][:count], "version", ) return result, 0 # maybe no sadness for audit? diff --git a/src/github_helper/api/_compare_versions.py b/src/github_helper/api/_compare_versions.py index 87dee40..cd572e5 100644 --- a/src/github_helper/api/_compare_versions.py +++ b/src/github_helper/api/_compare_versions.py @@ -149,17 +149,76 @@ def filter_versions(versions: list[dict]): } +def compare_audits(v, **audits): # noqa: C901 + all_tags = set() + for a in audits.values(): + audit = a.get(v, {}).get("audit", None) + if not audit: + continue + for project in audit.projects.values(): + all_tags.update(project.get("all_tags", {})) + results = {} + + # this inversion sucks + # what to do if project is missing + for tag in all_tags: + results[tag] = set() + for name, a in audits.items(): + audit = a.get(v, {}).get("audit", None) + if not audit: + results[tag].add(name) + continue + for project in audit.projects.values(): + if tag in project.get("all_tags", {}): + break + else: + results[tag].add(name) + if not results[tag]: + del results[tag] + output = "" + for tag, problems in results.items(): + output += f"{tag}: {', '.join(problems)}\n" + + return output + + class ReleaseAudit: + """Release audit turns a release object into a summary.""" + prerelease_agree: bool - projects: field(default_factory=dict) + """Does the version agree with the mark about prerelease.""" file_notes: field(default_factory=dict[str, dict]) + """A dict representing the first interpretation of any file.""" unknown_files: field(default_factory=set) + """Files that couldn't be understood trying to calculate notes.""" ignore_counter: field(default_factory=dict[str, int]) + projects: field(default_factory=dict) + """A list of the projects found in this release.""" + + def __repr__(self): + ignore_len = sum(self.ignore_counter.values()) + project_tag_count = [ + f"{k}: {len(v['all_tags'])}" for k, v in self.projects.items() + ] + + return ( + f"pre-agree: {self.prerelease_agree}; " + f"projects: {', '.join(project_tag_count)}; " + f"{len(self.file_notes)} files w/ notes; " + f"{len(self.unknown_files)} unknown files; " + f"{ignore_len} ignored files." + ) + def __init__(self, release, *, prerelease_respect=False): self.tag = release["tag"] self.version = self.explode_versions() + self.file_notes = {} + self.unknown_files = set() + self.ignore_counter = {} + self.projects = {} if not self.version: + self.prerelease_agree = None return self.prerelease_agree = ( (self.version["is_prerelease"] == release["prerelease"]) @@ -167,10 +226,6 @@ def __init__(self, release, *, prerelease_respect=False): else prerelease_respect ) - self.file_notes = {} - self.unknown_files = set() - self.ignore_counter = {} - self.projects = {} [self.summarize_file(file) for file in release["files"]] def explode_versions(self): @@ -258,6 +313,7 @@ def summarize_file(self, filename: str): self._build_python_project_summary(notes) case _: self.unknown_files.add(filename) + self.file_notes[filename] = notes return notes # the action you return will trigger behavior above @@ -305,12 +361,14 @@ def _build_python_project_summary(self, notes): # noqa: PLR0912, C901 "pure": [], "bdist-tree": None, "unknown-tags": [], + "all_tags": set(), } ref = self.projects[name] if notes.get("type") == "sdist": ref["sdist"] = True elif notes.get("type") == "bdist": ref["bdist"] = True + ref["all_tags"].update(notes.get("tags")) for t in notes.get("tags"): pair = f"{t.interpreter}-{t.abi}" # t.abi, t.interpreter, t.platform From a904958c5710eb3ce7c90447274524641fc37da4 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sat, 26 Apr 2025 15:20:41 -0500 Subject: [PATCH 09/63] Type and format colored --- src/github_helper/api/_compare_versions.py | 34 +++++++++------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/src/github_helper/api/_compare_versions.py b/src/github_helper/api/_compare_versions.py index cd572e5..c9ebc20 100644 --- a/src/github_helper/api/_compare_versions.py +++ b/src/github_helper/api/_compare_versions.py @@ -25,9 +25,9 @@ import sys from dataclasses import field -import colored import logistro import semver +from colored import Back, Fore, Style from packaging import utils, version _logger = logistro.getLogger(__name__) @@ -39,7 +39,7 @@ def __getattr__(self, name): return "" # Override colored's foreground, background, and style - colored.Fore = colored.Back = colored.Style = NoColor() + Fore = Back = Style = NoColor() # type: ignore[misc, assignment] # maybe combine these functions into a "normal versions" @@ -248,9 +248,9 @@ def explode_versions(self): def __str__(self): # noqa: C901, PLR0912 ret = "" if not self.prerelease_agree: - ret += f"{colored.Fore.red}PRERELEASE DISAGREEMENT{colored.Style.reset}\n" + ret += f"{Fore.red}PRERELEASE DISAGREEMENT{Style.reset}\n" if not self.projects: - ret += f"{colored.Fore.red}No Valid Projects Found.{colored.Style.reset}\n" + ret += f"{Fore.red}No Valid Projects Found.{Style.reset}\n" else: for k, v in self.projects.items(): if k.startswith("python/"): @@ -260,20 +260,15 @@ def __str__(self): # noqa: C901, PLR0912 warn = "bdist" if not v["sdist"]: warn = ", sdist" if warn else "sdist" - warn = ( - f", {colored.Fore.red}missing {warn}{colored.Style.reset}" - if warn - else "" - ) + warn = f", {Fore.red}missing {warn}{Style.reset}" if warn else "" pure = "" if v["pure"]: pure = ( - f", {colored.Fore.green}pure: {', '.join(v['pure'])}" - f"{colored.Style.reset}" + f", {Fore.green}pure: {', '.join(v['pure'])}{Style.reset}" ) - ret += f"{colored.Style.bold}{k}{colored.Style.reset}{warn}{pure}\n" + ret += f"{Style.bold}{k}{Style.reset}{warn}{pure}\n" ## look at tags ret += ( @@ -287,15 +282,15 @@ def __str__(self): # noqa: C901, PLR0912 ret += "\n ".join(v["unknown-tags"]) + "\n" if self.unknown_files: ret += ( - f"{colored.Fore.yellow}{colored.Style.bold}" + f"{Fore.yellow}{Style.bold}" f"{len(self.unknown_files)} Unknown Files:" - f"{colored.Style.reset}\n" + f"{Style.reset}\n" ) for i, f in enumerate(self.unknown_files): if i > 3: # noqa: PLR2004 - ret += f" {colored.Fore.yellow}...{colored.Style.reset}\n" + ret += f" {Fore.yellow}...{Style.reset}\n" break - ret += f" {colored.Fore.yellow}{f}{colored.Style.reset}\n" + ret += f" {Fore.yellow}{f}{Style.reset}\n" if self.ignore_counter: ret += "ignored: " for k, v in self.ignore_counter.items(): @@ -491,12 +486,9 @@ def _build_tree_str(self, obj, indent="", *, is_last=True): next_indent = indent + (" " if is_last else "| ") lines.append(f"{indent}{branch}{key}") if not value: - lines[-1] += f" {colored.Fore.red}missing{colored.Style.reset}" + lines[-1] += f" {Fore.red}missing{Style.reset}" elif isinstance(value, dict): lines.extend(self._build_tree_str(value, next_indent)) elif isinstance(value, (list, tuple)): - lines[-1] += ( - f" {colored.Fore.green}>> " - f"{', '.join(value)}{colored.Style.reset}" - ) + lines[-1] += f" {Fore.green}>> {', '.join(value)}{Style.reset}" return lines From 1bacaf8da6bdbad73fa5a8ffbcc7a76e75d3aef6 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sat, 26 Apr 2025 15:24:29 -0500 Subject: [PATCH 10/63] Fix typing --- src/github_helper/api/_audit.py | 2 +- src/github_helper/api/_compare_versions.py | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/github_helper/api/_audit.py b/src/github_helper/api/_audit.py index edbd86a..c1c580b 100644 --- a/src/github_helper/api/_audit.py +++ b/src/github_helper/api/_audit.py @@ -1,6 +1,6 @@ from pathlib import Path -import jsondiff as jd +import jsondiff as jd # type: ignore[import-untyped] from github_helper._services.gh import GHError from github_helper._utils import load_json diff --git a/src/github_helper/api/_compare_versions.py b/src/github_helper/api/_compare_versions.py index c9ebc20..32b4e5b 100644 --- a/src/github_helper/api/_compare_versions.py +++ b/src/github_helper/api/_compare_versions.py @@ -23,7 +23,6 @@ import copy import re import sys -from dataclasses import field import logistro import semver @@ -85,7 +84,7 @@ def check_conformant(name): def conform_versions(versions: list[dict]): - versions_dict = {} + versions_dict: dict = {} for v in versions: _, v["conformant"] = check_conformant(v["tag"]) if v["conformant"]: @@ -112,7 +111,7 @@ def filter_versions(versions: list[dict]): r"^" + version.VERSION_PATTERN + r"$", re.VERBOSE, ) - return [v for v in versions if _regex.match(v.get("tag"))] + return [v for v in versions if _regex.match(v.get("tag", ""))] bdist_template = { @@ -187,13 +186,13 @@ class ReleaseAudit: prerelease_agree: bool """Does the version agree with the mark about prerelease.""" - file_notes: field(default_factory=dict[str, dict]) + file_notes: dict[str, dict] """A dict representing the first interpretation of any file.""" - unknown_files: field(default_factory=set) + unknown_files: set """Files that couldn't be understood trying to calculate notes.""" - ignore_counter: field(default_factory=dict[str, int]) + ignore_counter: dict[str, int] - projects: field(default_factory=dict) + projects: dict """A list of the projects found in this release.""" def __repr__(self): From cde8d540cc728973d4063eb3e4d868f1e39d125f Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sat, 26 Apr 2025 16:27:11 -0500 Subject: [PATCH 11/63] Change module name. --- src/github_helper/api/__init__.py | 32 +++++++++---------- .../{_compare_versions.py => _versions.py} | 0 2 files changed, 16 insertions(+), 16 deletions(-) rename src/github_helper/api/{_compare_versions.py => _versions.py} (100%) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 151cbd8..af0b337 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -17,7 +17,7 @@ from github_helper._services import ssh_srv from github_helper._services.gh import GHError, ScopesError, ScopesWarning from github_helper._utils import load_json -from github_helper.api import _audit, _compare_versions +from github_helper.api import _audit, _versions _logger = logistro.getLogger(__name__) _SCRIPT_DIR = Path(__file__).resolve().parent @@ -334,8 +334,8 @@ async def get_remote_tags(self, repo, count=None, *, order_by_version=False): tags = tags_jq.input_value(orjson.loads(out)).first() sadness = int(not tags) if order_by_version: - tags = _compare_versions.order_versions( - _compare_versions.filter_versions(tags), + tags = _versions.order_versions( + _versions.filter_versions(tags), "tag", ) return tags[:count], sadness @@ -369,7 +369,7 @@ async def fetch_json(name): response = await session.get(url) pypi_json = await response.json() data = pypi_jq.input_value(pypi_json).first() - return _compare_versions.order_versions(data, "tag") + return _versions.order_versions(data, "tag") finally: await response.release() await session.close() @@ -394,7 +394,7 @@ async def audit_pypi(self, repo, count=7, *, testing=False): return None, sadness for release in releases: - release["audit"] = _compare_versions.ReleaseAudit( + release["audit"] = _versions.ReleaseAudit( release, prerelease_respect=True, ) @@ -402,7 +402,7 @@ async def audit_pypi(self, repo, count=7, *, testing=False): # I want count to be the API call or something # but it has to be ordered first. return ( - _compare_versions.order_versions(releases, "tag")[:count], + _versions.order_versions(releases, "tag")[:count], sadness, ) @@ -438,12 +438,12 @@ async def audit_releases(self, repo, count=7): return None, sadness for release in releases: - release["audit"] = _compare_versions.ReleaseAudit(release) + release["audit"] = _versions.ReleaseAudit(release) # I want count to be the API call or something # but it has to be ordered first. return ( - _compare_versions.order_versions(releases, "tag")[:count], + _versions.order_versions(releases, "tag")[:count], sadness, ) @@ -475,12 +475,12 @@ async def audit_versions(self, repo, count=15): pypi, _ = await pypi_task test_pypi, _ = await test_pypi_task - c_tags = _compare_versions.conform_versions( - _compare_versions.filter_versions(tags), + c_tags = _versions.conform_versions( + _versions.filter_versions(tags), ) - c_releases = _compare_versions.conform_versions(releases) - c_pypi = _compare_versions.conform_versions(pypi) - c_test_pypi = _compare_versions.conform_versions(test_pypi) + c_releases = _versions.conform_versions(releases) + c_pypi = _versions.conform_versions(pypi) + c_test_pypi = _versions.conform_versions(test_pypi) all_versions = ( c_tags.keys() | c_releases.keys() | c_pypi.keys() | c_test_pypi.keys() @@ -495,7 +495,7 @@ def no(x="False"): def empty(x="Empty"): return f"{colored.Fore.yellow}{x}{colored.Style.reset}" - result = _compare_versions.order_versions( + result = _versions.order_versions( [ { "version": v, @@ -521,8 +521,8 @@ def empty(x="Empty"): if c_test_pypi[v]["empty"] else yes() ), - "tag valid": (_compare_versions.check_conformant(v)[1] or no()), - "incongruency": _compare_versions.compare_audits( + "tag valid": (_versions.get_version_info(v)[1] or no()), + "incongruency": _versions.compare_audits( v, releases=c_releases, pypi=c_pypi, diff --git a/src/github_helper/api/_compare_versions.py b/src/github_helper/api/_versions.py similarity index 100% rename from src/github_helper/api/_compare_versions.py rename to src/github_helper/api/_versions.py From c91c8151877c5e9d08c707713ede67d3cfb32fa9 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sat, 26 Apr 2025 16:27:32 -0500 Subject: [PATCH 12/63] Rename some functions --- src/github_helper/api/_versions.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/github_helper/api/_versions.py b/src/github_helper/api/_versions.py index 32b4e5b..4eab877 100644 --- a/src/github_helper/api/_versions.py +++ b/src/github_helper/api/_versions.py @@ -41,6 +41,15 @@ def __getattr__(self, name): Fore = Back = Style = NoColor() # type: ignore[misc, assignment] +def filter_versions(versions: list[dict]): + # this needs to return a list of dictionaries + _regex = re.compile( + r"^" + version.VERSION_PATTERN + r"$", + re.VERBOSE, + ) + return [v for v in versions if _regex.match(v.get("tag", ""))] + + # maybe combine these functions into a "normal versions" # we talk a list of dictionaries, all have to have a "tag" key (version) # and what would we mark? @@ -57,7 +66,7 @@ def order_versions(versions: list[dict], key: str): ) -def check_conformant(name): +def get_version_info(name: str): try: parsed = version.Version(name) # we have to do this reverse check @@ -86,7 +95,7 @@ def check_conformant(name): def conform_versions(versions: list[dict]): versions_dict: dict = {} for v in versions: - _, v["conformant"] = check_conformant(v["tag"]) + _, v["conformant"] = get_version_info(v["tag"]) if v["conformant"]: if v["tag"].startswith("V"): v["tag"][0] = "v" @@ -105,15 +114,6 @@ def conform_versions(versions: list[dict]): return versions_dict -def filter_versions(versions: list[dict]): - # this needs to return a list of dictionaries - _regex = re.compile( - r"^" + version.VERSION_PATTERN + r"$", - re.VERBOSE, - ) - return [v for v in versions if _regex.match(v.get("tag", ""))] - - bdist_template = { "Windows": { "x86_64": [], @@ -228,7 +228,7 @@ def __init__(self, release, *, prerelease_respect=False): [self.summarize_file(file) for file in release["files"]] def explode_versions(self): - v, _ = check_conformant(self.tag) + v, _ = get_version_info(self.tag) if not v: return None ret = { From b2147b4f1050eef766daf77f86381aec902f8fe0 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sat, 26 Apr 2025 17:19:54 -0500 Subject: [PATCH 13/63] Refactor (and break) _versions.py --- src/github_helper/api/_versions.py | 135 ++++++++++++++++++----------- 1 file changed, 84 insertions(+), 51 deletions(-) diff --git a/src/github_helper/api/_versions.py b/src/github_helper/api/_versions.py index 4eab877..c7e540b 100644 --- a/src/github_helper/api/_versions.py +++ b/src/github_helper/api/_versions.py @@ -23,11 +23,13 @@ import copy import re import sys +from enum import StrEnum import logistro import semver from colored import Back, Fore, Style -from packaging import utils, version +from packaging import utils +from packaging import version as pyversion _logger = logistro.getLogger(__name__) @@ -44,58 +46,106 @@ def __getattr__(self, name): def filter_versions(versions: list[dict]): # this needs to return a list of dictionaries _regex = re.compile( - r"^" + version.VERSION_PATTERN + r"$", + r"^" + pyversion.VERSION_PATTERN + r"$", re.VERBOSE, ) return [v for v in versions if _regex.match(v.get("tag", ""))] -# maybe combine these functions into a "normal versions" -# we talk a list of dictionaries, all have to have a "tag" key (version) -# and what would we mark? -# a) we'd sort. -# b) we'd regulate how the tag is expressed (v or no) -# c) we'd determine if its empty or malformed def order_versions(versions: list[dict], key: str): if not versions: return versions return sorted( versions, - key=lambda x: version.parse(x[key]), + key=lambda x: pyversion.parse(x[key]), reverse=True, ) -def get_version_info(name: str): - try: - parsed = version.Version(name) - # we have to do this reverse check - # because python is flexible/tolerant with bad versions - if str(parsed) != name[1:] if name.startswith("v") else name: - old_parsed = parsed - try: - parsed = semver.Version.parse(name) - except ValueError: - return old_parsed, "Malformed Python" - else: - return parsed, "SemVer" - except version.InvalidVersion: - pass - else: - return parsed, "Python" - try: - parsed = semver.Version.parse(name) - except ValueError: - pass - else: - return parsed, "SemVer" - return None, None +_InputVersions = semver.Version | pyversion.Version + + +class Version: + """A unified version class.""" + + class Type(StrEnum): + PYTHON = "Python" + SEMVER = "SemVer" + MALFORMED = "Malformed Python" + + tag: str + valid: bool + type: Type + major: str + minor: str + patch: str + pre: str + dev: str + post: str + is_prerelease: bool + _parsed: _InputVersions + + def __init__(self, tag: str): + self.tag = tag + parsed_v, kind = self._test_parsers() + self.valid = bool(parsed_v) + if not self.valid or not kind: + return + self.type = kind + self._enumerate_version() + + def _test_parsers( + self, + ) -> ( + _InputVersions | None, + Type | None, + ): + """See which parsers handle the tag.""" + tag = self.tag + try: + parsed = pyversion.Version(tag) + + if str(parsed) != tag[1:] if tag.startswith("v") else tag: + old_parsed = parsed + try: + parsed = semver.Version.parse(tag) + except ValueError: + return old_parsed, Version.Status.MALFORMED + else: + return parsed, Version.Status.PYTHON + except pyversion.InvalidVersion: + pass + else: + return parsed, Version.Status.PYTHON + try: + parsed = semver.Version.parse(tag) + except ValueError: + pass + else: + return parsed, Version.Status.SemVer + return None, None + + def _enumerate_version(self, v: _InputVersions) -> None: + """Break tag attributes into unified attributes.""" + self._parsed = v + self.major = v.major + self.minor = v.minor + self.patch = v.patch if hasattr(v, "patch") else v.micro + self.pre = v.prerelease if hasattr(v, "prerelease") else v.pre + self.dev = v.dev if hasattr(v, "dev") else None + self.post = v.post if hasattr(v, "post") else None + self.is_prerelease = ( + v.is_prerelease if hasattr(v, "is_prerelease") else bool(v.pre) + ) + + #### HERE BE DRAGONS ##### def conform_versions(versions: list[dict]): versions_dict: dict = {} for v in versions: - _, v["conformant"] = get_version_info(v["tag"]) + temp = Version(v["tag"]) + v["conformant"] = temp.type if v["conformant"]: if v["tag"].startswith("V"): v["tag"][0] = "v" @@ -227,23 +277,6 @@ def __init__(self, release, *, prerelease_respect=False): [self.summarize_file(file) for file in release["files"]] - def explode_versions(self): - v, _ = get_version_info(self.tag) - if not v: - return None - ret = { - "major": v.major, - "minor": v.minor, - "patch": v.patch if hasattr(v, "patch") else v.micro, - "pre": v.prerelease if hasattr(v, "prerelease") else v.pre, - "dev": v.dev if hasattr(v, "dev") else None, - "post": v.post if hasattr(v, "post") else None, - } - ret["is_prerelease"] = ( - v.is_prerelease if hasattr(v, "is_prerelease") else bool(ret["pre"]) - ) - return ret - def __str__(self): # noqa: C901, PLR0912 ret = "" if not self.prerelease_agree: From de2a07f907a83ae2161cef2165280386373b72ff Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sat, 26 Apr 2025 17:20:57 -0500 Subject: [PATCH 14/63] Fix small syntax bug. --- src/github_helper/api/_versions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/github_helper/api/_versions.py b/src/github_helper/api/_versions.py index c7e540b..9974a4f 100644 --- a/src/github_helper/api/_versions.py +++ b/src/github_helper/api/_versions.py @@ -92,7 +92,7 @@ def __init__(self, tag: str): if not self.valid or not kind: return self.type = kind - self._enumerate_version() + self._enumerate_version(parsed_v) def _test_parsers( self, @@ -138,7 +138,8 @@ def _enumerate_version(self, v: _InputVersions) -> None: v.is_prerelease if hasattr(v, "is_prerelease") else bool(v.pre) ) - #### HERE BE DRAGONS ##### + +#### HERE BE DRAGONS ##### def conform_versions(versions: list[dict]): From f627eb051a1d842677706bcbd9f6f564eda0e454 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sat, 26 Apr 2025 17:31:36 -0500 Subject: [PATCH 15/63] Finish type checking. --- src/github_helper/api/_versions.py | 44 ++++++++++++++++-------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/src/github_helper/api/_versions.py b/src/github_helper/api/_versions.py index 9974a4f..4cb6f52 100644 --- a/src/github_helper/api/_versions.py +++ b/src/github_helper/api/_versions.py @@ -62,7 +62,7 @@ def order_versions(versions: list[dict], key: str): ) -_InputVersions = semver.Version | pyversion.Version +_VersionTypes = semver.Version | pyversion.Version class Version: @@ -76,66 +76,68 @@ class Type(StrEnum): tag: str valid: bool type: Type - major: str - minor: str - patch: str + major: int + minor: int + patch: int pre: str dev: str post: str is_prerelease: bool - _parsed: _InputVersions + _parsed: _VersionTypes def __init__(self, tag: str): self.tag = tag parsed_v, kind = self._test_parsers() - self.valid = bool(parsed_v) - if not self.valid or not kind: + if not parsed_v or not kind: + self.valid = False return + self.valid = True self.type = kind self._enumerate_version(parsed_v) def _test_parsers( self, - ) -> ( - _InputVersions | None, - Type | None, - ): + ) -> tuple[_VersionTypes | None, Type | None]: """See which parsers handle the tag.""" tag = self.tag try: - parsed = pyversion.Version(tag) + parsed: _VersionTypes = pyversion.Version(tag) if str(parsed) != tag[1:] if tag.startswith("v") else tag: old_parsed = parsed try: parsed = semver.Version.parse(tag) except ValueError: - return old_parsed, Version.Status.MALFORMED + return old_parsed, Version.Type.MALFORMED else: - return parsed, Version.Status.PYTHON + return parsed, Version.Type.PYTHON except pyversion.InvalidVersion: pass else: - return parsed, Version.Status.PYTHON + return parsed, Version.Type.PYTHON try: parsed = semver.Version.parse(tag) except ValueError: pass else: - return parsed, Version.Status.SemVer + return parsed, Version.Type.SEMVER return None, None - def _enumerate_version(self, v: _InputVersions) -> None: + def _enumerate_version(self, v: _VersionTypes) -> None: """Break tag attributes into unified attributes.""" self._parsed = v self.major = v.major self.minor = v.minor self.patch = v.patch if hasattr(v, "patch") else v.micro - self.pre = v.prerelease if hasattr(v, "prerelease") else v.pre - self.dev = v.dev if hasattr(v, "dev") else None - self.post = v.post if hasattr(v, "post") else None + self.pre = str( + v.prerelease + if hasattr(v, "prerelease") + else (f"{v.pre[0]}{v.pre[1]}" if v.pre else ""), + ) + self.dev = str(v.dev) if hasattr(v, "dev") else "" + self.post = str(v.post) if hasattr(v, "post") else "" self.is_prerelease = ( - v.is_prerelease if hasattr(v, "is_prerelease") else bool(v.pre) + v.is_prerelease if hasattr(v, "is_prerelease") else bool(v.prerelease) ) From 0e71f9ecee06a3bc778a8f1f8561785d0fe6dfec Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sat, 26 Apr 2025 17:54:24 -0500 Subject: [PATCH 16/63] Improve version class. --- src/github_helper/api/_versions.py | 84 ++++++++++++++++++++++++------ 1 file changed, 68 insertions(+), 16 deletions(-) diff --git a/src/github_helper/api/_versions.py b/src/github_helper/api/_versions.py index 4cb6f52..bcc557a 100644 --- a/src/github_helper/api/_versions.py +++ b/src/github_helper/api/_versions.py @@ -23,7 +23,9 @@ import copy import re import sys +from dataclasses import dataclass from enum import StrEnum +from functools import total_ordering import logistro import semver @@ -65,6 +67,8 @@ def order_versions(versions: list[dict], key: str): _VersionTypes = semver.Version | pyversion.Version +@total_ordering +@dataclass(frozen=True, slots=True) class Version: """A unified version class.""" @@ -86,13 +90,13 @@ class Type(StrEnum): _parsed: _VersionTypes def __init__(self, tag: str): - self.tag = tag + object.__setattr__(self, "tag", tag) parsed_v, kind = self._test_parsers() if not parsed_v or not kind: - self.valid = False + object.__setattr__(self, "valid", False) return - self.valid = True - self.type = kind + object.__setattr__(self, "valid", True) + object.__setattr__(self, "type", kind) self._enumerate_version(parsed_v) def _test_parsers( @@ -125,20 +129,68 @@ def _test_parsers( def _enumerate_version(self, v: _VersionTypes) -> None: """Break tag attributes into unified attributes.""" - self._parsed = v - self.major = v.major - self.minor = v.minor - self.patch = v.patch if hasattr(v, "patch") else v.micro - self.pre = str( - v.prerelease - if hasattr(v, "prerelease") - else (f"{v.pre[0]}{v.pre[1]}" if v.pre else ""), + object.__setattr__(self, "_parsed", v) + object.__setattr__(self, "major", v.major) + object.__setattr__(self, "minor", v.minor) + object.__setattr__( + self, + "patch", + v.patch if hasattr(v, "patch") else v.micro, ) - self.dev = str(v.dev) if hasattr(v, "dev") else "" - self.post = str(v.post) if hasattr(v, "post") else "" - self.is_prerelease = ( - v.is_prerelease if hasattr(v, "is_prerelease") else bool(v.prerelease) + object.__setattr__( + self, + "pre", + str( + v.prerelease + if hasattr(v, "prerelease") + else (f"{v.pre[0]}{v.pre[1]}" if v.pre else ""), + ), ) + object.__setattr__( + self, + "dev", + str(v.dev) if hasattr(v, "dev") else "", + ) + object.__setattr__( + self, + "post", + str(v.post) if hasattr(v, "post") else "", + ) + object.__setattr__( + self, + "is_prerelease", + (v.is_prerelease if hasattr(v, "is_prerelease") else bool(v.prerelease)), + ) + + def __str__(self): + return str(self._parsed) + + def __repr__(self): + return str(self._parsed) + + def _cmp_tuple(self) -> tuple: + return ( + self.major, + self.minor, + self.patch, + self.pre, + self.dev, + self.post, + ) + + def __eq__(self, other: object) -> bool: + if isinstance(other, _VersionTypes): + other = Version(str(other)) + elif not isinstance(other, Version): + raise NotImplementedError + return self._cmp_tuple() == other._cmp_tuple() + + def __lt__(self, other: object) -> bool: + if isinstance(other, _VersionTypes): + other = Version(str(other)) + elif not isinstance(other, Version): + raise NotImplementedError + return self._cmp_tuple() < other._cmp_tuple() #### HERE BE DRAGONS ##### From b871ea6c032e19136f7b8c8a2dcc1e492e9ddaf6 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sat, 26 Apr 2025 19:00:05 -0500 Subject: [PATCH 17/63] Add a bad-version parser --- src/github_helper/api/_versions.py | 66 ++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/src/github_helper/api/_versions.py b/src/github_helper/api/_versions.py index bcc557a..6b83793 100644 --- a/src/github_helper/api/_versions.py +++ b/src/github_helper/api/_versions.py @@ -64,25 +64,73 @@ def order_versions(versions: list[dict], key: str): ) -_VersionTypes = semver.Version | pyversion.Version +_tag_part_re = re.compile(r"^(\d*)(?:\.(.*))?$") + + +@dataclass(frozen=True, slots=True) +class BadVersion: + """A weak parse that looks for instances where someone tried to tag.""" + + major: int + minor: int + patch: int # semver name + prerelease = "" # semver name + + def __init__(self, tag: str): + """Look for tag-like structures that parsers won't return.""" + object.__setattr__(self, "tag", tag) + object.__setattr__(self, "major", 0) + object.__setattr__(self, "minor", 0) + object.__setattr__(self, "patch", 0) + if not tag.startswith(("v", "V")): + raise ValueError + else: + major_match = _tag_part_re.search(tag[1:]) + if not major_match: + raise ValueError + # else + object.__setattr__(self, "major", int(major_match.group(1))) + if not major_match.group(2): + return + # else + minor_match = _tag_part_re.search(major_match.group(2)) + if not minor_match: + object.__setattr__(self, "prerelease", major_match.group(2)) + return + # else + object.__setattr__(self, "minor", int(minor_match.group(1))) + if not minor_match.group(2): + return + # else + patch_match = _tag_part_re.search(minor_match.group(2)) + if not patch_match: + object.__setattr__(self, "prerelease", minor_match.group(2)) + return + # else + object.__setattr__(self, "patch", patch_match.group(1)) + object.__setattr__(self, "prerelease", patch_match.group(2) or "") + + +_VersionTypes = semver.Version | pyversion.Version | BadVersion @total_ordering @dataclass(frozen=True, slots=True) class Version: - """A unified version class.""" + """A unified version class. Most similar to Python, not SemVer.""" class Type(StrEnum): PYTHON = "Python" SEMVER = "SemVer" MALFORMED = "Malformed Python" + UNPARSABLE = "Unparsable" tag: str valid: bool type: Type major: int minor: int - patch: int + micro: int pre: str dev: str post: str @@ -134,16 +182,16 @@ def _enumerate_version(self, v: _VersionTypes) -> None: object.__setattr__(self, "minor", v.minor) object.__setattr__( self, - "patch", - v.patch if hasattr(v, "patch") else v.micro, + "micro", + v.micro if hasattr(v, "micro") else v.patch, ) object.__setattr__( self, "pre", str( - v.prerelease - if hasattr(v, "prerelease") - else (f"{v.pre[0]}{v.pre[1]}" if v.pre else ""), + (f"{v.pre[0]}{v.pre[1]}" if v.pre else "") + if hasattr(v, "pre") + else v.prerelease, ), ) object.__setattr__( @@ -172,7 +220,7 @@ def _cmp_tuple(self) -> tuple: return ( self.major, self.minor, - self.patch, + self.micro, self.pre, self.dev, self.post, From 4052bc26b377cd759729d2a03b6584d3a8eeb619 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sat, 26 Apr 2025 19:01:55 -0500 Subject: [PATCH 18/63] Make _versions.py public --- src/github_helper/api/__init__.py | 32 +++++++++---------- .../api/{_versions.py => versions.py} | 0 2 files changed, 16 insertions(+), 16 deletions(-) rename src/github_helper/api/{_versions.py => versions.py} (100%) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index af0b337..08fa677 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -17,7 +17,7 @@ from github_helper._services import ssh_srv from github_helper._services.gh import GHError, ScopesError, ScopesWarning from github_helper._utils import load_json -from github_helper.api import _audit, _versions +from github_helper.api import _audit, versions _logger = logistro.getLogger(__name__) _SCRIPT_DIR = Path(__file__).resolve().parent @@ -334,8 +334,8 @@ async def get_remote_tags(self, repo, count=None, *, order_by_version=False): tags = tags_jq.input_value(orjson.loads(out)).first() sadness = int(not tags) if order_by_version: - tags = _versions.order_versions( - _versions.filter_versions(tags), + tags = versions.order_versions( + versions.filter_versions(tags), "tag", ) return tags[:count], sadness @@ -369,7 +369,7 @@ async def fetch_json(name): response = await session.get(url) pypi_json = await response.json() data = pypi_jq.input_value(pypi_json).first() - return _versions.order_versions(data, "tag") + return versions.order_versions(data, "tag") finally: await response.release() await session.close() @@ -394,7 +394,7 @@ async def audit_pypi(self, repo, count=7, *, testing=False): return None, sadness for release in releases: - release["audit"] = _versions.ReleaseAudit( + release["audit"] = versions.ReleaseAudit( release, prerelease_respect=True, ) @@ -402,7 +402,7 @@ async def audit_pypi(self, repo, count=7, *, testing=False): # I want count to be the API call or something # but it has to be ordered first. return ( - _versions.order_versions(releases, "tag")[:count], + versions.order_versions(releases, "tag")[:count], sadness, ) @@ -438,12 +438,12 @@ async def audit_releases(self, repo, count=7): return None, sadness for release in releases: - release["audit"] = _versions.ReleaseAudit(release) + release["audit"] = versions.ReleaseAudit(release) # I want count to be the API call or something # but it has to be ordered first. return ( - _versions.order_versions(releases, "tag")[:count], + versions.order_versions(releases, "tag")[:count], sadness, ) @@ -475,12 +475,12 @@ async def audit_versions(self, repo, count=15): pypi, _ = await pypi_task test_pypi, _ = await test_pypi_task - c_tags = _versions.conform_versions( - _versions.filter_versions(tags), + c_tags = versions.conform_versions( + versions.filter_versions(tags), ) - c_releases = _versions.conform_versions(releases) - c_pypi = _versions.conform_versions(pypi) - c_test_pypi = _versions.conform_versions(test_pypi) + c_releases = versions.conform_versions(releases) + c_pypi = versions.conform_versions(pypi) + c_test_pypi = versions.conform_versions(test_pypi) all_versions = ( c_tags.keys() | c_releases.keys() | c_pypi.keys() | c_test_pypi.keys() @@ -495,7 +495,7 @@ def no(x="False"): def empty(x="Empty"): return f"{colored.Fore.yellow}{x}{colored.Style.reset}" - result = _versions.order_versions( + result = versions.order_versions( [ { "version": v, @@ -521,8 +521,8 @@ def empty(x="Empty"): if c_test_pypi[v]["empty"] else yes() ), - "tag valid": (_versions.get_version_info(v)[1] or no()), - "incongruency": _versions.compare_audits( + "tag valid": (versions.get_version_info(v)[1] or no()), + "incongruency": versions.compare_audits( v, releases=c_releases, pypi=c_pypi, diff --git a/src/github_helper/api/_versions.py b/src/github_helper/api/versions.py similarity index 100% rename from src/github_helper/api/_versions.py rename to src/github_helper/api/versions.py From fa983cc25cf831620a07abc23206f8a8fa04d6d2 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sat, 26 Apr 2025 19:11:56 -0500 Subject: [PATCH 19/63] Format --- src/github_helper/api/versions.py | 53 +++++++++++++++++-------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/src/github_helper/api/versions.py b/src/github_helper/api/versions.py index 6b83793..0c67937 100644 --- a/src/github_helper/api/versions.py +++ b/src/github_helper/api/versions.py @@ -45,6 +45,29 @@ def __getattr__(self, name): Fore = Back = Style = NoColor() # type: ignore[misc, assignment] +def conform_versions(versions: list[dict]): + versions_dict: dict = {} + for v in versions: + temp = Version(v["tag"]) + v["conformant"] = temp.type + if v["conformant"]: + if v["tag"].startswith("V"): + v["tag"][0] = "v" + elif not v["tag"].startswith("v"): + v["tag"] = f"v{v['tag']}" + # always true for tag, use audit + _logger.debug2(f"Files in {v['tag']}: {len(v.get('files', []))}") + v["empty"] = not bool(v.get("files")) + _logger.debug2(f"Empty {v['empty']} from {v.get('files')}") + if v["tag"] in versions_dict: + cur = versions_dict[v["tag"]] + cur["empty"] = v["empty"] + cur["files"].extend(v["files"]) + else: + versions_dict[v["tag"]] = v + return versions_dict + + def filter_versions(versions: list[dict]): # this needs to return a list of dictionaries _regex = re.compile( @@ -244,29 +267,6 @@ def __lt__(self, other: object) -> bool: #### HERE BE DRAGONS ##### -def conform_versions(versions: list[dict]): - versions_dict: dict = {} - for v in versions: - temp = Version(v["tag"]) - v["conformant"] = temp.type - if v["conformant"]: - if v["tag"].startswith("V"): - v["tag"][0] = "v" - elif not v["tag"].startswith("v"): - v["tag"] = f"v{v['tag']}" - # always true for tag, use audit - _logger.debug2(f"Files in {v['tag']}: {len(v.get('files', []))}") - v["empty"] = not bool(v.get("files")) - _logger.debug2(f"Empty {v['empty']} from {v.get('files')}") - if v["tag"] in versions_dict: - cur = versions_dict[v["tag"]] - cur["empty"] = v["empty"] - cur["files"].extend(v["files"]) - else: - versions_dict[v["tag"]] = v - return versions_dict - - bdist_template = { "Windows": { "x86_64": [], @@ -464,7 +464,12 @@ def _get_file_notes(self, filename: str): } elif filename.endswith(".whl"): try: - name, version, build, compat_tags = utils.parse_wheel_filename(filename) + ( + name, + version, + build, + compat_tags, + ) = utils.parse_wheel_filename(filename) except utils.InvalidWheelFilename: _logger.debug(f"Invalid Wheel Filename: {filename}") return {"error": "invalid name", "value": filename} From 7a64eb415ce327167ec07c5f6bc5824a9fa8683b Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sun, 27 Apr 2025 10:09:52 -0500 Subject: [PATCH 20/63] Type and comment --- src/github_helper/api/__init__.py | 41 +++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 08fa677..c5d3bf2 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -5,6 +5,7 @@ import tomllib import warnings from pathlib import Path +from typing import TypedDict, TypeVar import aiohttp import colored @@ -51,6 +52,11 @@ def _log_one_json(obj): _logger.debug2(f"gh result:\n {raw!s}") +_T = TypeVar("_T") +RetVal = tuple[_T, int] +"""Return type for api's that can exit the program.""" + + class GHApi: """Provides access to status functions ontop of gh program.""" @@ -286,6 +292,7 @@ async def query_version(repo): sadness = int(not repos) return repos, sadness + # this should take a name TODO (or several) async def get_project_configs(self, repo): """Find all projects in a repo.""" owner, repo = self._split_full_name(full_name=repo) @@ -322,7 +329,12 @@ async def get_project_configs(self, repo): sadness = int(not projects) return projects, sadness - async def get_remote_tags(self, repo, count=None, *, order_by_version=False): + class Tags(TypedDict): + """Return type for get_remote_tags.""" # noqa: D204 blank line ugly + + tag: str + + async def get_remote_tags(self, repo, count=None) -> RetVal[list[Tags]]: """Return tags ("tag":"name") for a repo.""" _ = await self.get_user() tags_jq = jq.compile("map({tag: .name})") @@ -333,16 +345,22 @@ async def get_remote_tags(self, repo, count=None, *, order_by_version=False): srv.check_retval(retval, err, endpoint=endpoint) tags = tags_jq.input_value(orjson.loads(out)).first() sadness = int(not tags) - if order_by_version: - tags = versions.order_versions( - versions.filter_versions(tags), - "tag", - ) return tags[:count], sadness - # probably need to handle specific projects - # project metadata usually has github repo - async def get_pypi(self, repo, *, testing=False, flatten=False): + class Release(TypedDict): + """Release object containing version and files.""" # noqa: D204 ugly + + tag: str + files: list[str] + + # TODO: take project name, not repo + async def get_pypi( + self, + repo: str, + *, + testing: bool = False, + flatten: bool = False, + ) -> RetVal[dict[str, list[Release]] | list[Release]]: """Get all pypi releases for ALL projects in a repo.""" project_configs, sadness = await self.get_project_configs(repo) project_names = set() @@ -380,7 +398,10 @@ async def fetch_json(name): sadness = int(not releases) if flatten: # two objects may have same tag - releases = [release for project in releases.values() for release in project] + return ( + [release for project in releases.values() for release in project], + sadness, + ) return releases, sadness async def audit_pypi(self, repo, count=7, *, testing=False): From aea3295faa7d58f78754e2ccc0245a684d0a80b1 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sun, 27 Apr 2025 10:13:33 -0500 Subject: [PATCH 21/63] Remove old call --- src/github_helper/api/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index c5d3bf2..70dd2b5 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -330,9 +330,10 @@ async def get_project_configs(self, repo): return projects, sadness class Tags(TypedDict): - """Return type for get_remote_tags.""" # noqa: D204 blank line ugly + """Return type for get_remote_tags.""" tag: str + # end async def get_remote_tags(self, repo, count=None) -> RetVal[list[Tags]]: """Return tags ("tag":"name") for a repo.""" @@ -348,10 +349,11 @@ async def get_remote_tags(self, repo, count=None) -> RetVal[list[Tags]]: return tags[:count], sadness class Release(TypedDict): - """Release object containing version and files.""" # noqa: D204 ugly + """Release object containing version and files.""" tag: str files: list[str] + # end # TODO: take project name, not repo async def get_pypi( @@ -483,7 +485,7 @@ async def audit_versions(self, repo, count=15): # todavia no probamos con mas de un projection en repositorio async with asyncio.TaskGroup() as tg: tags_task = tg.create_task( - self.get_remote_tags(repo, order_by_version=True), + self.get_remote_tags(repo), ) releases_task = tg.create_task(self.audit_releases(repo)) pypi_task = tg.create_task(self.audit_pypi(repo)) From 77670eb60c290866ff6f6ccc3cccb3a5ac321845 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sun, 27 Apr 2025 10:14:26 -0500 Subject: [PATCH 22/63] Change to use new Version, not explode_versions --- src/github_helper/api/versions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/github_helper/api/versions.py b/src/github_helper/api/versions.py index 0c67937..fe94010 100644 --- a/src/github_helper/api/versions.py +++ b/src/github_helper/api/versions.py @@ -364,11 +364,12 @@ def __repr__(self): def __init__(self, release, *, prerelease_respect=False): self.tag = release["tag"] - self.version = self.explode_versions() self.file_notes = {} self.unknown_files = set() self.ignore_counter = {} self.projects = {} + + self.version = Version(self.tag) if not self.version: self.prerelease_agree = None return From 58032093c31c386b7c092c2886808c053d5a6c61 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sun, 27 Apr 2025 19:45:51 -0500 Subject: [PATCH 23/63] Mid Major Audit Refactor: See Commit: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (english below) Esta refactorización es tan grande que no tiene sentido mirar los diffs de los commits. Mejor mirar a los archivos nuevos, después del ultimo commit de refactorización. La idea es: 1. Unas funciones de laAPI de audit fueron quitadas, para hacer todo más sencillo. 2. Las funciones restantes están ya más sencillas porque: a) Son menos flexibles, no lidian con toda situación, solo la especificada. b) Se usa `@dataclass` para devoluciones (más en "Charla"). 3. Con un mejor entendimiento de la organización requisito, una jerarquia more clara ha estado establecida. Pero, dada que estamos estancado en un solo archivo (`__init__`) en una sola clase, `GHApi` esa jerarchia encaja una clase en otras, todo en un archivo. Escapar el archivo sería buevo (v2?) pero se necesita más cambios. Charla: jq es muy expresivo en la transformación de datos pero también devuelve dicts sin tipo. Pasando tales dicts entre capas se molesta porque queda difícil limitar y entender el estado del objeto en cualquier monento. `TypedDict` tiene soluciones pero también desventajas. 1. No se puede usar `isinstance()` 2. No se puede tener métodos. La ventaja es que se puede traver con `items()` cuando se necesita todos los atributos/claves. `@dataclass` es mejor dado que no tiene las desventajas dichas. También `fields()`, parecido a`keys()`. `@dataclass` también tiene `asdict(...)`, pero se copia. Se usa con `tabulate()`, de forma automatica. Pero, al aceso es diferente. `.clave` en vez de `[clave]`. Los diccionarios tienen unas ventajas pero todavía necesitan más herramientas, talvez glom. ENGLISH This refactor is so big it doesn't make sense to look at the diffs of the commits. It makes sense to simply to look at the new files, after the end-of- refactor commit. The basic idea is here: 1. Audit functions we're removed to simplify. 2. Existing functions were made more simple by a) being less flexible (not looking for all possible situations, but whatever is specified). b) using `@dataclass` for types returned (see discussion). 3. With a better understanding of the organization required, a more clear hierarchy has been established. However, since we're locked into a single file, in a single class, that hierarchy involves classes embedded in classes, everything in one file. Breaking out of one file and splitting the API into multiple files (as described for version two), would be good, but would require a restructuring. Discussion: jq is very expressive in transforming data but it also returns typeless dicts. Passing typeless dicts between layers is annoying because it's difficult to constrain and understand the state of any single return. `TypedDict` presented a solution but it has some problems: 1. it can't be used in `isinstance()` 2. it can't have methods Its advantages is that it can be traversed with `items()` in the case that an operation needs to be performed on every members. `@dataclass` is a little bit better because it doesn't have the above two drawbacks. It also has `fields()` which is like `keys()`. `@dataclass` also has `asdict(...)` but that creates a copy. --- src/github_helper/_adapters/api_to_cli.py | 11 +- src/github_helper/api/__init__.py | 261 ++++++++++------------ src/github_helper/api/versions.py | 258 +++++++++++++-------- 3 files changed, 293 insertions(+), 237 deletions(-) diff --git a/src/github_helper/_adapters/api_to_cli.py b/src/github_helper/_adapters/api_to_cli.py index 4fc4db9..fac6848 100644 --- a/src/github_helper/_adapters/api_to_cli.py +++ b/src/github_helper/_adapters/api_to_cli.py @@ -123,8 +123,8 @@ async def transform_releases_data(self, releases_data): return to_table.format_table( [ [ - release["tag"], - delim.join(release["files"]), + release.tag, + delim.join(release.files), ] for release in releases_data ], @@ -139,11 +139,10 @@ async def transform_pypi_data(self, releases_data): return to_table.format_table( [ [ - f"{name}-{release['tag']}", - delim.join(release["files"]), + f"{release.tag}", + delim.join(release.files), ] - for name, subobject in releases_data.items() - for release in subobject + for release in releases_data ], pretty=self._pretty, headers=("version", "files"), diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 70dd2b5..39b51b7 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -1,17 +1,19 @@ """A CLI dashboard for github status.""" import asyncio +import itertools import re import tomllib import warnings +from dataclasses import dataclass from pathlib import Path -from typing import TypedDict, TypeVar +from typing import TypeVar import aiohttp -import colored import jq # type: ignore [import-not-found] import logistro import orjson +from colored import Fore, Style from github_helper._services import gh as srv from github_helper._services import repos as repo_srv @@ -293,11 +295,16 @@ async def query_version(repo): return repos, sadness # this should take a name TODO (or several) - async def get_project_configs(self, repo): + async def get_project_configs( + self, + repo: str, + *name: str, + ref: str | None = None, + ): """Find all projects in a repo.""" owner, repo = self._split_full_name(full_name=repo) folder_repos = repo_srv.RepoFolder() - projects = {} + projects: dict = {} _logger.debug(f"Downloading repo {owner}/{repo}") r = await folder_repos.add_repo( @@ -329,13 +336,20 @@ async def get_project_configs(self, repo): sadness = int(not projects) return projects, sadness - class Tags(TypedDict): + @dataclass(slots=True, kw_only=True) + class Tag: """Return type for get_remote_tags.""" tag: str + _with_v: bool = False + + def tag_eq(self, cannon): + """Check if tag is equal to another tag.""" + return self.tag == (f"v{cannon}" if self._with_v else cannon) + # end - async def get_remote_tags(self, repo, count=None) -> RetVal[list[Tags]]: + async def get_remote_tags(self, repo, count=None) -> RetVal[list[Tag]]: """Return tags ("tag":"name") for a repo.""" _ = await self.get_user() tags_jq = jq.compile("map({tag: .name})") @@ -344,16 +358,21 @@ async def get_remote_tags(self, repo, count=None) -> RetVal[list[Tags]]: _logger.debug(f"Calling API: {endpoint}") retval, out, err = await srv.gh_api(endpoint) srv.check_retval(retval, err, endpoint=endpoint) - tags = tags_jq.input_value(orjson.loads(out)).first() + tags_dict = tags_jq.input_value(orjson.loads(out)).first() + tags = [GHApi.Tag(**tag, _with_v=True) for tag in tags_dict] sadness = int(not tags) return tags[:count], sadness - class Release(TypedDict): + @dataclass(slots=True, kw_only=True) + class Release(Tag): """Release object containing version and files.""" - tag: str + prerelease: bool + """Does the source mark it as prerelease?""" files: list[str] - # end + """Files that came with it.""" + version: versions.Version + """Calculated version.""" # TODO: take project name, not repo async def get_pypi( @@ -361,8 +380,7 @@ async def get_pypi( repo: str, *, testing: bool = False, - flatten: bool = False, - ) -> RetVal[dict[str, list[Release]] | list[Release]]: + ) -> RetVal[list[Release]]: """Get all pypi releases for ALL projects in a repo.""" project_configs, sadness = await self.get_project_configs(repo) project_names = set() @@ -380,7 +398,7 @@ async def fetch_json(name): jq_dir = ( r".releases // {} | " r"to_entries | map(" - r'{tag: "v\(.key)", files:' + r"{tag: .key, files:" r"[ .value[] | select(.yanked != true) | .filename ]" r"})" ) @@ -388,48 +406,27 @@ async def fetch_json(name): session = aiohttp.ClientSession() response = await session.get(url) pypi_json = await response.json() - data = pypi_jq.input_value(pypi_json).first() - return versions.order_versions(data, "tag") + return pypi_jq.input_value(pypi_json).first() finally: await response.release() await session.close() - releases = {} + releases: dict = {} for name in project_names: releases[name] = await fetch_json(name) sadness = int(not releases) - if flatten: - # two objects may have same tag - return ( - [release for project in releases.values() for release in project], - sadness, - ) - return releases, sadness - - async def audit_pypi(self, repo, count=7, *, testing=False): - """Get all pypi releases for a project and audit it.""" - releases, sadness = await self.get_pypi( - repo, - testing=testing, - flatten=True, - ) - if sadness: - return None, sadness - - for release in releases: - release["audit"] = versions.ReleaseAudit( - release, - prerelease_respect=True, + flat_releases: list[GHApi.Release] = [ + GHApi.Release( + **r, + version=(v := versions.Version(r["tag"])), + prerelease=v.is_prerelease, ) + for project in releases.values() + for r in project + ] + return flat_releases, sadness - # I want count to be the API call or something - # but it has to be ordered first. - return ( - versions.order_versions(releases, "tag")[:count], - sadness, - ) - - async def get_releases(self, repo): + async def get_releases(self, repo: str) -> RetVal[list[Release]]: """Return releases for a repo.""" _ = await self.get_user() releases_jq = jq.compile( @@ -452,25 +449,21 @@ async def get_releases(self, repo): _log_one_json(obj) releases = releases_jq.input_value(obj).first() sadness = int(not releases) - return releases, sadness - - async def audit_releases(self, repo, count=7): - """Run get_releases and process information.""" - releases, sadness = await self.get_releases(repo) - if sadness: - return None, sadness - - for release in releases: - release["audit"] = versions.ReleaseAudit(release) - - # I want count to be the API call or something - # but it has to be ordered first. - return ( - versions.order_versions(releases, "tag")[:count], - sadness, - ) + coerced_releases: list[GHApi.Release] = [ + GHApi.Release( + **r, + version=versions.Version(r["tag"]), + _with_v=True, + ) + for r in releases + ] + return coerced_releases, sadness - async def audit_versions(self, repo, count=15): + async def audit_versions( + self, + repo: str, + count: int = 15, + ) -> RetVal[list[dict]]: """ Verify that version of a repository have differences. @@ -480,82 +473,76 @@ async def audit_versions(self, repo, count=15): count: the number of versions to look at """ - # puede mezclar proyectos acá - # Ignoramos nombre de proyecto - # todavia no probamos con mas de un projection en repositorio - async with asyncio.TaskGroup() as tg: - tags_task = tg.create_task( - self.get_remote_tags(repo), - ) - releases_task = tg.create_task(self.audit_releases(repo)) - pypi_task = tg.create_task(self.audit_pypi(repo)) - test_pypi_task = tg.create_task( - self.audit_pypi(repo, testing=True), - ) - # que hacemos con sadness? - tags, _ = await tags_task - releases, _ = await releases_task - pypi, _ = await pypi_task - test_pypi, _ = await test_pypi_task - - c_tags = versions.conform_versions( - versions.filter_versions(tags), - ) - c_releases = versions.conform_versions(releases) - c_pypi = versions.conform_versions(pypi) - c_test_pypi = versions.conform_versions(test_pypi) - all_versions = ( - c_tags.keys() | c_releases.keys() | c_pypi.keys() | c_test_pypi.keys() + # should this be a dictionary so we can iterate through names? + @dataclass(slots=True) + class VersionSet: + gh_tags: GHApi.Tag | None = None + gh_releases: GHApi.Release | None = None + pypi: GHApi.Release | None = None + test_pypi: GHApi.Release | None = None + + # maybe audits should carry their own adapters + # this is an adapter + def print_source_status(self, name: str, canonical: str) -> str: + attr = getattr(self, name) + if not attr: + return "" + if not isinstance(attr, GHApi.Release) or attr.files: + empty = "" + else: + empty = f"{Fore.red}Yanked/Empty{Style.reset}" + if not attr.tag_eq(canonical): + return f"{empty}{Fore.yellow}({attr.tag}){Style.reset}" + elif empty: + return f"{empty}" + else: + return f"{Fore.green}True{Style.reset}" + + ( + (tags, _), + (release, _), + (pypi, _), + (test_pypi, _), + ) = await asyncio.gather( + self.get_remote_tags(repo), + self.get_releases(repo), + self.get_pypi(repo), + self.get_pypi(repo, testing=True), ) - def yes(x="True"): - return f"{colored.Fore.green}{x}{colored.Style.reset}" - - def no(x="False"): - return f"{colored.Fore.red}{x}{colored.Style.reset}" - - def empty(x="Empty"): - return f"{colored.Fore.yellow}{x}{colored.Style.reset}" - - result = versions.order_versions( - [ - { - "version": v, - "tags": (no() if v not in c_tags else yes()), - "releases": ( - no() - if v not in c_releases - else empty(empty) - if c_releases[v]["empty"] - else yes() - ), - "pypi": ( - no() - if v not in c_pypi - else empty("yanked") - if c_pypi[v]["empty"] - else yes() - ), - "test.pypi": ( - no() - if v not in c_test_pypi - else empty("yanked") - if c_test_pypi[v]["empty"] - else yes() - ), - "tag valid": (versions.get_version_info(v)[1] or no()), - "incongruency": versions.compare_audits( - v, - releases=c_releases, - pypi=c_pypi, - test_pypi=c_test_pypi, - ), - } - for v in all_versions - ][:count], - "version", - ) + all_versions: dict[ + versions.Version, + VersionSet, + ] = {} + + for attrname, o in itertools.chain( + (("gh_tags", t) for t in tags), + (("gh_releases", r) for r in release), + (("pypi", r) for r in pypi), + (("test_pypi", r) for r in test_pypi), + ): + v = getattr(o, "version", None) or versions.Version(o.tag) + # TODO: this is where we have to check for doubles + if not v.valid: + continue + if v not in all_versions: + all_versions[v] = VersionSet() + setattr(all_versions[v], attrname, o) + + all_versions = dict(sorted(all_versions.items(), reverse=True)) + + result = [ + { + "version": str(v), + "gh_tags": r.print_source_status("gh_tags", str(v)), + "gh_releases": r.print_source_status("gh_releases", str(v)), + "pypi": r.print_source_status("pypi", str(v)), + "test.pypi": r.print_source_status("test_pypi", str(v)), + "validity": (v.kind), + } + for v, r in list(all_versions.items())[:count] # count + ] return result, 0 # maybe no sadness for audit? async def _get_ruleset(self, owner, repo, ruleset_id): diff --git a/src/github_helper/api/versions.py b/src/github_helper/api/versions.py index fe94010..806a6f7 100644 --- a/src/github_helper/api/versions.py +++ b/src/github_helper/api/versions.py @@ -23,13 +23,14 @@ import copy import re import sys +import warnings from dataclasses import dataclass from enum import StrEnum from functools import total_ordering import logistro import semver -from colored import Back, Fore, Style +from colored import Fore, Style from packaging import utils from packaging import version as pyversion @@ -37,67 +38,24 @@ if not sys.stdout.isatty(): - class NoColor: + class _NoColor: def __getattr__(self, name): return "" # Override colored's foreground, background, and style - Fore = Back = Style = NoColor() # type: ignore[misc, assignment] - - -def conform_versions(versions: list[dict]): - versions_dict: dict = {} - for v in versions: - temp = Version(v["tag"]) - v["conformant"] = temp.type - if v["conformant"]: - if v["tag"].startswith("V"): - v["tag"][0] = "v" - elif not v["tag"].startswith("v"): - v["tag"] = f"v{v['tag']}" - # always true for tag, use audit - _logger.debug2(f"Files in {v['tag']}: {len(v.get('files', []))}") - v["empty"] = not bool(v.get("files")) - _logger.debug2(f"Empty {v['empty']} from {v.get('files')}") - if v["tag"] in versions_dict: - cur = versions_dict[v["tag"]] - cur["empty"] = v["empty"] - cur["files"].extend(v["files"]) - else: - versions_dict[v["tag"]] = v - return versions_dict - - -def filter_versions(versions: list[dict]): - # this needs to return a list of dictionaries - _regex = re.compile( - r"^" + pyversion.VERSION_PATTERN + r"$", - re.VERBOSE, - ) - return [v for v in versions if _regex.match(v.get("tag", ""))] - - -def order_versions(versions: list[dict], key: str): - if not versions: - return versions - return sorted( - versions, - key=lambda x: pyversion.parse(x[key]), - reverse=True, - ) - - -_tag_part_re = re.compile(r"^(\d*)(?:\.(.*))?$") + Fore = Style = _NoColor() # type: ignore[misc, assignment] @dataclass(frozen=True, slots=True) class BadVersion: - """A weak parse that looks for instances where someone tried to tag.""" + """A weak parser that looks for instances where someone tried to tag.""" + _tag_part_re = re.compile(r"^(\d*)(?:\.(.*))?$") major: int minor: int patch: int # semver name - prerelease = "" # semver name + prerelease: None = None + build: str = "" # semver name def __init__(self, tag: str): """Look for tag-like structures that parsers won't return.""" @@ -105,10 +63,10 @@ def __init__(self, tag: str): object.__setattr__(self, "major", 0) object.__setattr__(self, "minor", 0) object.__setattr__(self, "patch", 0) - if not tag.startswith(("v", "V")): - raise ValueError + if tag.startswith(("v", "V")): + tag = tag[1:] else: - major_match = _tag_part_re.search(tag[1:]) + major_match = self._tag_part_re.search(tag[1:]) if not major_match: raise ValueError # else @@ -116,22 +74,22 @@ def __init__(self, tag: str): if not major_match.group(2): return # else - minor_match = _tag_part_re.search(major_match.group(2)) + minor_match = self._tag_part_re.search(major_match.group(2)) if not minor_match: - object.__setattr__(self, "prerelease", major_match.group(2)) + object.__setattr__(self, "build", major_match.group(2)) return # else object.__setattr__(self, "minor", int(minor_match.group(1))) if not minor_match.group(2): return # else - patch_match = _tag_part_re.search(minor_match.group(2)) + patch_match = self._tag_part_re.search(minor_match.group(2)) if not patch_match: - object.__setattr__(self, "prerelease", minor_match.group(2)) + object.__setattr__(self, "build", minor_match.group(2)) return # else object.__setattr__(self, "patch", patch_match.group(1)) - object.__setattr__(self, "prerelease", patch_match.group(2) or "") + object.__setattr__(self, "build", patch_match.group(2) or None) _VersionTypes = semver.Version | pyversion.Version | BadVersion @@ -143,35 +101,44 @@ class Version: """A unified version class. Most similar to Python, not SemVer.""" class Type(StrEnum): + """The possible types of version formats.""" + PYTHON = "Python" SEMVER = "SemVer" - MALFORMED = "Malformed Python" + MALFORMED_PY = "Malformed Python" + MALFORMED = "Malformed" UNPARSABLE = "Unparsable" tag: str valid: bool - type: Type + kind: Type major: int minor: int micro: int - pre: str - dev: str - post: str + pre: tuple[str, int] | None + dev: int | None + post: int | None + local: str | None is_prerelease: bool _parsed: _VersionTypes - def __init__(self, tag: str): + def __init__(self, tag: str, *, raise_parse_exc: bool = False): + """Convert a tag into a Version object, collecting info.""" + if not isinstance(tag, str): + raise TypeError("tag argument must be string.") object.__setattr__(self, "tag", tag) - parsed_v, kind = self._test_parsers() - if not parsed_v or not kind: + parsed_v, kind = self._test_parsers(rpe=raise_parse_exc) + object.__setattr__(self, "kind", kind) + if not parsed_v: object.__setattr__(self, "valid", False) return object.__setattr__(self, "valid", True) - object.__setattr__(self, "type", kind) self._enumerate_version(parsed_v) def _test_parsers( self, + *, + rpe: bool = False, ) -> tuple[_VersionTypes | None, Type | None]: """See which parsers handle the tag.""" tag = self.tag @@ -183,23 +150,29 @@ def _test_parsers( try: parsed = semver.Version.parse(tag) except ValueError: - return old_parsed, Version.Type.MALFORMED + return old_parsed, Version.Type.MALFORMED_PY else: return parsed, Version.Type.PYTHON - except pyversion.InvalidVersion: - pass + except Exception: + if rpe: + raise else: return parsed, Version.Type.PYTHON try: parsed = semver.Version.parse(tag) - except ValueError: - pass + except Exception: + if rpe: + raise else: return parsed, Version.Type.SEMVER - return None, None + return None, Version.Type.UNPARSABLE def _enumerate_version(self, v: _VersionTypes) -> None: """Break tag attributes into unified attributes.""" + if hasattr(v, "epoch") and v.epoch: + raise NotImplementedError( + "Not working with versions with epochs, but would be easy tbh.", + ) object.__setattr__(self, "_parsed", v) object.__setattr__(self, "major", v.major) object.__setattr__(self, "minor", v.minor) @@ -208,24 +181,42 @@ def _enumerate_version(self, v: _VersionTypes) -> None: "micro", v.micro if hasattr(v, "micro") else v.patch, ) + if hasattr(v, "prerelease"): + if not v.prerelease: + pre = None + else: + match = re.match(r"([a-zA-Z]+)(?:[.\-]?(\d+))?", v.prerelease) + if not match: + warnings.warn( + "Prerelease string parsed but is not valid", + stacklevel=1, + ) + pre = (v.prerelease, 0) + else: + label = match.group(1).lower() + number = int(match.group(2)) if match.group(2) else 0 + pre = (label, number) + else: + pre = v.pre object.__setattr__( self, "pre", - str( - (f"{v.pre[0]}{v.pre[1]}" if v.pre else "") - if hasattr(v, "pre") - else v.prerelease, - ), + pre, ) object.__setattr__( self, "dev", - str(v.dev) if hasattr(v, "dev") else "", + v.dev if hasattr(v, "dev") else None, ) object.__setattr__( self, "post", - str(v.post) if hasattr(v, "post") else "", + v.post if hasattr(v, "post") else None, + ) + object.__setattr__( + self, + "local", + v.local if hasattr(v, "local") else v.build, ) object.__setattr__( self, @@ -234,34 +225,108 @@ def _enumerate_version(self, v: _VersionTypes) -> None: ) def __str__(self): - return str(self._parsed) + """Print Version as string.""" + return self.__repr__() def __repr__(self): - return str(self._parsed) + """Print Version as python-compatible string if possible.""" + if not self.valid: + return "" + post = f".post{self.post}" if self.post else "" + dev = f".dev{self.dev}" if self.dev else "" + pre = f"{self.pre[0]}{self.pre[1]}" if self.pre else "" + local = f"+{self.local}" if self.local else "" + return f"{self.major}.{self.minor}.{self.micro}{pre}{post}{dev}{local}" def _cmp_tuple(self) -> tuple: + if not self.valid: + return (float("NaN"),) return ( self.major, self.minor, self.micro, self.pre, - self.dev, self.post, + self.dev, + self.local, ) - def __eq__(self, other: object) -> bool: - if isinstance(other, _VersionTypes): - other = Version(str(other)) - elif not isinstance(other, Version): - raise NotImplementedError - return self._cmp_tuple() == other._cmp_tuple() - - def __lt__(self, other: object) -> bool: + def _compare( # noqa: C901, PLR0911, PLR0912 + self, + other, + ) -> int: # intrinsically special NotImplemented accepted as return + """ + Compare two versions with major, minor, micro, pre, post, dev. + + Returns: + -1 if self < other + 0 if self == other + 1 if self > other + NotImplemented if wrong type + + """ + if not self.valid: + return NotImplemented if isinstance(other, _VersionTypes): other = Version(str(other)) elif not isinstance(other, Version): - raise NotImplementedError - return self._cmp_tuple() < other._cmp_tuple() + return NotImplemented + # Compare major, minor, micro + if (c := (self.major - other.major)) != 0: + return (c > 0) - (c < 0) + if (c := (self.minor - other.minor)) != 0: + return (c > 0) - (c < 0) + if (c := (self.micro - other.micro)) != 0: + return (c > 0) - (c < 0) + + # Compare pre-releases + if self.pre is None and other.pre is not None: + return 1 # final > pre + if self.pre is not None and other.pre is None: + return -1 # pre < final + if self.pre and other.pre: + if self.pre[0] != other.pre[0]: + return (self.pre[0] > other.pre[0]) - (self.pre[0] < other.pre[0]) + if self.pre[1] != other.pre[1]: + return (self.pre[1] > other.pre[1]) - (self.pre[1] < other.pre[1]) + + # Compare post-releases (before dev!) + if self.post is None and other.post is not None: + return -1 # no post < post + if self.post is not None and other.post is None: + return 1 # post > no post + if self.post and other.post: # noqa: SIM102 clarity + if self.post != other.post: + return (self.post > other.post) - (self.post < other.post) + + # Compare dev-releases (only if same pre/post/final) + if self.dev is None and other.dev is not None: + return 1 # no dev > dev + if self.dev is not None and other.dev is None: + return -1 # dev < no dev + if self.dev and other.dev: # noqa: SIM102 clarity + if self.dev != other.dev: + return (self.dev > other.dev) - (self.dev < other.dev) + + return 0 + + def __eq__(self, other) -> bool: + """Check if equal.""" + c = self._compare(other) + if c is NotImplemented: + return NotImplemented + return c == 0 + + def __lt__(self, other) -> bool: + """Check if less than.""" + c = self._compare(other) + if c is NotImplemented: + return NotImplemented + return c < 0 + + def __hash__(self): + """Return hash based on normalized value.""" + return self._cmp_tuple().__hash__() #### HERE BE DRAGONS ##### @@ -302,6 +367,7 @@ def __lt__(self, other: object) -> bool: def compare_audits(v, **audits): # noqa: C901 + """Produce a diff between several audits.""" all_tags = set() for a in audits.values(): audit = a.get(v, {}).get("audit", None) @@ -349,6 +415,7 @@ class ReleaseAudit: """A list of the projects found in this release.""" def __repr__(self): + """Return a summary of the audit.""" ignore_len = sum(self.ignore_counter.values()) project_tag_count = [ f"{k}: {len(v['all_tags'])}" for k, v in self.projects.items() @@ -363,6 +430,7 @@ def __repr__(self): ) def __init__(self, release, *, prerelease_respect=False): + """Audit a release object.""" self.tag = release["tag"] self.file_notes = {} self.unknown_files = set() @@ -374,7 +442,7 @@ def __init__(self, release, *, prerelease_respect=False): self.prerelease_agree = None return self.prerelease_agree = ( - (self.version["is_prerelease"] == release["prerelease"]) + (self.version.is_prerelease == release["prerelease"]) if "prerelease" in release else prerelease_respect ) @@ -382,6 +450,7 @@ def __init__(self, release, *, prerelease_respect=False): [self.summarize_file(file) for file in release["files"]] def __str__(self): # noqa: C901, PLR0912 + """Return a string diff of a python project.""" ret = "" if not self.prerelease_agree: ret += f"{Fore.red}PRERELEASE DISAGREEMENT{Style.reset}\n" @@ -435,6 +504,7 @@ def __str__(self): # noqa: C901, PLR0912 return ret def summarize_file(self, filename: str): + """Analyze filename and dispatch analysis for further processing.""" notes = self._get_file_notes(filename) match notes.get("action"): case "ignore": From 3d96c764c9fe4c99d4e5636d6532e78172d64b89 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sun, 27 Apr 2025 20:31:10 -0500 Subject: [PATCH 24/63] Force get_projects_config to need filename --- src/github_helper/api/__init__.py | 68 ++++++++++++++++++------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 39b51b7..3b0b0e8 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -7,7 +7,7 @@ import warnings from dataclasses import dataclass from pathlib import Path -from typing import TypeVar +from typing import Any, TypedDict, TypeVar import aiohttp import jq # type: ignore [import-not-found] @@ -295,13 +295,23 @@ async def query_version(repo): return repos, sadness # this should take a name TODO (or several) + class ConfigDescription(TypedDict): + """Description of a config.""" + + object: Any + _original: str + + ConfigSet = dict[str, ConfigDescription] + async def get_project_configs( self, repo: str, - *name: str, + *filenames: str, ref: str | None = None, - ): + ) -> RetVal[ConfigSet]: """Find all projects in a repo.""" + if not filenames: + raise ValueError("A least one filename must be supplied.") owner, repo = self._split_full_name(full_name=repo) folder_repos = repo_srv.RepoFolder() projects: dict = {} @@ -312,29 +322,25 @@ async def get_project_configs( repo, url="ssh://git@github.com", ) - ref = "main" if "main" in await r.list_branches() else "master" - py_files = await r.get_files_by_name("pyproject.toml", ref=ref) - js_files = await r.get_files_by_name("package.json", ref=ref) - if py_files: - projects["py"] = {} - for f in py_files: - _logger.debug2(f"Found py: {f['path']}") - projects["py"][f["path"]] = { - "object": tomllib.loads(f["content"].decode()), - "_original": f["content"].decode(), - } - if js_files: - projects["js"] = {} - for f in js_files: - _logger.debug2(f"Found js: {f['path']}") + if not ref: + ref = "main" if "main" in await r.list_branches() else "master" + files = [] + for name in filenames: + files.extend(await r.get_files_by_name(name, ref=ref)) + configs: GHApi.ConfigSet = {} + for f in files: + obj = None + if f["path"].endswith(".json"): obj = orjson.loads(f["content"]) - projects["js"][f["path"]] = { - "object": obj, - "_original": f["content"].decode(), - } + elif f["path"].endswith(".toml"): + obj = tomllib.loads(f["content"].decode()) + configs[f["path"]] = { + "object": obj, + "_original": f["content"].decode(), + } sadness = int(not projects) - return projects, sadness + return configs, sadness @dataclass(slots=True, kw_only=True) class Tag: @@ -382,13 +388,17 @@ async def get_pypi( testing: bool = False, ) -> RetVal[list[Release]]: """Get all pypi releases for ALL projects in a repo.""" - project_configs, sadness = await self.get_project_configs(repo) + project_configs, sadness = await self.get_project_configs( + repo, + "pyproject.toml", + ) project_names = set() - if "py" in project_configs: - for config in project_configs["py"].values(): - name = config.get("object", {}).get("project", {}).get("name", {}) - if name: - project_names.add(name) + for path, config in project_configs.items(): + if not path.endswith("pyproject.toml"): + continue + name = config.get("object", {}).get("project", {}).get("name", {}) + if name: + project_names.add(name) prefix = "test." if testing else "" async def fetch_json(name): From ef9c0d59330738f233a8c61b0729918f628b5c81 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sun, 27 Apr 2025 21:10:30 -0500 Subject: [PATCH 25/63] Change get_pypi to take project name --- src/github_helper/api/__init__.py | 90 ++++++++++++++++--------------- 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 3b0b0e8..7e0900f 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -380,61 +380,44 @@ class Release(Tag): version: versions.Version """Calculated version.""" - # TODO: take project name, not repo async def get_pypi( self, - repo: str, + project_name: str, *, testing: bool = False, ) -> RetVal[list[Release]]: """Get all pypi releases for ALL projects in a repo.""" - project_configs, sadness = await self.get_project_configs( - repo, - "pyproject.toml", - ) - project_names = set() - for path, config in project_configs.items(): - if not path.endswith("pyproject.toml"): - continue - name = config.get("object", {}).get("project", {}).get("name", {}) - if name: - project_names.add(name) prefix = "test." if testing else "" - async def fetch_json(name): - url = f"https://{prefix}pypi.org/pypi/{name}/json" - _logger.debug(url) - try: - jq_dir = ( - r".releases // {} | " - r"to_entries | map(" - r"{tag: .key, files:" - r"[ .value[] | select(.yanked != true) | .filename ]" - r"})" - ) - pypi_jq = jq.compile(jq_dir) - session = aiohttp.ClientSession() - response = await session.get(url) - pypi_json = await response.json() - return pypi_jq.input_value(pypi_json).first() - finally: - await response.release() - await session.close() - - releases: dict = {} - for name in project_names: - releases[name] = await fetch_json(name) + url = f"https://{prefix}pypi.org/pypi/{project_name}/json" + _logger.debug(url) + try: + jq_dir = ( + r".releases // {} | " + r"to_entries | map(" + r"{tag: .key, files:" + r"[ .value[] | select(.yanked != true) | .filename ]" + r"})" + ) + pypi_jq = jq.compile(jq_dir) + session = aiohttp.ClientSession() + response = await session.get(url) + pypi_json = await response.json() + releases = pypi_jq.input_value(pypi_json).first() + finally: + await response.release() + await session.close() + sadness = int(not releases) - flat_releases: list[GHApi.Release] = [ + coerced_releases: list[GHApi.Release] = [ GHApi.Release( **r, version=(v := versions.Version(r["tag"])), prerelease=v.is_prerelease, ) - for project in releases.values() - for r in project + for r in releases ] - return flat_releases, sadness + return coerced_releases, sadness async def get_releases(self, repo: str) -> RetVal[list[Release]]: """Return releases for a repo.""" @@ -469,7 +452,7 @@ async def get_releases(self, repo: str) -> RetVal[list[Release]]: ] return coerced_releases, sadness - async def audit_versions( + async def audit_versions( # noqa: C901 self, repo: str, count: int = 15, @@ -509,6 +492,21 @@ def print_source_status(self, name: str, canonical: str) -> str: else: return f"{Fore.green}True{Style.reset}" + project_configs, sadness = await self.get_project_configs( + repo, + "pyproject.toml", + ) + + project_names = [] + for path, config in project_configs.items(): + if not path.endswith("pyproject.toml"): + continue + name = config.get("object", {}).get("project", {}).get("name", {}) + if name: + project_names.append(name) + if len(project_names) > 1: + raise NotImplementedError("Repo has more than one project :-(") + ( (tags, _), (release, _), @@ -517,8 +515,8 @@ def print_source_status(self, name: str, canonical: str) -> str: ) = await asyncio.gather( self.get_remote_tags(repo), self.get_releases(repo), - self.get_pypi(repo), - self.get_pypi(repo, testing=True), + self.get_pypi(project_names[0]), + self.get_pypi(project_names[0], testing=True), ) all_versions: dict[ @@ -533,11 +531,15 @@ def print_source_status(self, name: str, canonical: str) -> str: (("test_pypi", r) for r in test_pypi), ): v = getattr(o, "version", None) or versions.Version(o.tag) - # TODO: this is where we have to check for doubles if not v.valid: continue if v not in all_versions: all_versions[v] = VersionSet() + if getattr(all_versions[v], attrname, None): + warnings.warn( + "Looks like conflicting poorly-written versions caused overwrite.", + stacklevel=2, + ) setattr(all_versions[v], attrname, o) all_versions = dict(sorted(all_versions.items(), reverse=True)) From 46053112d19954626c241749ff529aef0aa35fd8 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sun, 27 Apr 2025 21:15:45 -0500 Subject: [PATCH 26/63] Detect Not Found in get_pypi --- src/github_helper/api/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 7e0900f..fae8be2 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -403,6 +403,8 @@ async def get_pypi( session = aiohttp.ClientSession() response = await session.get(url) pypi_json = await response.json() + if pypi_json.get("message", None) == "Not Found": + return [], 1 releases = pypi_jq.input_value(pypi_json).first() finally: await response.release() From cc6f2a77732616f14e46756372abd87b78d2b7f9 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sun, 27 Apr 2025 21:21:13 -0500 Subject: [PATCH 27/63] Skip columns if get_pypi() returns sadness>0 --- src/github_helper/api/__init__.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index fae8be2..c7288f8 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -512,8 +512,8 @@ def print_source_status(self, name: str, canonical: str) -> str: ( (tags, _), (release, _), - (pypi, _), - (test_pypi, _), + (pypi, pypi_sadness), + (test_pypi, test_pypi_sadness), ) = await asyncio.gather( self.get_remote_tags(repo), self.get_releases(repo), @@ -531,6 +531,7 @@ def print_source_status(self, name: str, canonical: str) -> str: (("gh_releases", r) for r in release), (("pypi", r) for r in pypi), (("test_pypi", r) for r in test_pypi), + [], ): v = getattr(o, "version", None) or versions.Version(o.tag) if not v.valid: @@ -551,8 +552,16 @@ def print_source_status(self, name: str, canonical: str) -> str: "version": str(v), "gh_tags": r.print_source_status("gh_tags", str(v)), "gh_releases": r.print_source_status("gh_releases", str(v)), - "pypi": r.print_source_status("pypi", str(v)), - "test.pypi": r.print_source_status("test_pypi", str(v)), + **( + {"pypi": r.print_source_status("pypi", str(v))} + if not pypi_sadness + else {} + ), + **( + {"test.pypi": r.print_source_status("test_pypi", str(v))} + if not test_pypi_sadness + else {} + ), "validity": (v.kind), } for v, r in list(all_versions.items())[:count] # count From 54f7656fef44fb9d47135c7d60881505d1ab47c7 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sun, 27 Apr 2025 22:33:47 -0500 Subject: [PATCH 28/63] Add more color silencing --- src/github_helper/api/__init__.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index c7288f8..4149010 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -3,6 +3,7 @@ import asyncio import itertools import re +import sys import tomllib import warnings from dataclasses import dataclass @@ -26,6 +27,16 @@ _SCRIPT_DIR = Path(__file__).resolve().parent _TEMPLATE_PATH = _SCRIPT_DIR / "templates" +# could just put this in CLI and be done with it +# could also force +if not sys.stdout.isatty(): + + class _NoColor: + def __getattr__(self, name): + return "" + + # Override colored's foreground, background, and style + Fore = Style = _NoColor() # type: ignore[misc, assignment] _check_ran = False @@ -478,7 +489,7 @@ class VersionSet: test_pypi: GHApi.Release | None = None # maybe audits should carry their own adapters - # this is an adapter + # this is an adapter, colors is an adapter def print_source_status(self, name: str, canonical: str) -> str: attr = getattr(self, name) if not attr: @@ -544,7 +555,6 @@ def print_source_status(self, name: str, canonical: str) -> str: stacklevel=2, ) setattr(all_versions[v], attrname, o) - all_versions = dict(sorted(all_versions.items(), reverse=True)) result = [ From 88406766c175a7d149b12a4c5767859589932654 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sun, 27 Apr 2025 22:33:55 -0500 Subject: [PATCH 29/63] Make bad verisons work. --- src/github_helper/api/__init__.py | 2 +- src/github_helper/api/versions.py | 94 +++++++++++++++++++------------ 2 files changed, 60 insertions(+), 36 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 4149010..cc23a5e 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -572,7 +572,7 @@ def print_source_status(self, name: str, canonical: str) -> str: if not test_pypi_sadness else {} ), - "validity": (v.kind), + "validity": v.kind, } for v, r in list(all_versions.items())[:count] # count ] diff --git a/src/github_helper/api/versions.py b/src/github_helper/api/versions.py index 806a6f7..b4e1251 100644 --- a/src/github_helper/api/versions.py +++ b/src/github_helper/api/versions.py @@ -50,46 +50,49 @@ def __getattr__(self, name): class BadVersion: """A weak parser that looks for instances where someone tried to tag.""" - _tag_part_re = re.compile(r"^(\d*)(?:\.(.*))?$") + _tag_part_re = re.compile(r"^(\d*)(?:\.?(.*))?$") + tag: str major: int minor: int - patch: int # semver name - prerelease: None = None - build: str = "" # semver name + micro: int + pre: None = None + local: str | None = None + is_prerelease: bool = False def __init__(self, tag: str): """Look for tag-like structures that parsers won't return.""" object.__setattr__(self, "tag", tag) object.__setattr__(self, "major", 0) object.__setattr__(self, "minor", 0) - object.__setattr__(self, "patch", 0) - if tag.startswith(("v", "V")): - tag = tag[1:] - else: - major_match = self._tag_part_re.search(tag[1:]) - if not major_match: - raise ValueError - # else - object.__setattr__(self, "major", int(major_match.group(1))) - if not major_match.group(2): - return - # else - minor_match = self._tag_part_re.search(major_match.group(2)) - if not minor_match: - object.__setattr__(self, "build", major_match.group(2)) - return - # else - object.__setattr__(self, "minor", int(minor_match.group(1))) - if not minor_match.group(2): - return - # else - patch_match = self._tag_part_re.search(minor_match.group(2)) - if not patch_match: - object.__setattr__(self, "build", minor_match.group(2)) - return - # else - object.__setattr__(self, "patch", patch_match.group(1)) - object.__setattr__(self, "build", patch_match.group(2) or None) + object.__setattr__(self, "micro", 0) + object.__setattr__(self, "pre", None) + object.__setattr__(self, "local", None) + object.__setattr__(self, "is_prerelease", False) + tag = tag.removeprefix("v") + major_match = self._tag_part_re.search(tag) + if not major_match or not major_match.group(1): + raise ValueError + # else + object.__setattr__(self, "major", int(major_match.group(1))) + if not major_match.group(2): + return + # else + minor_match = self._tag_part_re.search(major_match.group(2)) + if not minor_match or not minor_match.group(1): + object.__setattr__(self, "local", major_match.group(2)) + return + # else + object.__setattr__(self, "minor", int(minor_match.group(1))) + if not minor_match.group(2): + return + # else + micro_match = self._tag_part_re.search(minor_match.group(2)) + if not micro_match or not micro_match.group(1): + object.__setattr__(self, "local", minor_match.group(2)) + return + # else + object.__setattr__(self, "micro", int(micro_match.group(1))) + object.__setattr__(self, "local", micro_match.group(2) or None) _VersionTypes = semver.Version | pyversion.Version | BadVersion @@ -135,7 +138,7 @@ def __init__(self, tag: str, *, raise_parse_exc: bool = False): object.__setattr__(self, "valid", True) self._enumerate_version(parsed_v) - def _test_parsers( + def _test_parsers( # noqa: C901 self, *, rpe: bool = False, @@ -145,7 +148,7 @@ def _test_parsers( try: parsed: _VersionTypes = pyversion.Version(tag) - if str(parsed) != tag[1:] if tag.startswith("v") else tag: + if str(parsed) != tag.removeprefix("v"): old_parsed = parsed try: parsed = semver.Version.parse(tag) @@ -165,6 +168,12 @@ def _test_parsers( raise else: return parsed, Version.Type.SEMVER + try: + parsed = BadVersion(tag) + except ValueError: + pass + else: + return parsed, Version.Type.MALFORMED return None, Version.Type.UNPARSABLE def _enumerate_version(self, v: _VersionTypes) -> None: @@ -251,6 +260,9 @@ def _cmp_tuple(self) -> tuple: self.local, ) + # this could def be simplified to tuple comparison + # but some values would need to be normalized first + # thx ai def _compare( # noqa: C901, PLR0911, PLR0912 self, other, @@ -308,7 +320,19 @@ def _compare( # noqa: C901, PLR0911, PLR0912 if self.dev != other.dev: return (self.dev > other.dev) - (self.dev < other.dev) - return 0 + # local is not used in official sorting + # but if we do not provide some arbitrary tie breaking + # python will sometimes think two versions are equal + # in some instances (like when physical sorting) + # and disequal in others (like when looking for keys) + # so functions like sorted() which do both will break + # so just sort arbitrarily unless they _really_ _are_ _equal_ + if self.local == other.local: + return 0 + elif self.local is not None and other.local is None: + return 1 + else: + return -1 def __eq__(self, other) -> bool: """Check if equal.""" From de41bcb4ff83d0edbf0dff9fd4964322491ee492 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sun, 27 Apr 2025 23:32:33 -0500 Subject: [PATCH 30/63] Add color and indication to bad versions --- src/github_helper/api/__init__.py | 2 +- src/github_helper/api/versions.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index cc23a5e..ef47d9f 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -468,7 +468,7 @@ async def get_releases(self, repo: str) -> RetVal[list[Release]]: async def audit_versions( # noqa: C901 self, repo: str, - count: int = 15, + count: int = 10, ) -> RetVal[list[dict]]: """ Verify that version of a repository have differences. diff --git a/src/github_helper/api/versions.py b/src/github_helper/api/versions.py index b4e1251..8dcf39a 100644 --- a/src/github_helper/api/versions.py +++ b/src/github_helper/api/versions.py @@ -94,6 +94,10 @@ def __init__(self, tag: str): object.__setattr__(self, "micro", int(micro_match.group(1))) object.__setattr__(self, "local", micro_match.group(2) or None) + def __str__(self): + """Return string rep of broken version.""" + return "BROKEN" + _VersionTypes = semver.Version | pyversion.Version | BadVersion @@ -109,8 +113,8 @@ class Type(StrEnum): PYTHON = "Python" SEMVER = "SemVer" MALFORMED_PY = "Malformed Python" - MALFORMED = "Malformed" - UNPARSABLE = "Unparsable" + MALFORMED = f"{Fore.red}Malformed{Style.reset}" + UNPARSABLE = f"{Fore.red}Unparsable{Style.reset}" tag: str valid: bool @@ -241,6 +245,8 @@ def __repr__(self): """Print Version as python-compatible string if possible.""" if not self.valid: return "" + if self.kind == Version.Type.MALFORMED: + return f"{Fore.red}BROKEN{Style.reset}" post = f".post{self.post}" if self.post else "" dev = f".dev{self.dev}" if self.dev else "" pre = f"{self.pre[0]}{self.pre[1]}" if self.pre else "" From 77940e4758fa902d38bbe59b29057f70b966fa28 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sun, 27 Apr 2025 23:45:45 -0500 Subject: [PATCH 31/63] Increase default count to 20 --- src/github_helper/api/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index ef47d9f..9ef7fb2 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -468,7 +468,7 @@ async def get_releases(self, repo: str) -> RetVal[list[Release]]: async def audit_versions( # noqa: C901 self, repo: str, - count: int = 10, + count: int = 20, ) -> RetVal[list[dict]]: """ Verify that version of a repository have differences. From ad9ea0117d5a7efa90a5af20dabfadead90cad3c Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sun, 27 Apr 2025 23:46:01 -0500 Subject: [PATCH 32/63] Separate package audit into new file --- src/github_helper/api/_pkg_audit.py | 384 ++++++++++++++++++++++++++ src/github_helper/api/versions.py | 400 +--------------------------- 2 files changed, 385 insertions(+), 399 deletions(-) create mode 100644 src/github_helper/api/_pkg_audit.py diff --git a/src/github_helper/api/_pkg_audit.py b/src/github_helper/api/_pkg_audit.py new file mode 100644 index 0000000..5c241d3 --- /dev/null +++ b/src/github_helper/api/_pkg_audit.py @@ -0,0 +1,384 @@ +"""Tools for auditing packages.""" + +import copy +import re + +import logistro +from colored import Fore, Style +from packaging import utils + +from .versions import Version + +_logger = logistro.getLogger(__name__) + +bdist_template = { + "Windows": { + "x86_64": [], + "arm64": [], + "win32": [], + }, + "Mac": { + "x86_64": {}, + "arm64": {}, + "universal2": {}, + }, + "Linux": { + "x86_64": { + "Glibc": { + "2.17": [], + }, + "musl": {}, + }, + "aarch64": { + "Glibc": { + "2.17": [], + }, + "musl": {}, + }, + "armv7l": { + "Glibc": { + "2.17": [], + }, + "musl": {}, + }, + }, +} + + +def compare_audits(v, **audits): # noqa: C901 + """Produce a diff between several audits.""" + all_tags = set() + for a in audits.values(): + audit = a.get(v, {}).get("audit", None) + if not audit: + continue + for project in audit.projects.values(): + all_tags.update(project.get("all_tags", {})) + results = {} + + # this inversion sucks + # what to do if project is missing + for tag in all_tags: + results[tag] = set() + for name, a in audits.items(): + audit = a.get(v, {}).get("audit", None) + if not audit: + results[tag].add(name) + continue + for project in audit.projects.values(): + if tag in project.get("all_tags", {}): + break + else: + results[tag].add(name) + if not results[tag]: + del results[tag] + output = "" + for tag, problems in results.items(): + output += f"{tag}: {', '.join(problems)}\n" + + return output + + +class ReleaseAudit: + """Release audit turns a release object into a summary.""" + + prerelease_agree: bool + """Does the version agree with the mark about prerelease.""" + file_notes: dict[str, dict] + """A dict representing the first interpretation of any file.""" + unknown_files: set + """Files that couldn't be understood trying to calculate notes.""" + ignore_counter: dict[str, int] + + projects: dict + """A list of the projects found in this release.""" + + def __repr__(self): + """Return a summary of the audit.""" + ignore_len = sum(self.ignore_counter.values()) + project_tag_count = [ + f"{k}: {len(v['all_tags'])}" for k, v in self.projects.items() + ] + + return ( + f"pre-agree: {self.prerelease_agree}; " + f"projects: {', '.join(project_tag_count)}; " + f"{len(self.file_notes)} files w/ notes; " + f"{len(self.unknown_files)} unknown files; " + f"{ignore_len} ignored files." + ) + + def __init__(self, release, *, prerelease_respect=False): + """Audit a release object.""" + self.tag = release["tag"] + self.file_notes = {} + self.unknown_files = set() + self.ignore_counter = {} + self.projects = {} + + self.version = Version(self.tag) + if not self.version: + self.prerelease_agree = None + return + self.prerelease_agree = ( + (self.version.is_prerelease == release["prerelease"]) + if "prerelease" in release + else prerelease_respect + ) + + [self.summarize_file(file) for file in release["files"]] + + def __str__(self): # noqa: C901, PLR0912 + """Return a string diff of a python project.""" + ret = "" + if not self.prerelease_agree: + ret += f"{Fore.red}PRERELEASE DISAGREEMENT{Style.reset}\n" + if not self.projects: + ret += f"{Fore.red}No Valid Projects Found.{Style.reset}\n" + else: + for k, v in self.projects.items(): + if k.startswith("python/"): + ## look at bdist/sdist + warn = "" + if not v["bdist"]: + warn = "bdist" + if not v["sdist"]: + warn = ", sdist" if warn else "sdist" + warn = f", {Fore.red}missing {warn}{Style.reset}" if warn else "" + + pure = "" + if v["pure"]: + pure = ( + f", {Fore.green}pure: {', '.join(v['pure'])}{Style.reset}" + ) + + ret += f"{Style.bold}{k}{Style.reset}{warn}{pure}\n" + + ## look at tags + ret += ( + "\n".join(self._build_tree_str(v["bdist-tree"])) + "\n" + if v["bdist-tree"] + else "" + ) + + if v["unknown-tags"]: + ret += " Unknown Tags:\n " + ret += "\n ".join(v["unknown-tags"]) + "\n" + if self.unknown_files: + ret += ( + f"{Fore.yellow}{Style.bold}" + f"{len(self.unknown_files)} Unknown Files:" + f"{Style.reset}\n" + ) + for i, f in enumerate(self.unknown_files): + if i > 3: # noqa: PLR2004 + ret += f" {Fore.yellow}...{Style.reset}\n" + break + ret += f" {Fore.yellow}{f}{Style.reset}\n" + if self.ignore_counter: + ret += "ignored: " + for k, v in self.ignore_counter.items(): + ret += f"{k} {v} time(s)" + ret += "\n" + return ret + + def summarize_file(self, filename: str): + """Analyze filename and dispatch analysis for further processing.""" + notes = self._get_file_notes(filename) + match notes.get("action"): + case "ignore": + why = notes.get("type", "other") + self.ignore_counter[why] = self.ignore_counter.get(why, 0) + 1 + case "python-compat": + self._build_python_project_summary(notes) + case _: + self.unknown_files.add(filename) + self.file_notes[filename] = notes + return notes + + # the action you return will trigger behavior above + def _get_file_notes(self, filename: str): + if filename in (f"{self.tag}.zip", f"{self.tag}.tar.gz"): + return {"type": "gh-archive", "action": "ignore"} + elif filename.endswith("tar.gz"): + try: + name, version = utils.parse_sdist_filename(filename) + except utils.InvalidSdistFilename: + _logger.debug(f"Invalid Sdist Filename: {filename}") + else: + return { + "name": name, + "type": "sdist", + "language": "python", + "action": "python-compat", + } + elif filename.endswith(".whl"): + try: + ( + name, + version, + build, + compat_tags, + ) = utils.parse_wheel_filename(filename) + except utils.InvalidWheelFilename: + _logger.debug(f"Invalid Wheel Filename: {filename}") + return {"error": "invalid name", "value": filename} + else: + return { + "name": name, + "type": "bdist", + "language": "python", + "tags": compat_tags, + "action": "python-compat", + } + elif filename.endswith(("sigstore.json", ".sha256")): + return {"type": "metadata", "action": "ignore"} + return {"error": "unrecognized name", "value": filename} + + # So, refactor, python project should be it's own class + # And maybe should have its own adapters? + def _build_python_project_summary(self, notes): # noqa: PLR0912, C901 + name = f"python/{notes['name']}" + if name not in self.projects: + self.projects[name] = { + "sdist": False, + "bdist": False, + "pure": [], + "bdist-tree": None, + "unknown-tags": [], + "all_tags": set(), + } + ref = self.projects[name] + if notes.get("type") == "sdist": + ref["sdist"] = True + elif notes.get("type") == "bdist": + ref["bdist"] = True + ref["all_tags"].update(notes.get("tags")) + for t in notes.get("tags"): + pair = f"{t.interpreter}-{t.abi}" + # t.abi, t.interpreter, t.platform + if t.platform == "any" and t.abi == "none": + ref["pure"].append(t.interpreter) + continue + + if not ref["bdist-tree"]: + ref["bdist-tree"] = copy.deepcopy(bdist_template) + + if t.platform == "any": + if "All OS" not in ref["bdist-tree"]: + ref["bdist-tree"]["All OS"] = [pair] + else: + ref["bdist-tree"]["All OS"].append(pair) + elif r := self._parse_mac_platform(t.platform): + arch_dict = ref["bdist-tree"]["Mac"][r["arch"]] + if r["version"] not in arch_dict: + arch_dict[r["version"]] = [pair] + else: + arch_dict[r["version"]].append(pair) + elif r := self._parse_win_platform(t.platform): + ref["bdist-tree"]["Windows"][r].append(pair) + elif (r := self._parse_manylinux_platform(t.platform)) or ( + r := self._parse_musllinux_platform(t.platform) + ): + if r["arch"] not in ref["bdist-tree"]["Linux"]: + ref["bdist-tree"]["Linux"][r["arch"]] = { + "Glibc": {"2.17": []}, + "musl": {}, + } + libc = r["libc"] + libc_dict = ref["bdist-tree"]["Linux"][r["arch"]][libc] + if r["version"] not in libc_dict: + libc_dict[r["version"]] = [pair] + else: + libc_dict[r["version"]].append(pair) + else: + ref["unknown-tags"].append(str(t)) + + def _parse_mac_platform(self, tag): + pattern = r"^macosx_(\d+)(?:_(\d+))?_(.+)$" + m = re.match(pattern, tag) + if not m: + return {} + + major = int(m.group(1)) + minor = int(m.group(2) or 0) # Default minor version to 0 if omitted + arch = m.group(3) + return {"version": f"{major!s}.{minor!s}", "arch": arch} + + def _parse_win_platform(self, tag: str): + tag = tag.lower() + if tag == "win32": + return "win32" + if tag.startswith("win_"): + arch = tag.split("_", 1)[1] + if arch == "amd64": + return "x86_64" + elif arch == "arm64": + return "arm64" + return False + + def _parse_manylinux_platform(self, tag: str): + if tag.startswith("manylinux_"): + # PEP 600 perennial tag: manylinux_X_Y_arch + m = re.match(r"^manylinux_([0-9]+)_([0-9]+)_(.+)$", tag) + if not m: + return {} + glibc_major = int(m.group(1)) + glibc_minor = int(m.group(2)) + arch = m.group(3) + return { + "version": f"{glibc_major}.{glibc_minor}", + "arch": arch, + "libc": "Glibc", + } + elif tag.startswith("manylinux"): + # Legacies: *1_x86_64, *2010_i686, *2014_x86_64, etc. + m = re.match(r"^manylinux(\d+)_(.+)$", tag) + if not m: + return {} + identifier = m.group(1) # e.g. "1", "2010", "2014" + arch = m.group(2) + # Map known legacy identifiers to glibc versions (optional) + version = None + if identifier == "1": + version = "2.5" + elif identifier == "2010": + version = "2.12" + elif identifier == "2014": + version = "2.17" + else: + return {} + return {"version": version, "arch": arch, "libc": "Glibc"} + else: + return {} + + def _parse_musllinux_platform(self, tag: str): + m = re.match(r"^musllinux_([0-9]+)_([0-9]+)_(.+)$", tag) + if not m: + return False + musl_major = int(m.group(1)) + musl_minor = int(m.group(2)) + arch = m.group(3) + return { + "version": f"{musl_major}.{musl_minor}", + "arch": arch, + "libc": "musl", + } + + def _build_tree_str(self, obj, indent="", *, is_last=True): + lines = [] + + if isinstance(obj, dict): + items = list(obj.items()) + for i, (key, value) in enumerate(items): + is_last = i == len(items) - 1 + branch = "`-- " if is_last else "|-- " + next_indent = indent + (" " if is_last else "| ") + lines.append(f"{indent}{branch}{key}") + if not value: + lines[-1] += f" {Fore.red}missing{Style.reset}" + elif isinstance(value, dict): + lines.extend(self._build_tree_str(value, next_indent)) + elif isinstance(value, (list, tuple)): + lines[-1] += f" {Fore.green}>> {', '.join(value)}{Style.reset}" + return lines diff --git a/src/github_helper/api/versions.py b/src/github_helper/api/versions.py index 8dcf39a..8a587cf 100644 --- a/src/github_helper/api/versions.py +++ b/src/github_helper/api/versions.py @@ -1,26 +1,5 @@ -""" -Github and Pypi versions can contain multiple files: here are tools. +"""For a unified version framework.""" -How do I improve parsing? - -1. If you want to process a new type of file: - a. Modify `_get_file_notes()` - b. Return a dictionary with a known action + needed key-value pairs - I. Possibly modify the existing action in `add_file()` to deal - with new information - II. Make sure `__str__` knows how to print that information. - c. Create a new action: - I. Add the action to `add_file()` - II. If it's going to add to `self.projects`, you either need to - - Create a new entry in `__str__()` to print it - - Modify the existing code (but you'd probably be doing b.) -2. If you want to update the OS Compatibility tree, to include new tags, - look at the bdist branch conditional in _build_python_summary - - -""" - -import copy import re import sys import warnings @@ -31,7 +10,6 @@ import logistro import semver from colored import Fore, Style -from packaging import utils from packaging import version as pyversion _logger = logistro.getLogger(__name__) @@ -357,379 +335,3 @@ def __lt__(self, other) -> bool: def __hash__(self): """Return hash based on normalized value.""" return self._cmp_tuple().__hash__() - - -#### HERE BE DRAGONS ##### - - -bdist_template = { - "Windows": { - "x86_64": [], - "arm64": [], - "win32": [], - }, - "Mac": { - "x86_64": {}, - "arm64": {}, - "universal2": {}, - }, - "Linux": { - "x86_64": { - "Glibc": { - "2.17": [], - }, - "musl": {}, - }, - "aarch64": { - "Glibc": { - "2.17": [], - }, - "musl": {}, - }, - "armv7l": { - "Glibc": { - "2.17": [], - }, - "musl": {}, - }, - }, -} - - -def compare_audits(v, **audits): # noqa: C901 - """Produce a diff between several audits.""" - all_tags = set() - for a in audits.values(): - audit = a.get(v, {}).get("audit", None) - if not audit: - continue - for project in audit.projects.values(): - all_tags.update(project.get("all_tags", {})) - results = {} - - # this inversion sucks - # what to do if project is missing - for tag in all_tags: - results[tag] = set() - for name, a in audits.items(): - audit = a.get(v, {}).get("audit", None) - if not audit: - results[tag].add(name) - continue - for project in audit.projects.values(): - if tag in project.get("all_tags", {}): - break - else: - results[tag].add(name) - if not results[tag]: - del results[tag] - output = "" - for tag, problems in results.items(): - output += f"{tag}: {', '.join(problems)}\n" - - return output - - -class ReleaseAudit: - """Release audit turns a release object into a summary.""" - - prerelease_agree: bool - """Does the version agree with the mark about prerelease.""" - file_notes: dict[str, dict] - """A dict representing the first interpretation of any file.""" - unknown_files: set - """Files that couldn't be understood trying to calculate notes.""" - ignore_counter: dict[str, int] - - projects: dict - """A list of the projects found in this release.""" - - def __repr__(self): - """Return a summary of the audit.""" - ignore_len = sum(self.ignore_counter.values()) - project_tag_count = [ - f"{k}: {len(v['all_tags'])}" for k, v in self.projects.items() - ] - - return ( - f"pre-agree: {self.prerelease_agree}; " - f"projects: {', '.join(project_tag_count)}; " - f"{len(self.file_notes)} files w/ notes; " - f"{len(self.unknown_files)} unknown files; " - f"{ignore_len} ignored files." - ) - - def __init__(self, release, *, prerelease_respect=False): - """Audit a release object.""" - self.tag = release["tag"] - self.file_notes = {} - self.unknown_files = set() - self.ignore_counter = {} - self.projects = {} - - self.version = Version(self.tag) - if not self.version: - self.prerelease_agree = None - return - self.prerelease_agree = ( - (self.version.is_prerelease == release["prerelease"]) - if "prerelease" in release - else prerelease_respect - ) - - [self.summarize_file(file) for file in release["files"]] - - def __str__(self): # noqa: C901, PLR0912 - """Return a string diff of a python project.""" - ret = "" - if not self.prerelease_agree: - ret += f"{Fore.red}PRERELEASE DISAGREEMENT{Style.reset}\n" - if not self.projects: - ret += f"{Fore.red}No Valid Projects Found.{Style.reset}\n" - else: - for k, v in self.projects.items(): - if k.startswith("python/"): - ## look at bdist/sdist - warn = "" - if not v["bdist"]: - warn = "bdist" - if not v["sdist"]: - warn = ", sdist" if warn else "sdist" - warn = f", {Fore.red}missing {warn}{Style.reset}" if warn else "" - - pure = "" - if v["pure"]: - pure = ( - f", {Fore.green}pure: {', '.join(v['pure'])}{Style.reset}" - ) - - ret += f"{Style.bold}{k}{Style.reset}{warn}{pure}\n" - - ## look at tags - ret += ( - "\n".join(self._build_tree_str(v["bdist-tree"])) + "\n" - if v["bdist-tree"] - else "" - ) - - if v["unknown-tags"]: - ret += " Unknown Tags:\n " - ret += "\n ".join(v["unknown-tags"]) + "\n" - if self.unknown_files: - ret += ( - f"{Fore.yellow}{Style.bold}" - f"{len(self.unknown_files)} Unknown Files:" - f"{Style.reset}\n" - ) - for i, f in enumerate(self.unknown_files): - if i > 3: # noqa: PLR2004 - ret += f" {Fore.yellow}...{Style.reset}\n" - break - ret += f" {Fore.yellow}{f}{Style.reset}\n" - if self.ignore_counter: - ret += "ignored: " - for k, v in self.ignore_counter.items(): - ret += f"{k} {v} time(s)" - ret += "\n" - return ret - - def summarize_file(self, filename: str): - """Analyze filename and dispatch analysis for further processing.""" - notes = self._get_file_notes(filename) - match notes.get("action"): - case "ignore": - why = notes.get("type", "other") - self.ignore_counter[why] = self.ignore_counter.get(why, 0) + 1 - case "python-compat": - self._build_python_project_summary(notes) - case _: - self.unknown_files.add(filename) - self.file_notes[filename] = notes - return notes - - # the action you return will trigger behavior above - def _get_file_notes(self, filename: str): - if filename in (f"{self.tag}.zip", f"{self.tag}.tar.gz"): - return {"type": "gh-archive", "action": "ignore"} - elif filename.endswith("tar.gz"): - try: - name, version = utils.parse_sdist_filename(filename) - except utils.InvalidSdistFilename: - _logger.debug(f"Invalid Sdist Filename: {filename}") - else: - return { - "name": name, - "type": "sdist", - "language": "python", - "action": "python-compat", - } - elif filename.endswith(".whl"): - try: - ( - name, - version, - build, - compat_tags, - ) = utils.parse_wheel_filename(filename) - except utils.InvalidWheelFilename: - _logger.debug(f"Invalid Wheel Filename: {filename}") - return {"error": "invalid name", "value": filename} - else: - return { - "name": name, - "type": "bdist", - "language": "python", - "tags": compat_tags, - "action": "python-compat", - } - elif filename.endswith(("sigstore.json", ".sha256")): - return {"type": "metadata", "action": "ignore"} - return {"error": "unrecognized name", "value": filename} - - # So, refactor, python project should be it's own class - # And maybe should have its own adapters? - def _build_python_project_summary(self, notes): # noqa: PLR0912, C901 - name = f"python/{notes['name']}" - if name not in self.projects: - self.projects[name] = { - "sdist": False, - "bdist": False, - "pure": [], - "bdist-tree": None, - "unknown-tags": [], - "all_tags": set(), - } - ref = self.projects[name] - if notes.get("type") == "sdist": - ref["sdist"] = True - elif notes.get("type") == "bdist": - ref["bdist"] = True - ref["all_tags"].update(notes.get("tags")) - for t in notes.get("tags"): - pair = f"{t.interpreter}-{t.abi}" - # t.abi, t.interpreter, t.platform - if t.platform == "any" and t.abi == "none": - ref["pure"].append(t.interpreter) - continue - - if not ref["bdist-tree"]: - ref["bdist-tree"] = copy.deepcopy(bdist_template) - - if t.platform == "any": - if "All OS" not in ref["bdist-tree"]: - ref["bdist-tree"]["All OS"] = [pair] - else: - ref["bdist-tree"]["All OS"].append(pair) - elif r := self._parse_mac_platform(t.platform): - arch_dict = ref["bdist-tree"]["Mac"][r["arch"]] - if r["version"] not in arch_dict: - arch_dict[r["version"]] = [pair] - else: - arch_dict[r["version"]].append(pair) - elif r := self._parse_win_platform(t.platform): - ref["bdist-tree"]["Windows"][r].append(pair) - elif (r := self._parse_manylinux_platform(t.platform)) or ( - r := self._parse_musllinux_platform(t.platform) - ): - if r["arch"] not in ref["bdist-tree"]["Linux"]: - ref["bdist-tree"]["Linux"][r["arch"]] = { - "Glibc": {"2.17": []}, - "musl": {}, - } - libc = r["libc"] - libc_dict = ref["bdist-tree"]["Linux"][r["arch"]][libc] - if r["version"] not in libc_dict: - libc_dict[r["version"]] = [pair] - else: - libc_dict[r["version"]].append(pair) - else: - ref["unknown-tags"].append(str(t)) - - def _parse_mac_platform(self, tag): - pattern = r"^macosx_(\d+)(?:_(\d+))?_(.+)$" - m = re.match(pattern, tag) - if not m: - return {} - - major = int(m.group(1)) - minor = int(m.group(2) or 0) # Default minor version to 0 if omitted - arch = m.group(3) - return {"version": f"{major!s}.{minor!s}", "arch": arch} - - def _parse_win_platform(self, tag: str): - tag = tag.lower() - if tag == "win32": - return "win32" - if tag.startswith("win_"): - arch = tag.split("_", 1)[1] - if arch == "amd64": - return "x86_64" - elif arch == "arm64": - return "arm64" - return False - - def _parse_manylinux_platform(self, tag: str): - if tag.startswith("manylinux_"): - # PEP 600 perennial tag: manylinux_X_Y_arch - m = re.match(r"^manylinux_([0-9]+)_([0-9]+)_(.+)$", tag) - if not m: - return {} - glibc_major = int(m.group(1)) - glibc_minor = int(m.group(2)) - arch = m.group(3) - return { - "version": f"{glibc_major}.{glibc_minor}", - "arch": arch, - "libc": "Glibc", - } - elif tag.startswith("manylinux"): - # Legacies: *1_x86_64, *2010_i686, *2014_x86_64, etc. - m = re.match(r"^manylinux(\d+)_(.+)$", tag) - if not m: - return {} - identifier = m.group(1) # e.g. "1", "2010", "2014" - arch = m.group(2) - # Map known legacy identifiers to glibc versions (optional) - version = None - if identifier == "1": - version = "2.5" - elif identifier == "2010": - version = "2.12" - elif identifier == "2014": - version = "2.17" - else: - return {} - return {"version": version, "arch": arch, "libc": "Glibc"} - else: - return {} - - def _parse_musllinux_platform(self, tag: str): - m = re.match(r"^musllinux_([0-9]+)_([0-9]+)_(.+)$", tag) - if not m: - return False - musl_major = int(m.group(1)) - musl_minor = int(m.group(2)) - arch = m.group(3) - return { - "version": f"{musl_major}.{musl_minor}", - "arch": arch, - "libc": "musl", - } - - def _build_tree_str(self, obj, indent="", *, is_last=True): - lines = [] - - if isinstance(obj, dict): - items = list(obj.items()) - for i, (key, value) in enumerate(items): - is_last = i == len(items) - 1 - branch = "`-- " if is_last else "|-- " - next_indent = indent + (" " if is_last else "| ") - lines.append(f"{indent}{branch}{key}") - if not value: - lines[-1] += f" {Fore.red}missing{Style.reset}" - elif isinstance(value, dict): - lines.extend(self._build_tree_str(value, next_indent)) - elif isinstance(value, (list, tuple)): - lines[-1] += f" {Fore.green}>> {', '.join(value)}{Style.reset}" - return lines From 5a1a54a000165678db315ba45c85c15923de078c Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Mon, 28 Apr 2025 19:50:01 -0500 Subject: [PATCH 33/63] Refactor audit framework --- src/github_helper/api/_pkg_audit.py | 384 ------------------ src/github_helper/api/_pkg_audit/__init__.py | 75 ++++ src/github_helper/api/_pkg_audit/_py_audit.py | 206 ++++++++++ src/github_helper/api/_pkg_audit/_types.py | 15 + src/github_helper/api/_pkg_audit/note | 121 ++++++ 5 files changed, 417 insertions(+), 384 deletions(-) delete mode 100644 src/github_helper/api/_pkg_audit.py create mode 100644 src/github_helper/api/_pkg_audit/__init__.py create mode 100644 src/github_helper/api/_pkg_audit/_py_audit.py create mode 100644 src/github_helper/api/_pkg_audit/_types.py create mode 100644 src/github_helper/api/_pkg_audit/note diff --git a/src/github_helper/api/_pkg_audit.py b/src/github_helper/api/_pkg_audit.py deleted file mode 100644 index 5c241d3..0000000 --- a/src/github_helper/api/_pkg_audit.py +++ /dev/null @@ -1,384 +0,0 @@ -"""Tools for auditing packages.""" - -import copy -import re - -import logistro -from colored import Fore, Style -from packaging import utils - -from .versions import Version - -_logger = logistro.getLogger(__name__) - -bdist_template = { - "Windows": { - "x86_64": [], - "arm64": [], - "win32": [], - }, - "Mac": { - "x86_64": {}, - "arm64": {}, - "universal2": {}, - }, - "Linux": { - "x86_64": { - "Glibc": { - "2.17": [], - }, - "musl": {}, - }, - "aarch64": { - "Glibc": { - "2.17": [], - }, - "musl": {}, - }, - "armv7l": { - "Glibc": { - "2.17": [], - }, - "musl": {}, - }, - }, -} - - -def compare_audits(v, **audits): # noqa: C901 - """Produce a diff between several audits.""" - all_tags = set() - for a in audits.values(): - audit = a.get(v, {}).get("audit", None) - if not audit: - continue - for project in audit.projects.values(): - all_tags.update(project.get("all_tags", {})) - results = {} - - # this inversion sucks - # what to do if project is missing - for tag in all_tags: - results[tag] = set() - for name, a in audits.items(): - audit = a.get(v, {}).get("audit", None) - if not audit: - results[tag].add(name) - continue - for project in audit.projects.values(): - if tag in project.get("all_tags", {}): - break - else: - results[tag].add(name) - if not results[tag]: - del results[tag] - output = "" - for tag, problems in results.items(): - output += f"{tag}: {', '.join(problems)}\n" - - return output - - -class ReleaseAudit: - """Release audit turns a release object into a summary.""" - - prerelease_agree: bool - """Does the version agree with the mark about prerelease.""" - file_notes: dict[str, dict] - """A dict representing the first interpretation of any file.""" - unknown_files: set - """Files that couldn't be understood trying to calculate notes.""" - ignore_counter: dict[str, int] - - projects: dict - """A list of the projects found in this release.""" - - def __repr__(self): - """Return a summary of the audit.""" - ignore_len = sum(self.ignore_counter.values()) - project_tag_count = [ - f"{k}: {len(v['all_tags'])}" for k, v in self.projects.items() - ] - - return ( - f"pre-agree: {self.prerelease_agree}; " - f"projects: {', '.join(project_tag_count)}; " - f"{len(self.file_notes)} files w/ notes; " - f"{len(self.unknown_files)} unknown files; " - f"{ignore_len} ignored files." - ) - - def __init__(self, release, *, prerelease_respect=False): - """Audit a release object.""" - self.tag = release["tag"] - self.file_notes = {} - self.unknown_files = set() - self.ignore_counter = {} - self.projects = {} - - self.version = Version(self.tag) - if not self.version: - self.prerelease_agree = None - return - self.prerelease_agree = ( - (self.version.is_prerelease == release["prerelease"]) - if "prerelease" in release - else prerelease_respect - ) - - [self.summarize_file(file) for file in release["files"]] - - def __str__(self): # noqa: C901, PLR0912 - """Return a string diff of a python project.""" - ret = "" - if not self.prerelease_agree: - ret += f"{Fore.red}PRERELEASE DISAGREEMENT{Style.reset}\n" - if not self.projects: - ret += f"{Fore.red}No Valid Projects Found.{Style.reset}\n" - else: - for k, v in self.projects.items(): - if k.startswith("python/"): - ## look at bdist/sdist - warn = "" - if not v["bdist"]: - warn = "bdist" - if not v["sdist"]: - warn = ", sdist" if warn else "sdist" - warn = f", {Fore.red}missing {warn}{Style.reset}" if warn else "" - - pure = "" - if v["pure"]: - pure = ( - f", {Fore.green}pure: {', '.join(v['pure'])}{Style.reset}" - ) - - ret += f"{Style.bold}{k}{Style.reset}{warn}{pure}\n" - - ## look at tags - ret += ( - "\n".join(self._build_tree_str(v["bdist-tree"])) + "\n" - if v["bdist-tree"] - else "" - ) - - if v["unknown-tags"]: - ret += " Unknown Tags:\n " - ret += "\n ".join(v["unknown-tags"]) + "\n" - if self.unknown_files: - ret += ( - f"{Fore.yellow}{Style.bold}" - f"{len(self.unknown_files)} Unknown Files:" - f"{Style.reset}\n" - ) - for i, f in enumerate(self.unknown_files): - if i > 3: # noqa: PLR2004 - ret += f" {Fore.yellow}...{Style.reset}\n" - break - ret += f" {Fore.yellow}{f}{Style.reset}\n" - if self.ignore_counter: - ret += "ignored: " - for k, v in self.ignore_counter.items(): - ret += f"{k} {v} time(s)" - ret += "\n" - return ret - - def summarize_file(self, filename: str): - """Analyze filename and dispatch analysis for further processing.""" - notes = self._get_file_notes(filename) - match notes.get("action"): - case "ignore": - why = notes.get("type", "other") - self.ignore_counter[why] = self.ignore_counter.get(why, 0) + 1 - case "python-compat": - self._build_python_project_summary(notes) - case _: - self.unknown_files.add(filename) - self.file_notes[filename] = notes - return notes - - # the action you return will trigger behavior above - def _get_file_notes(self, filename: str): - if filename in (f"{self.tag}.zip", f"{self.tag}.tar.gz"): - return {"type": "gh-archive", "action": "ignore"} - elif filename.endswith("tar.gz"): - try: - name, version = utils.parse_sdist_filename(filename) - except utils.InvalidSdistFilename: - _logger.debug(f"Invalid Sdist Filename: {filename}") - else: - return { - "name": name, - "type": "sdist", - "language": "python", - "action": "python-compat", - } - elif filename.endswith(".whl"): - try: - ( - name, - version, - build, - compat_tags, - ) = utils.parse_wheel_filename(filename) - except utils.InvalidWheelFilename: - _logger.debug(f"Invalid Wheel Filename: {filename}") - return {"error": "invalid name", "value": filename} - else: - return { - "name": name, - "type": "bdist", - "language": "python", - "tags": compat_tags, - "action": "python-compat", - } - elif filename.endswith(("sigstore.json", ".sha256")): - return {"type": "metadata", "action": "ignore"} - return {"error": "unrecognized name", "value": filename} - - # So, refactor, python project should be it's own class - # And maybe should have its own adapters? - def _build_python_project_summary(self, notes): # noqa: PLR0912, C901 - name = f"python/{notes['name']}" - if name not in self.projects: - self.projects[name] = { - "sdist": False, - "bdist": False, - "pure": [], - "bdist-tree": None, - "unknown-tags": [], - "all_tags": set(), - } - ref = self.projects[name] - if notes.get("type") == "sdist": - ref["sdist"] = True - elif notes.get("type") == "bdist": - ref["bdist"] = True - ref["all_tags"].update(notes.get("tags")) - for t in notes.get("tags"): - pair = f"{t.interpreter}-{t.abi}" - # t.abi, t.interpreter, t.platform - if t.platform == "any" and t.abi == "none": - ref["pure"].append(t.interpreter) - continue - - if not ref["bdist-tree"]: - ref["bdist-tree"] = copy.deepcopy(bdist_template) - - if t.platform == "any": - if "All OS" not in ref["bdist-tree"]: - ref["bdist-tree"]["All OS"] = [pair] - else: - ref["bdist-tree"]["All OS"].append(pair) - elif r := self._parse_mac_platform(t.platform): - arch_dict = ref["bdist-tree"]["Mac"][r["arch"]] - if r["version"] not in arch_dict: - arch_dict[r["version"]] = [pair] - else: - arch_dict[r["version"]].append(pair) - elif r := self._parse_win_platform(t.platform): - ref["bdist-tree"]["Windows"][r].append(pair) - elif (r := self._parse_manylinux_platform(t.platform)) or ( - r := self._parse_musllinux_platform(t.platform) - ): - if r["arch"] not in ref["bdist-tree"]["Linux"]: - ref["bdist-tree"]["Linux"][r["arch"]] = { - "Glibc": {"2.17": []}, - "musl": {}, - } - libc = r["libc"] - libc_dict = ref["bdist-tree"]["Linux"][r["arch"]][libc] - if r["version"] not in libc_dict: - libc_dict[r["version"]] = [pair] - else: - libc_dict[r["version"]].append(pair) - else: - ref["unknown-tags"].append(str(t)) - - def _parse_mac_platform(self, tag): - pattern = r"^macosx_(\d+)(?:_(\d+))?_(.+)$" - m = re.match(pattern, tag) - if not m: - return {} - - major = int(m.group(1)) - minor = int(m.group(2) or 0) # Default minor version to 0 if omitted - arch = m.group(3) - return {"version": f"{major!s}.{minor!s}", "arch": arch} - - def _parse_win_platform(self, tag: str): - tag = tag.lower() - if tag == "win32": - return "win32" - if tag.startswith("win_"): - arch = tag.split("_", 1)[1] - if arch == "amd64": - return "x86_64" - elif arch == "arm64": - return "arm64" - return False - - def _parse_manylinux_platform(self, tag: str): - if tag.startswith("manylinux_"): - # PEP 600 perennial tag: manylinux_X_Y_arch - m = re.match(r"^manylinux_([0-9]+)_([0-9]+)_(.+)$", tag) - if not m: - return {} - glibc_major = int(m.group(1)) - glibc_minor = int(m.group(2)) - arch = m.group(3) - return { - "version": f"{glibc_major}.{glibc_minor}", - "arch": arch, - "libc": "Glibc", - } - elif tag.startswith("manylinux"): - # Legacies: *1_x86_64, *2010_i686, *2014_x86_64, etc. - m = re.match(r"^manylinux(\d+)_(.+)$", tag) - if not m: - return {} - identifier = m.group(1) # e.g. "1", "2010", "2014" - arch = m.group(2) - # Map known legacy identifiers to glibc versions (optional) - version = None - if identifier == "1": - version = "2.5" - elif identifier == "2010": - version = "2.12" - elif identifier == "2014": - version = "2.17" - else: - return {} - return {"version": version, "arch": arch, "libc": "Glibc"} - else: - return {} - - def _parse_musllinux_platform(self, tag: str): - m = re.match(r"^musllinux_([0-9]+)_([0-9]+)_(.+)$", tag) - if not m: - return False - musl_major = int(m.group(1)) - musl_minor = int(m.group(2)) - arch = m.group(3) - return { - "version": f"{musl_major}.{musl_minor}", - "arch": arch, - "libc": "musl", - } - - def _build_tree_str(self, obj, indent="", *, is_last=True): - lines = [] - - if isinstance(obj, dict): - items = list(obj.items()) - for i, (key, value) in enumerate(items): - is_last = i == len(items) - 1 - branch = "`-- " if is_last else "|-- " - next_indent = indent + (" " if is_last else "| ") - lines.append(f"{indent}{branch}{key}") - if not value: - lines[-1] += f" {Fore.red}missing{Style.reset}" - elif isinstance(value, dict): - lines.extend(self._build_tree_str(value, next_indent)) - elif isinstance(value, (list, tuple)): - lines[-1] += f" {Fore.green}>> {', '.join(value)}{Style.reset}" - return lines diff --git a/src/github_helper/api/_pkg_audit/__init__.py b/src/github_helper/api/_pkg_audit/__init__.py new file mode 100644 index 0000000..892de06 --- /dev/null +++ b/src/github_helper/api/_pkg_audit/__init__.py @@ -0,0 +1,75 @@ +"""Tools for auditing packages.""" + +import logistro + +from github_helper.api.versions import Version + +from ._py_audit import PythonAudit +from ._types import ReturnMessages + +_logger = logistro.getLogger(__name__) + +class ReleaseAudit: + """ + Release audit turns a release object into a summary. + + The release audit goes through an audit, file by file, + and tries to sort them as well as looking to see if sub-processors + understand them. + """ + + prerelease_agree: bool + """Does the version agree with the mark about prerelease.""" + unknown_files: set + """Files that couldn't be understood trying to calculate notes.""" + ignored_files: dict[str, int] + + projects: dict + """A list of the projects found in this release.""" + + def __init__(self, release): + """Audit a release object.""" + self.tag = release.tag + self.unknown_files = set() + self.ignored_files = {} + self.version = Version(self.tag) + self.python_audit = PythonAudit(self.version) + + + if not self.version.valid: # we should not audit not valid + self.prerelease_agree = None + return + + self.prerelease_agree = ( + self.version.is_prerelease == release.prerelease + ) + + for filename in release.files: + notes = self.process_file(filename) + + match notes.get("action"): + case "ignore": + why = notes.get("type", "other") + self.ignored_files[why] = self.ignored_files.get(why, 0) + 1 + case "resolved": + pass + case _: + self.unknown_files.add(filename) + + + # move these to python + # create specific types that can be returned + # python-compat to audit-projects and use language + # use glom + + # the action you return will trigger behavior above + def process_file(self, filename: str) -> ReturnMessages: + if filename in (f"{self.tag}.zip", f"{self.tag}.tar.gz"): + return {"type": "gh-archive", "action": "ignore"} + elif filename.endswith(("sigstore.json", ".sha256")): + return {"type": "metadata", "action": "ignore"} + elif (note := self.python_audit.check_file(filename)): + return note + return {"error": "unrecognized name", "value": filename} + + diff --git a/src/github_helper/api/_pkg_audit/_py_audit.py b/src/github_helper/api/_pkg_audit/_py_audit.py new file mode 100644 index 0000000..06cbe4a --- /dev/null +++ b/src/github_helper/api/_pkg_audit/_py_audit.py @@ -0,0 +1,206 @@ +import copy +import re +from dataclasses import dataclass, field +from typing import Literal + +import logistro +from colored import Fore, Style +from packaging import tags, utils + +from github_helper.api.versions import Version + +from ._types import Audit, Error + +_logger = logistro.getLogger(__name__) + +bdist_template = { + "Windows": { + "x86_64": [], + "arm64": [], + "win32": [], + }, + "Mac": { + "x86_64": {}, + "arm64": {}, + "universal2": {}, + }, + "Linux": { + "x86_64": { + "Glibc": { + "2.17": [], + }, + "musl": {}, + }, + "aarch64": { + "Glibc": { + "2.17": [], + }, + "musl": {}, + }, + "armv7l": { + "Glibc": { + "2.17": [], + }, + "musl": {}, + }, + }, +} + +class PythonAudit: + projects: dict[str, "Project"] + version: Version + + def __init__(self, v: Version) -> None: + self.projects: dict[str, Project] = {} + self.version = v + + def check_file(self, filename) -> Audit | Error | None: + if filename.endswith("tar.gz"): + try: + name, version = utils.parse_sdist_filename(filename) + except utils.InvalidSdistFilename: + _logger.debug(f"Invalid Sdist Filename: {filename}") + return None + else: + self.projects.setdefault(name, Project()) + self.projects[name].add_sdist(filename) + # TODO check version against tag + return { "name": name, "action": "resolved" } + elif filename.endswith(".whl"): + try: + ( + name, + version, + build, + tags, + ) = utils.parse_wheel_filename(filename) + except utils.InvalidWheelFilename: + _logger.debug(f"Invalid Wheel Filename: {filename}") + return {"error": "invalid name", "value": filename} + else: + self.projects.setdefault(name, Project()) + self.projects[name].add_bdist(filename, tags) + # TODO check version against + return { "name": name, "action": "resolved" } + return None + + +@dataclass(slots=True, kw_only=True) +class Project: + sdist: bool = False + bdist: bool = False + weird_version: bool = False + pure_tags: set[tags.Tag] = field(default_factory=set) + all_tags: set[tags.Tag] = field(default_factory=set) + unknown_tags: set[tags.Tag] = field(default_factory=set) + files: set[str] = field(default_factory=set) + bdist_tree: dict | None = None + + def add_sdist(self, file: str): + self.sdist = True + self.files.add(file) + + def add_bdist(self, file: str, tags: set[tags.Tag] | frozenset[tags.Tag]) -> None: + self.bdist = True + self.files.add(file) + self.all_tags.update(tags) + for t in tags: + # t.interpreter, t.abi, t.platform + pair = f"{t.interpreter}-{t.abi}" + if t.abi == "none" and t.platform == "any": + self.pure_tags.add(t) + continue + if not self.bdist_tree: + self.bdist_tree = copy.deepcopy(bdist_template) + if t.platform == "any": + x = self.bdist_tree.setdefault("All OS", []) + x.append(pair) + elif (mactag := self._parse_mac_platform(t.platform)): + x = self.bdist_tree["Mac"][mactag.arch].setdefault("version", []) + x.append(pair) + elif (arch := self._parse_win_platform(t.platform)): + self.bdist_tree["Windows"][arch].append(pair) + elif (linuxtag := self._parse_linux_platform(t.platform)): + arch_default: dict = { "Glibc": {"2.17": []}, "musl": {} } + x = self.bdist_tree["Linux"].setdefault(linuxtag.arch, arch_default) + x[linuxtag.arch].setdefault(linuxtag.version, []).append(pair) + else: + self.unknown_tags.add(t) + + @dataclass(slots=True, frozen=True) + class MacTag: + version: str + arch: str + def _parse_mac_platform(self, tag) -> MacTag | None: + pattern = r"^macosx_(\d+)(?:_(\d+))?_(.+)$" + m = re.match(pattern, tag) + if not m: + return None + + major = int(m.group(1)) + minor = int(m.group(2) or 0) # Default minor version to 0 if omitted + arch = m.group(3) + return Project.MacTag(version=f"{major!s}.{minor!s}", arch= arch) + + def _parse_win_platform( + self, tag: str + ) -> Literal["win32", "x86_64", "arm64"] | None: + tag = tag.lower() + if tag == "win32": + return "win32" + if tag.startswith("win_"): + arch = tag.split("_", 1)[1] + if arch == "amd64": + return "x86_64" + elif arch == "arm64": + return "arm64" + return None + + @dataclass(frozen=True, slots=True) + class LinuxTag: + version: str + arch: str + libc: Literal["Glibc", "musl"] + + def _parse_linux_platform(self, tag: str) -> LinuxTag | None: + libc: Literal["Glibc", "musl"] + if tag.startswith("manylinux_"): + libc = "Glibc" + # PEP 600 perennial tag: manylinux_X_Y_arch + m = re.match(r"^manylinux_([0-9]+)_([0-9]+)_(.+)$", tag) + if not m: + return None + glibc_major = int(m.group(1)) + glibc_minor = int(m.group(2)) + arch = m.group(3) + version = f"{glibc_major}.{glibc_minor}" + elif tag.startswith("manylinux"): + libc = "Glibc" + # Legacies: *1_x86_64, *2010_i686, *2014_x86_64, etc. + m = re.match(r"^manylinux(\d+)_(.+)$", tag) + if not m: + return None + identifier = m.group(1) # e.g. "1", "2010", "2014" + arch = m.group(2) + # Map known legacy identifiers to glibc versions (optional) + if identifier == "1": + version = "2.5" + elif identifier == "2010": + version = "2.12" + elif identifier == "2014": + version = "2.17" + else: + return None + elif (m := re.match(r"^musllinux_([0-9]+)_([0-9]+)_(.+)$", tag)): + libc = "musl" + musl_major = int(m.group(1)) + musl_minor = int(m.group(2)) + arch = m.group(3) + version = f"{musl_major}.{musl_minor}" + else: + return None + return Project.LinuxTag( + version= version, + arch = arch, + libc = libc, + ) diff --git a/src/github_helper/api/_pkg_audit/_types.py b/src/github_helper/api/_pkg_audit/_types.py new file mode 100644 index 0000000..063e0bb --- /dev/null +++ b/src/github_helper/api/_pkg_audit/_types.py @@ -0,0 +1,15 @@ + +from typing import Literal, TypedDict + + +class Ignore(TypedDict): + type: str + action: Literal["ignore"] +class Error(TypedDict): + error: str + value: str +class Audit(TypedDict): + name: str + action: Literal["resolved"] + +ReturnMessages = Ignore | Error | Audit diff --git a/src/github_helper/api/_pkg_audit/note b/src/github_helper/api/_pkg_audit/note new file mode 100644 index 0000000..9d32ed7 --- /dev/null +++ b/src/github_helper/api/_pkg_audit/note @@ -0,0 +1,121 @@ +def __repr__(self): + """Return a summary of the audit.""" + ignore_len = sum(self.ignore_counter.values()) + project_tag_count = [ + f"{k}: {len(v['all_tags'])}" for k, v in self.projects.items() + ] + + return ( + f"pre-agree: {self.prerelease_agree}; " + f"projects: {', '.join(project_tag_count)}; " + f"{len(self.file_notes)} files w/ notes; " + f"{len(self.unknown_files)} unknown files; " + f"{ignore_len} ignored files." + ) + +def compare_audits(v, **audits): # noqa: C901 + """Produce a diff between several audits.""" + all_tags = set() + for a in audits.values(): + audit = a.get(v, {}).get("audit", None) + if not audit: + continue + for project in audit.projects.values(): + all_tags.update(project.get("all_tags", {})) + results = {} + + # this inversion sucks + # what to do if project is missing + for tag in all_tags: + results[tag] = set() + for name, a in audits.items(): + audit = a.get(v, {}).get("audit", None) + if not audit: + results[tag].add(name) + continue + for project in audit.projects.values(): + if tag in project.get("all_tags", {}): + break + else: + results[tag].add(name) + if not results[tag]: + del results[tag] + output = "" + for tag, problems in results.items(): + output += f"{tag}: {', '.join(problems)}\n" + + return output + + + def __str__(self): # noqa: C901, PLR0912 + """Return a string diff of a python project.""" + ret = "" + if not self.prerelease_agree: + ret += f"{Fore.red}PRERELEASE DISAGREEMENT{Style.reset}\n" + if not self.projects: + ret += f"{Fore.red}No Valid Projects Found.{Style.reset}\n" + else: + for k, v in self.projects.items(): + if k.startswith("python/"): + ## look at bdist/sdist + warn = "" + if not v["bdist"]: + warn = "bdist" + if not v["sdist"]: + warn = ", sdist" if warn else "sdist" + warn = f", {Fore.red}missing {warn}{Style.reset}" if warn else "" + + pure = "" + if v["pure"]: + pure = ( + f", {Fore.green}pure: {', '.join(v['pure'])}{Style.reset}" + ) + + ret += f"{Style.bold}{k}{Style.reset}{warn}{pure}\n" + + ## look at tags + ret += ( + "\n".join(self._build_tree_str(v["bdist-tree"])) + "\n" + if v["bdist-tree"] + else "" + ) + + if v["unknown-tags"]: + ret += " Unknown Tags:\n " + ret += "\n ".join(v["unknown-tags"]) + "\n" + if self.unknown_files: + ret += ( + f"{Fore.yellow}{Style.bold}" + f"{len(self.unknown_files)} Unknown Files:" + f"{Style.reset}\n" + ) + for i, f in enumerate(self.unknown_files): + if i > 3: # noqa: PLR2004 + ret += f" {Fore.yellow}...{Style.reset}\n" + break + ret += f" {Fore.yellow}{f}{Style.reset}\n" + if self.ignore_counter: + ret += "ignored: " + for k, v in self.ignore_counter.items(): + ret += f"{k} {v} time(s)" + ret += "\n" + return ret + + + def _build_tree_str(self, obj, indent="", *, is_last=True): + lines = [] + + if isinstance(obj, dict): + items = list(obj.items()) + for i, (key, value) in enumerate(items): + is_last = i == len(items) - 1 + branch = "`-- " if is_last else "|-- " + next_indent = indent + (" " if is_last else "| ") + lines.append(f"{indent}{branch}{key}") + if not value: + lines[-1] += f" {Fore.red}missing{Style.reset}" + elif isinstance(value, dict): + lines.extend(self._build_tree_str(value, next_indent)) + elif isinstance(value, (list, tuple)): + lines[-1] += f" {Fore.green}>> {', '.join(value)}{Style.reset}" + return lines From 76849acd64c67c79bf7ef2e3453b564a5044e13d Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Mon, 28 Apr 2025 19:53:22 -0500 Subject: [PATCH 34/63] Format and check versionweird --- src/github_helper/api/_pkg_audit/__init__.py | 13 +++---- src/github_helper/api/_pkg_audit/_py_audit.py | 35 +++++++++++-------- src/github_helper/api/_pkg_audit/_types.py | 6 +++- 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/src/github_helper/api/_pkg_audit/__init__.py b/src/github_helper/api/_pkg_audit/__init__.py index 892de06..ebec63e 100644 --- a/src/github_helper/api/_pkg_audit/__init__.py +++ b/src/github_helper/api/_pkg_audit/__init__.py @@ -9,6 +9,7 @@ _logger = logistro.getLogger(__name__) + class ReleaseAudit: """ Release audit turns a release object into a summary. @@ -35,14 +36,11 @@ def __init__(self, release): self.version = Version(self.tag) self.python_audit = PythonAudit(self.version) - - if not self.version.valid: # we should not audit not valid + if not self.version.valid: # we should not audit not valid self.prerelease_agree = None return - self.prerelease_agree = ( - self.version.is_prerelease == release.prerelease - ) + self.prerelease_agree = self.version.is_prerelease == release.prerelease for filename in release.files: notes = self.process_file(filename) @@ -56,7 +54,6 @@ def __init__(self, release): case _: self.unknown_files.add(filename) - # move these to python # create specific types that can be returned # python-compat to audit-projects and use language @@ -68,8 +65,6 @@ def process_file(self, filename: str) -> ReturnMessages: return {"type": "gh-archive", "action": "ignore"} elif filename.endswith(("sigstore.json", ".sha256")): return {"type": "metadata", "action": "ignore"} - elif (note := self.python_audit.check_file(filename)): + elif note := self.python_audit.check_file(filename): return note return {"error": "unrecognized name", "value": filename} - - diff --git a/src/github_helper/api/_pkg_audit/_py_audit.py b/src/github_helper/api/_pkg_audit/_py_audit.py index 06cbe4a..fb4709c 100644 --- a/src/github_helper/api/_pkg_audit/_py_audit.py +++ b/src/github_helper/api/_pkg_audit/_py_audit.py @@ -46,6 +46,7 @@ }, } + class PythonAudit: projects: dict[str, "Project"] version: Version @@ -64,8 +65,10 @@ def check_file(self, filename) -> Audit | Error | None: else: self.projects.setdefault(name, Project()) self.projects[name].add_sdist(filename) - # TODO check version against tag - return { "name": name, "action": "resolved" } + self.projects[name].version_weird = ( + self.version != version or self.projects[name].version_weird + ) + return {"name": name, "action": "resolved"} elif filename.endswith(".whl"): try: ( @@ -80,8 +83,10 @@ def check_file(self, filename) -> Audit | Error | None: else: self.projects.setdefault(name, Project()) self.projects[name].add_bdist(filename, tags) - # TODO check version against - return { "name": name, "action": "resolved" } + self.projects[name].version_weird = ( + self.version != version or self.projects[name].version_weird + ) + return {"name": name, "action": "resolved"} return None @@ -115,13 +120,13 @@ def add_bdist(self, file: str, tags: set[tags.Tag] | frozenset[tags.Tag]) -> Non if t.platform == "any": x = self.bdist_tree.setdefault("All OS", []) x.append(pair) - elif (mactag := self._parse_mac_platform(t.platform)): + elif mactag := self._parse_mac_platform(t.platform): x = self.bdist_tree["Mac"][mactag.arch].setdefault("version", []) x.append(pair) - elif (arch := self._parse_win_platform(t.platform)): + elif arch := self._parse_win_platform(t.platform): self.bdist_tree["Windows"][arch].append(pair) - elif (linuxtag := self._parse_linux_platform(t.platform)): - arch_default: dict = { "Glibc": {"2.17": []}, "musl": {} } + elif linuxtag := self._parse_linux_platform(t.platform): + arch_default: dict = {"Glibc": {"2.17": []}, "musl": {}} x = self.bdist_tree["Linux"].setdefault(linuxtag.arch, arch_default) x[linuxtag.arch].setdefault(linuxtag.version, []).append(pair) else: @@ -131,6 +136,7 @@ def add_bdist(self, file: str, tags: set[tags.Tag] | frozenset[tags.Tag]) -> Non class MacTag: version: str arch: str + def _parse_mac_platform(self, tag) -> MacTag | None: pattern = r"^macosx_(\d+)(?:_(\d+))?_(.+)$" m = re.match(pattern, tag) @@ -140,10 +146,11 @@ def _parse_mac_platform(self, tag) -> MacTag | None: major = int(m.group(1)) minor = int(m.group(2) or 0) # Default minor version to 0 if omitted arch = m.group(3) - return Project.MacTag(version=f"{major!s}.{minor!s}", arch= arch) + return Project.MacTag(version=f"{major!s}.{minor!s}", arch=arch) def _parse_win_platform( - self, tag: str + self, + tag: str, ) -> Literal["win32", "x86_64", "arm64"] | None: tag = tag.lower() if tag == "win32": @@ -191,7 +198,7 @@ def _parse_linux_platform(self, tag: str) -> LinuxTag | None: version = "2.17" else: return None - elif (m := re.match(r"^musllinux_([0-9]+)_([0-9]+)_(.+)$", tag)): + elif m := re.match(r"^musllinux_([0-9]+)_([0-9]+)_(.+)$", tag): libc = "musl" musl_major = int(m.group(1)) musl_minor = int(m.group(2)) @@ -200,7 +207,7 @@ def _parse_linux_platform(self, tag: str) -> LinuxTag | None: else: return None return Project.LinuxTag( - version= version, - arch = arch, - libc = libc, + version=version, + arch=arch, + libc=libc, ) diff --git a/src/github_helper/api/_pkg_audit/_types.py b/src/github_helper/api/_pkg_audit/_types.py index 063e0bb..0381082 100644 --- a/src/github_helper/api/_pkg_audit/_types.py +++ b/src/github_helper/api/_pkg_audit/_types.py @@ -1,15 +1,19 @@ - from typing import Literal, TypedDict class Ignore(TypedDict): type: str action: Literal["ignore"] + + class Error(TypedDict): error: str value: str + + class Audit(TypedDict): name: str action: Literal["resolved"] + ReturnMessages = Ignore | Error | Audit From fa6af877ef916f49879fabf4e4307805369e4031 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Mon, 28 Apr 2025 19:54:18 -0500 Subject: [PATCH 35/63] Remove comments --- src/github_helper/api/_pkg_audit/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/github_helper/api/_pkg_audit/__init__.py b/src/github_helper/api/_pkg_audit/__init__.py index ebec63e..740731e 100644 --- a/src/github_helper/api/_pkg_audit/__init__.py +++ b/src/github_helper/api/_pkg_audit/__init__.py @@ -54,11 +54,6 @@ def __init__(self, release): case _: self.unknown_files.add(filename) - # move these to python - # create specific types that can be returned - # python-compat to audit-projects and use language - # use glom - # the action you return will trigger behavior above def process_file(self, filename: str) -> ReturnMessages: if filename in (f"{self.tag}.zip", f"{self.tag}.tar.gz"): From 6c77bb8f6550d930b6727f08f407c48d20d4e57d Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Mon, 28 Apr 2025 20:42:13 -0500 Subject: [PATCH 36/63] Fix bad var name --- src/github_helper/api/_pkg_audit/_py_audit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/github_helper/api/_pkg_audit/_py_audit.py b/src/github_helper/api/_pkg_audit/_py_audit.py index fb4709c..75fcd94 100644 --- a/src/github_helper/api/_pkg_audit/_py_audit.py +++ b/src/github_helper/api/_pkg_audit/_py_audit.py @@ -94,7 +94,7 @@ def check_file(self, filename) -> Audit | Error | None: class Project: sdist: bool = False bdist: bool = False - weird_version: bool = False + version_weird: bool = False pure_tags: set[tags.Tag] = field(default_factory=set) all_tags: set[tags.Tag] = field(default_factory=set) unknown_tags: set[tags.Tag] = field(default_factory=set) From 968c67b32b43742964bd66c54282137e56a65112 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Mon, 28 Apr 2025 20:42:38 -0500 Subject: [PATCH 37/63] Bug fix and run audit --- src/github_helper/api/__init__.py | 12 ++++++++---- src/github_helper/api/_pkg_audit/_py_audit.py | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 9ef7fb2..eff031d 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -23,6 +23,8 @@ from github_helper._utils import load_json from github_helper.api import _audit, versions +from ._pkg_audit import ReleaseAudit + _logger = logistro.getLogger(__name__) _SCRIPT_DIR = Path(__file__).resolve().parent _TEMPLATE_PATH = _SCRIPT_DIR / "templates" @@ -390,6 +392,7 @@ class Release(Tag): """Files that came with it.""" version: versions.Version """Calculated version.""" + audit: ReleaseAudit | None = None async def get_pypi( self, @@ -547,14 +550,15 @@ def print_source_status(self, name: str, canonical: str) -> str: v = getattr(o, "version", None) or versions.Version(o.tag) if not v.valid: continue - if v not in all_versions: - all_versions[v] = VersionSet() - if getattr(all_versions[v], attrname, None): + vset = all_versions.setdefault(v, VersionSet()) + if getattr(vset, attrname, None): warnings.warn( "Looks like conflicting poorly-written versions caused overwrite.", stacklevel=2, ) - setattr(all_versions[v], attrname, o) + if isinstance(o, GHApi.Release): + o.audit = ReleaseAudit(o) + setattr(vset, attrname, o) all_versions = dict(sorted(all_versions.items(), reverse=True)) result = [ diff --git a/src/github_helper/api/_pkg_audit/_py_audit.py b/src/github_helper/api/_pkg_audit/_py_audit.py index 75fcd94..765ba2e 100644 --- a/src/github_helper/api/_pkg_audit/_py_audit.py +++ b/src/github_helper/api/_pkg_audit/_py_audit.py @@ -128,7 +128,7 @@ def add_bdist(self, file: str, tags: set[tags.Tag] | frozenset[tags.Tag]) -> Non elif linuxtag := self._parse_linux_platform(t.platform): arch_default: dict = {"Glibc": {"2.17": []}, "musl": {}} x = self.bdist_tree["Linux"].setdefault(linuxtag.arch, arch_default) - x[linuxtag.arch].setdefault(linuxtag.version, []).append(pair) + x.setdefault(linuxtag.version, []).append(pair) else: self.unknown_tags.add(t) From ca7cf044b1a86a2a47265748da51991ccd520b2c Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Mon, 28 Apr 2025 20:43:47 -0500 Subject: [PATCH 38/63] Add verison and count arguments. --- src/github_helper/_cli.py | 27 ++++++++++++++++++++++++++- src/github_helper/api/__init__.py | 2 ++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/github_helper/_cli.py b/src/github_helper/_cli.py index 1834d22..96dbe92 100644 --- a/src/github_helper/_cli.py +++ b/src/github_helper/_cli.py @@ -203,6 +203,25 @@ def _get_cli_args(): required=True, ) + group = audit_versions.add_mutually_exclusive_group() + + group.add_argument( + "-c", + "--count", + action="store", + type=int, + default=15, + help="How many versions to audit", + ) + group.add_argument( + "-v", + "--version", + action="store", + type=str, + default=None, + help="Audit a specific version in more detail", + ) + basic_args = parser.parse_args() return parser, vars(basic_args) @@ -229,6 +248,8 @@ async def _run_cli_async(): # noqa: C901, PLR0912, PLR0915 complex repo = cli_args.pop("repo", None) paginate = cli_args.pop("all", False) # Internamente en gh api es un --paginate command = cli_args.pop("command", None) + count = cli_args.pop("count", None) + version = cli_args.pop("version", None) cli_args.pop("log") cli_args.pop("human") testing = cli_args.pop("testing", False) @@ -273,7 +294,11 @@ async def _run_cli_async(): # noqa: C901, PLR0912, PLR0915 complex data, sadness = await gh.audit_rulesets(repo) data = await adpt.transform_audit_rulesets_data(data) case "audit-versions": - data, sadness = await gh.audit_versions(repo) + data, sadness = await gh.audit_versions( + repo, + count=count, + version=version, + ) data = await adpt.transform_audit_versions_data(data) case _: print("No command supplied.", file=sys.stderr) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index eff031d..ab3c78e 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -472,6 +472,7 @@ async def audit_versions( # noqa: C901 self, repo: str, count: int = 20, + version: str | None = None, ) -> RetVal[list[dict]]: """ Verify that version of a repository have differences. @@ -480,6 +481,7 @@ async def audit_versions( # noqa: C901 repo: the name of the repo to verify. Can be "owner/repo" or just "repo" and owner is assumed to be the current user. count: the number of versions to look at + version: deep dive on one version """ From 26ea2ee0a16a92b13eab63448ff9be353229bafd Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Mon, 28 Apr 2025 22:54:20 -0500 Subject: [PATCH 39/63] Refactor a bit --- src/github_helper/api/__init__.py | 41 +++++++++----------- src/github_helper/api/_pkg_audit/__init__.py | 12 ++++++ src/github_helper/api/versions.py | 6 --- 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index ab3c78e..40632eb 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -360,12 +360,6 @@ class Tag: """Return type for get_remote_tags.""" tag: str - _with_v: bool = False - - def tag_eq(self, cannon): - """Check if tag is equal to another tag.""" - return self.tag == (f"v{cannon}" if self._with_v else cannon) - # end async def get_remote_tags(self, repo, count=None) -> RetVal[list[Tag]]: @@ -378,7 +372,7 @@ async def get_remote_tags(self, repo, count=None) -> RetVal[list[Tag]]: retval, out, err = await srv.gh_api(endpoint) srv.check_retval(retval, err, endpoint=endpoint) tags_dict = tags_jq.input_value(orjson.loads(out)).first() - tags = [GHApi.Tag(**tag, _with_v=True) for tag in tags_dict] + tags = [GHApi.Tag(**tag) for tag in tags_dict] sadness = int(not tags) return tags[:count], sadness @@ -462,7 +456,6 @@ async def get_releases(self, repo: str) -> RetVal[list[Release]]: GHApi.Release( **r, version=versions.Version(r["tag"]), - _with_v=True, ) for r in releases ] @@ -488,6 +481,7 @@ async def audit_versions( # noqa: C901 # should this be a dictionary so we can iterate through names? @dataclass(slots=True) class VersionSet: + v: versions.Version gh_tags: GHApi.Tag | None = None gh_releases: GHApi.Release | None = None pypi: GHApi.Release | None = None @@ -495,7 +489,7 @@ class VersionSet: # maybe audits should carry their own adapters # this is an adapter, colors is an adapter - def print_source_status(self, name: str, canonical: str) -> str: + def print_attr_diff(self, name: str) -> str: attr = getattr(self, name) if not attr: return "" @@ -503,7 +497,7 @@ def print_source_status(self, name: str, canonical: str) -> str: empty = "" else: empty = f"{Fore.red}Yanked/Empty{Style.reset}" - if not attr.tag_eq(canonical): + if attr.tag.removeprefix("v") != str(v).removeprefix("v"): return f"{empty}{Fore.yellow}({attr.tag}){Style.reset}" elif empty: return f"{empty}" @@ -552,7 +546,7 @@ def print_source_status(self, name: str, canonical: str) -> str: v = getattr(o, "version", None) or versions.Version(o.tag) if not v.valid: continue - vset = all_versions.setdefault(v, VersionSet()) + vset = all_versions.setdefault(v, VersionSet(v)) if getattr(vset, attrname, None): warnings.warn( "Looks like conflicting poorly-written versions caused overwrite.", @@ -561,27 +555,28 @@ def print_source_status(self, name: str, canonical: str) -> str: if isinstance(o, GHApi.Release): o.audit = ReleaseAudit(o) setattr(vset, attrname, o) + + if version: + v = versions.Version(version) + r = all_versions.get(v) + if not r: + return [], 1 + all_versions = {v: r} + all_versions = dict(sorted(all_versions.items(), reverse=True)) result = [ { "version": str(v), - "gh_tags": r.print_source_status("gh_tags", str(v)), - "gh_releases": r.print_source_status("gh_releases", str(v)), - **( - {"pypi": r.print_source_status("pypi", str(v))} - if not pypi_sadness - else {} - ), - **( - {"test.pypi": r.print_source_status("test_pypi", str(v))} - if not test_pypi_sadness - else {} - ), + "gh_tags": r.print_attr_diff("gh_tags"), + "gh_releases": r.print_attr_diff("gh_releases"), + "pypi": r.print_attr_diff("pypi"), + "test.pypi": r.print_attr_diff("test_pypi"), "validity": v.kind, } for v, r in list(all_versions.items())[:count] # count ] + return result, 0 # maybe no sadness for audit? async def _get_ruleset(self, owner, repo, ruleset_id): diff --git a/src/github_helper/api/_pkg_audit/__init__.py b/src/github_helper/api/_pkg_audit/__init__.py index 740731e..32aeb4d 100644 --- a/src/github_helper/api/_pkg_audit/__init__.py +++ b/src/github_helper/api/_pkg_audit/__init__.py @@ -28,6 +28,18 @@ class ReleaseAudit: projects: dict """A list of the projects found in this release.""" + def full_print(self): + ret = "" + if self.unknown_files: + ret += "Unknown Files: \n " + ret += "\n ".join(self.unknown_files) + if self.ignored_files: + ret += "Ignored Files: \n" + for k, v in self.ignored_files: + ret += f" {k}:\n " + ret += "\n ".join(v) + return ret + def __init__(self, release): """Audit a release object.""" self.tag = release.tag diff --git a/src/github_helper/api/versions.py b/src/github_helper/api/versions.py index 8a587cf..396e029 100644 --- a/src/github_helper/api/versions.py +++ b/src/github_helper/api/versions.py @@ -72,10 +72,6 @@ def __init__(self, tag: str): object.__setattr__(self, "micro", int(micro_match.group(1))) object.__setattr__(self, "local", micro_match.group(2) or None) - def __str__(self): - """Return string rep of broken version.""" - return "BROKEN" - _VersionTypes = semver.Version | pyversion.Version | BadVersion @@ -223,8 +219,6 @@ def __repr__(self): """Print Version as python-compatible string if possible.""" if not self.valid: return "" - if self.kind == Version.Type.MALFORMED: - return f"{Fore.red}BROKEN{Style.reset}" post = f".post{self.post}" if self.post else "" dev = f".dev{self.dev}" if self.dev else "" pre = f"{self.pre[0]}{self.pre[1]}" if self.pre else "" From dc9fbbbf32c3023754fdc278d4da756e94600017 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Mon, 28 Apr 2025 23:29:00 -0500 Subject: [PATCH 40/63] Change names and add debugs --- src/github_helper/_cli.py | 2 +- src/github_helper/api/__init__.py | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/github_helper/_cli.py b/src/github_helper/_cli.py index 96dbe92..b1fe813 100644 --- a/src/github_helper/_cli.py +++ b/src/github_helper/_cli.py @@ -297,7 +297,7 @@ async def _run_cli_async(): # noqa: C901, PLR0912, PLR0915 complex data, sadness = await gh.audit_versions( repo, count=count, - version=version, + only_version=version, ) data = await adpt.transform_audit_versions_data(data) case _: diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 40632eb..895bbc2 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -448,8 +448,6 @@ async def get_releases(self, repo: str) -> RetVal[list[Release]]: retval, out, err = await srv.gh_api(endpoint) srv.check_retval(retval, err, endpoint=endpoint) obj = orjson.loads(out) - _logger.debug2("get_releases") - _log_one_json(obj) releases = releases_jq.input_value(obj).first() sadness = int(not releases) coerced_releases: list[GHApi.Release] = [ @@ -465,7 +463,7 @@ async def audit_versions( # noqa: C901 self, repo: str, count: int = 20, - version: str | None = None, + only_version: str | None = None, ) -> RetVal[list[dict]]: """ Verify that version of a repository have differences. @@ -474,7 +472,7 @@ async def audit_versions( # noqa: C901 repo: the name of the repo to verify. Can be "owner/repo" or just "repo" and owner is assumed to be the current user. count: the number of versions to look at - version: deep dive on one version + only_version: deep dive on one version """ @@ -544,6 +542,7 @@ def print_attr_diff(self, name: str) -> str: [], ): v = getattr(o, "version", None) or versions.Version(o.tag) + _logger.debug2(str(v)) if not v.valid: continue vset = all_versions.setdefault(v, VersionSet(v)) @@ -556,8 +555,8 @@ def print_attr_diff(self, name: str) -> str: o.audit = ReleaseAudit(o) setattr(vset, attrname, o) - if version: - v = versions.Version(version) + if only_version: + v = versions.Version(only_version) r = all_versions.get(v) if not r: return [], 1 From afedd767172b673caf8e76af8bf3136c7944d4eb Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Mon, 28 Apr 2025 23:31:34 -0500 Subject: [PATCH 41/63] Fix closure bug --- src/github_helper/api/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 895bbc2..c2dae2a 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -495,7 +495,10 @@ def print_attr_diff(self, name: str) -> str: empty = "" else: empty = f"{Fore.red}Yanked/Empty{Style.reset}" - if attr.tag.removeprefix("v") != str(v).removeprefix("v"): + if attr.tag.removeprefix("v") != str(self.v).removeprefix("v"): + _logger.debug2( + f"{attr.tag.removeprefix('v')}={str(v).removeprefix('v')}", + ) return f"{empty}{Fore.yellow}({attr.tag}){Style.reset}" elif empty: return f"{empty}" From 7cc068b97f6629e7a001ca96ce521db5b82e4f86 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Mon, 28 Apr 2025 23:31:48 -0500 Subject: [PATCH 42/63] Fix 0 falsy bug --- src/github_helper/api/versions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/github_helper/api/versions.py b/src/github_helper/api/versions.py index 396e029..a1cb8a9 100644 --- a/src/github_helper/api/versions.py +++ b/src/github_helper/api/versions.py @@ -219,10 +219,10 @@ def __repr__(self): """Print Version as python-compatible string if possible.""" if not self.valid: return "" - post = f".post{self.post}" if self.post else "" - dev = f".dev{self.dev}" if self.dev else "" - pre = f"{self.pre[0]}{self.pre[1]}" if self.pre else "" - local = f"+{self.local}" if self.local else "" + post = f".post{self.post}" if self.post is not None else "" + dev = f".dev{self.dev}" if self.dev is not None else "" + pre = f"{self.pre[0]}{self.pre[1]}" if self.pre is not None else "" + local = f"+{self.local}" if self.local is not None else "" return f"{self.major}.{self.minor}.{self.micro}{pre}{post}{dev}{local}" def _cmp_tuple(self) -> tuple: From bc400cf4e61cebba21ee3807f51e1377995de09e Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Tue, 29 Apr 2025 07:39:26 -0500 Subject: [PATCH 43/63] Move version calculation to post_init --- src/github_helper/api/__init__.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index c2dae2a..4d4f612 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -360,7 +360,14 @@ class Tag: """Return type for get_remote_tags.""" tag: str - # end + """Tag""" + + version: versions.Version = None + """Calculated version.""" + + def __post_init__(self): + """Initialize derivative values.""" + self.version = versions.Version(self.tag) async def get_remote_tags(self, repo, count=None) -> RetVal[list[Tag]]: """Return tags ("tag":"name") for a repo.""" @@ -384,8 +391,6 @@ class Release(Tag): """Does the source mark it as prerelease?""" files: list[str] """Files that came with it.""" - version: versions.Version - """Calculated version.""" audit: ReleaseAudit | None = None async def get_pypi( @@ -422,8 +427,7 @@ async def get_pypi( coerced_releases: list[GHApi.Release] = [ GHApi.Release( **r, - version=(v := versions.Version(r["tag"])), - prerelease=v.is_prerelease, + prerelease=versions.Version(r["tag"]).is_prerelease, ) for r in releases ] @@ -453,7 +457,6 @@ async def get_releases(self, repo: str) -> RetVal[list[Release]]: coerced_releases: list[GHApi.Release] = [ GHApi.Release( **r, - version=versions.Version(r["tag"]), ) for r in releases ] From 836d121337ac3ef1b98bcdf61c744023383cbd09 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Tue, 29 Apr 2025 08:10:20 -0500 Subject: [PATCH 44/63] Refactor printing a bit --- src/github_helper/api/__init__.py | 85 ++++++++++++++++++------------- 1 file changed, 49 insertions(+), 36 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 4d4f612..98a4711 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -369,6 +369,18 @@ def __post_init__(self): """Initialize derivative values.""" self.version = versions.Version(self.tag) + def tag_diff(self) -> str: + """Check if our version tag correctly formatted.""" + # NOTE: this is biased towards python, semver is different! + if self.tag.removeprefix("v") != str(self.version).removeprefix("v"): + return f"{Fore.yellow}({self.tag}){Style.reset}" + return "" + + def return_status(self) -> list[str]: + """Return all printed status in array.""" + td = self.tag_diff() + return [td] if td else [] + async def get_remote_tags(self, repo, count=None) -> RetVal[list[Tag]]: """Return tags ("tag":"name") for a repo.""" _ = await self.get_user() @@ -393,6 +405,20 @@ class Release(Tag): """Files that came with it.""" audit: ReleaseAudit | None = None + def is_empty(self) -> str: + """Check if our release empty.""" + if not self.files: + return f"{Fore.red}Yanked/Empty{Style.reset}" + return "" + + def return_status(self) -> list[str]: + """Return all printed status in array.""" + # can use super() once not subclass + ret = GHApi.Tag.return_status(self) + if ie := self.is_empty(): + ret.append(ie) + return ret + async def get_pypi( self, project_name: str, @@ -478,36 +504,6 @@ async def audit_versions( # noqa: C901 only_version: deep dive on one version """ - - # should this be a dictionary so we can iterate through names? - @dataclass(slots=True) - class VersionSet: - v: versions.Version - gh_tags: GHApi.Tag | None = None - gh_releases: GHApi.Release | None = None - pypi: GHApi.Release | None = None - test_pypi: GHApi.Release | None = None - - # maybe audits should carry their own adapters - # this is an adapter, colors is an adapter - def print_attr_diff(self, name: str) -> str: - attr = getattr(self, name) - if not attr: - return "" - if not isinstance(attr, GHApi.Release) or attr.files: - empty = "" - else: - empty = f"{Fore.red}Yanked/Empty{Style.reset}" - if attr.tag.removeprefix("v") != str(self.v).removeprefix("v"): - _logger.debug2( - f"{attr.tag.removeprefix('v')}={str(v).removeprefix('v')}", - ) - return f"{empty}{Fore.yellow}({attr.tag}){Style.reset}" - elif empty: - return f"{empty}" - else: - return f"{Fore.green}True{Style.reset}" - project_configs, sadness = await self.get_project_configs( repo, "pyproject.toml", @@ -535,6 +531,13 @@ def print_attr_diff(self, name: str) -> str: self.get_pypi(project_names[0], testing=True), ) + @dataclass(slots=True) + class VersionSet: + gh_tags: GHApi.Tag | None = None + gh_releases: GHApi.Release | None = None + pypi: GHApi.Release | None = None + test_pypi: GHApi.Release | None = None + all_versions: dict[ versions.Version, VersionSet, @@ -551,7 +554,7 @@ def print_attr_diff(self, name: str) -> str: _logger.debug2(str(v)) if not v.valid: continue - vset = all_versions.setdefault(v, VersionSet(v)) + vset = all_versions.setdefault(v, VersionSet()) if getattr(vset, attrname, None): warnings.warn( "Looks like conflicting poorly-written versions caused overwrite.", @@ -569,14 +572,24 @@ def print_attr_diff(self, name: str) -> str: all_versions = {v: r} all_versions = dict(sorted(all_versions.items(), reverse=True)) - + ok = f"{Fore.green}OK{Style.reset}" result = [ { "version": str(v), - "gh_tags": r.print_attr_diff("gh_tags"), - "gh_releases": r.print_attr_diff("gh_releases"), - "pypi": r.print_attr_diff("pypi"), - "test.pypi": r.print_attr_diff("test_pypi"), + "gh_tags": ( + (", ".join(r.gh_tags.return_status()) or ok) if r.gh_tags else "" + ), + "gh_releases": ( + (", ".join(r.gh_releases.return_status()) or ok) + if r.gh_releases + else "" + ), + "pypi": ((", ".join(r.pypi.return_status()) or ok) if r.pypi else ""), + "test.pypi": ( + (", ".join(r.test_pypi.return_status()) or ok) + if r.test_pypi + else "" + ), "validity": v.kind, } for v, r in list(all_versions.items())[:count] # count From c3d7a6c605e7d466b4bfad857f49b826569507a1 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Tue, 29 Apr 2025 08:17:11 -0500 Subject: [PATCH 45/63] Remove loud debugs --- src/github_helper/api/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 98a4711..dafb210 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -551,7 +551,6 @@ class VersionSet: [], ): v = getattr(o, "version", None) or versions.Version(o.tag) - _logger.debug2(str(v)) if not v.valid: continue vset = all_versions.setdefault(v, VersionSet()) From 49428324e2d084e84e02e26f91eeb528f4651d55 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Tue, 29 Apr 2025 16:24:11 -0500 Subject: [PATCH 46/63] Remove notes file --- src/github_helper/api/_pkg_audit/note | 121 -------------------------- 1 file changed, 121 deletions(-) delete mode 100644 src/github_helper/api/_pkg_audit/note diff --git a/src/github_helper/api/_pkg_audit/note b/src/github_helper/api/_pkg_audit/note deleted file mode 100644 index 9d32ed7..0000000 --- a/src/github_helper/api/_pkg_audit/note +++ /dev/null @@ -1,121 +0,0 @@ -def __repr__(self): - """Return a summary of the audit.""" - ignore_len = sum(self.ignore_counter.values()) - project_tag_count = [ - f"{k}: {len(v['all_tags'])}" for k, v in self.projects.items() - ] - - return ( - f"pre-agree: {self.prerelease_agree}; " - f"projects: {', '.join(project_tag_count)}; " - f"{len(self.file_notes)} files w/ notes; " - f"{len(self.unknown_files)} unknown files; " - f"{ignore_len} ignored files." - ) - -def compare_audits(v, **audits): # noqa: C901 - """Produce a diff between several audits.""" - all_tags = set() - for a in audits.values(): - audit = a.get(v, {}).get("audit", None) - if not audit: - continue - for project in audit.projects.values(): - all_tags.update(project.get("all_tags", {})) - results = {} - - # this inversion sucks - # what to do if project is missing - for tag in all_tags: - results[tag] = set() - for name, a in audits.items(): - audit = a.get(v, {}).get("audit", None) - if not audit: - results[tag].add(name) - continue - for project in audit.projects.values(): - if tag in project.get("all_tags", {}): - break - else: - results[tag].add(name) - if not results[tag]: - del results[tag] - output = "" - for tag, problems in results.items(): - output += f"{tag}: {', '.join(problems)}\n" - - return output - - - def __str__(self): # noqa: C901, PLR0912 - """Return a string diff of a python project.""" - ret = "" - if not self.prerelease_agree: - ret += f"{Fore.red}PRERELEASE DISAGREEMENT{Style.reset}\n" - if not self.projects: - ret += f"{Fore.red}No Valid Projects Found.{Style.reset}\n" - else: - for k, v in self.projects.items(): - if k.startswith("python/"): - ## look at bdist/sdist - warn = "" - if not v["bdist"]: - warn = "bdist" - if not v["sdist"]: - warn = ", sdist" if warn else "sdist" - warn = f", {Fore.red}missing {warn}{Style.reset}" if warn else "" - - pure = "" - if v["pure"]: - pure = ( - f", {Fore.green}pure: {', '.join(v['pure'])}{Style.reset}" - ) - - ret += f"{Style.bold}{k}{Style.reset}{warn}{pure}\n" - - ## look at tags - ret += ( - "\n".join(self._build_tree_str(v["bdist-tree"])) + "\n" - if v["bdist-tree"] - else "" - ) - - if v["unknown-tags"]: - ret += " Unknown Tags:\n " - ret += "\n ".join(v["unknown-tags"]) + "\n" - if self.unknown_files: - ret += ( - f"{Fore.yellow}{Style.bold}" - f"{len(self.unknown_files)} Unknown Files:" - f"{Style.reset}\n" - ) - for i, f in enumerate(self.unknown_files): - if i > 3: # noqa: PLR2004 - ret += f" {Fore.yellow}...{Style.reset}\n" - break - ret += f" {Fore.yellow}{f}{Style.reset}\n" - if self.ignore_counter: - ret += "ignored: " - for k, v in self.ignore_counter.items(): - ret += f"{k} {v} time(s)" - ret += "\n" - return ret - - - def _build_tree_str(self, obj, indent="", *, is_last=True): - lines = [] - - if isinstance(obj, dict): - items = list(obj.items()) - for i, (key, value) in enumerate(items): - is_last = i == len(items) - 1 - branch = "`-- " if is_last else "|-- " - next_indent = indent + (" " if is_last else "| ") - lines.append(f"{indent}{branch}{key}") - if not value: - lines[-1] += f" {Fore.red}missing{Style.reset}" - elif isinstance(value, dict): - lines.extend(self._build_tree_str(value, next_indent)) - elif isinstance(value, (list, tuple)): - lines[-1] += f" {Fore.green}>> {', '.join(value)}{Style.reset}" - return lines From 85236b627e81c690213caf8afca2b64c9e57b836 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Tue, 29 Apr 2025 16:39:47 -0500 Subject: [PATCH 47/63] Specify None to avoid 0 false positives --- src/github_helper/api/__init__.py | 1 - src/github_helper/api/versions.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index dafb210..83be05b 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -569,7 +569,6 @@ class VersionSet: if not r: return [], 1 all_versions = {v: r} - all_versions = dict(sorted(all_versions.items(), reverse=True)) ok = f"{Fore.green}OK{Style.reset}" result = [ diff --git a/src/github_helper/api/versions.py b/src/github_helper/api/versions.py index a1cb8a9..f347244 100644 --- a/src/github_helper/api/versions.py +++ b/src/github_helper/api/versions.py @@ -274,7 +274,7 @@ def _compare( # noqa: C901, PLR0911, PLR0912 return 1 # final > pre if self.pre is not None and other.pre is None: return -1 # pre < final - if self.pre and other.pre: + if self.pre is not None and other.pre is not None: if self.pre[0] != other.pre[0]: return (self.pre[0] > other.pre[0]) - (self.pre[0] < other.pre[0]) if self.pre[1] != other.pre[1]: @@ -285,7 +285,7 @@ def _compare( # noqa: C901, PLR0911, PLR0912 return -1 # no post < post if self.post is not None and other.post is None: return 1 # post > no post - if self.post and other.post: # noqa: SIM102 clarity + if self.post is not None and other.post is not None: # noqa: SIM102 clarity if self.post != other.post: return (self.post > other.post) - (self.post < other.post) @@ -294,7 +294,7 @@ def _compare( # noqa: C901, PLR0911, PLR0912 return 1 # no dev > dev if self.dev is not None and other.dev is None: return -1 # dev < no dev - if self.dev and other.dev: # noqa: SIM102 clarity + if self.dev is not None and other.dev is not None: # noqa: SIM102 clarity if self.dev != other.dev: return (self.dev > other.dev) - (self.dev < other.dev) From 82bb1eaf2ae935ad2a922f46cc0a73b96e3e5e34 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Tue, 29 Apr 2025 16:55:47 -0500 Subject: [PATCH 48/63] Allow async noops --- src/github_helper/api/__init__.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 83be05b..78f8603 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -40,6 +40,15 @@ def __getattr__(self, name): # Override colored's foreground, background, and style Fore = Style = _NoColor() # type: ignore[misc, assignment] + +async def _noop(tup=0, ret=None): + ### Allows us to fake async noops + if not tup: + return ret + else: + return (ret,) * tup + + _check_ran = False @@ -339,7 +348,7 @@ async def get_project_configs( ref = "main" if "main" in await r.list_branches() else "master" files = [] for name in filenames: - files.extend(await r.get_files_by_name(name, ref=ref)) + files.extend(await r.get_files_by_name(name, ref=ref) or []) configs: GHApi.ConfigSet = {} for f in files: obj = None @@ -527,8 +536,12 @@ async def audit_versions( # noqa: C901 ) = await asyncio.gather( self.get_remote_tags(repo), self.get_releases(repo), - self.get_pypi(project_names[0]), - self.get_pypi(project_names[0], testing=True), + (self.get_pypi(project_names[0]) if project_names else _noop(2, [])), + ( + self.get_pypi(project_names[0], testing=True) + if project_names + else _noop(2, []) + ), ) @dataclass(slots=True) From 876018d58f0ca2089e27a8efd249420325606540 Mon Sep 17 00:00:00 2001 From: David Angarita Date: Tue, 29 Apr 2025 17:47:16 -0500 Subject: [PATCH 49/63] Add new test for audit-versions --- integration_test/{test.sh => test.es.sh} | 19 +++++++++++++++---- pyproject.toml | 3 +++ 2 files changed, 18 insertions(+), 4 deletions(-) rename integration_test/{test.sh => test.es.sh} (80%) diff --git a/integration_test/test.sh b/integration_test/test.es.sh similarity index 80% rename from integration_test/test.sh rename to integration_test/test.es.sh index a571973..001f990 100644 --- a/integration_test/test.sh +++ b/integration_test/test.es.sh @@ -10,6 +10,7 @@ REPO="$1" total=0 success=0 fail=0 +failed_cmds=() print_and_run() { echo -e "\n==============================" @@ -19,7 +20,8 @@ print_and_run() { ((success++)) else ((fail++)) - echo -e "\033[0;31m❌ Falló el commando:\033[0m" + failed_cmds+=("$*") + echo -e "\033[0;31m❌ Falló el comando:\033[0m" echo -e "\033[0;31m$*\033[0m" fi } @@ -66,6 +68,8 @@ run_pretty_commands() { for cmd in "${repo_cmds[@]}"; do print_and_run "$base_cmd $cmd" done + + print_and_run "$base_cmd audit-versions -r $REPO -c 3" } run_pretty_json_commands() { @@ -96,6 +100,13 @@ run_pretty_json_commands echo -e "\n==============================" echo -e "Resumen de ejecución:" -printf "Total de commandos : %d\n" "$total" -printf "Commandos exitosos : \033[0;32m%d\033[0m\n" "$success" -printf "Commandos fallidos : \033[0;31m%d\033[0m\n" "$fail" +printf "Total de comandos : %d\n" "$total" +printf "Comandos exitosos : \033[0;32m%d\033[0m\n" "$success" +printf "Comandos fallidos : \033[0;31m%d\033[0m\n" "$fail" + +if (( fail > 0 )); then + echo -e "\n\033[0;31m❌ Lista de comandos fallidos:\033[0m" + for cmd in "${failed_cmds[@]}"; do + echo -e "\033[0;31m$cmd\033[0m" + done +fi diff --git a/pyproject.toml b/pyproject.toml index 8292b15..9b16c20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,3 +101,6 @@ help = "Run test by test, slowly, quitting after first error" [tool.poe.tasks.filter-test] cmd = "pytest -W error -vvvx -rA" help = "Run any/all tests one by one with basic settings: can include filename and -k filters" + +[tool.typos] +files.extend-exclude = ["*.es.*"] From 14ffd3e12611a4312f970fe97f397559f89b06bd Mon Sep 17 00:00:00 2001 From: David Angarita Date: Tue, 29 Apr 2025 17:49:42 -0500 Subject: [PATCH 50/63] Remove audit-releases command --- src/github_helper/_adapters/api_to_cli.py | 6 ++---- src/github_helper/_cli.py | 15 --------------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/src/github_helper/_adapters/api_to_cli.py b/src/github_helper/_adapters/api_to_cli.py index fac6848..9ef2f29 100644 --- a/src/github_helper/_adapters/api_to_cli.py +++ b/src/github_helper/_adapters/api_to_cli.py @@ -14,10 +14,8 @@ class GHAdapter: def _check_options(self, formatters): formats = {k for k, v in formatters.items() if v} invalid_formats = {"html", "url"} & formats - if ( - (self._command == "auth-status" and formats) - or (self._command != "repos" and invalid_formats) - or (self._command == "audit-releases" and self._json) + if (self._command == "auth-status" and formats) or ( + self._command != "repos" and invalid_formats ): raise NotImplementedError( f"{', '.join(invalid_formats)} not valid flags for {self._command}", diff --git a/src/github_helper/_cli.py b/src/github_helper/_cli.py index b1fe813..299a64b 100644 --- a/src/github_helper/_cli.py +++ b/src/github_helper/_cli.py @@ -168,18 +168,6 @@ def _get_cli_args(): help="Use testing pypi, not regular.", ) - releases_audit_parser = subparsers.add_parser( - "audit-releases", - description="Show all releases from a repo with comments.", - help="Return all releases of a repo with comments.", - ) - releases_audit_parser.add_argument( - "-r", - "--repo", - help="Name of repository required.", - required=True, - ) - audit_repo = subparsers.add_parser( "audit-repo", description="", @@ -287,9 +275,6 @@ async def _run_cli_async(): # noqa: C901, PLR0912, PLR0915 complex case "audit-pypi": data, sadness = await gh.audit_pypi(repo, testing=testing) data = await adpt.transform_audit_releases_data(data) - case "audit-releases": - data, sadness = await gh.audit_releases(repo) - data = await adpt.transform_audit_releases_data(data) case "audit-repo": data, sadness = await gh.audit_rulesets(repo) data = await adpt.transform_audit_rulesets_data(data) From 3dfadb169f34ec78953366ddb0012d5fede7913a Mon Sep 17 00:00:00 2001 From: David Angarita Date: Tue, 29 Apr 2025 17:50:12 -0500 Subject: [PATCH 51/63] Remove audit-releases from test --- integration_test/test.es.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/integration_test/test.es.sh b/integration_test/test.es.sh index 001f990..a1e0592 100644 --- a/integration_test/test.es.sh +++ b/integration_test/test.es.sh @@ -38,7 +38,6 @@ run_basic_commands() { "project-configs -r $REPO" "releases -r $REPO" "pypi -r $REPO" - "audit-releases -r $REPO" "audit-repo -r $REPO" "audit-versions -r $REPO" ) @@ -60,7 +59,6 @@ run_pretty_commands() { "project-configs -r $REPO" "releases -r $REPO" "pypi -r $REPO" - "audit-releases -r $REPO" "audit-repo -r $REPO" "audit-versions -r $REPO" ) @@ -84,7 +82,6 @@ run_pretty_json_commands() { "project-configs -r $REPO" "releases -r $REPO" "pypi -r $REPO" - "audit-releases -r $REPO" "audit-repo -r $REPO" "audit-versions -r $REPO" ) From a4bf54efc4af1c8b27ab4b91d4689de27b0f4300 Mon Sep 17 00:00:00 2001 From: David Angarita Date: Tue, 29 Apr 2025 17:59:26 -0500 Subject: [PATCH 52/63] Remove audit-pypi command --- src/github_helper/_cli.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/github_helper/_cli.py b/src/github_helper/_cli.py index 299a64b..3cff646 100644 --- a/src/github_helper/_cli.py +++ b/src/github_helper/_cli.py @@ -150,24 +150,6 @@ def _get_cli_args(): help="Use testing pypi, not regular.", ) - pypi_audit_parser = subparsers.add_parser( - "audit-pypi", - description="Show all pypi packages from a repo with comments.", - help="Return all pypi releases of a repo.", - ) - pypi_audit_parser.add_argument( - "-r", - "--repo", - help="Name of repository required.", - required=True, - ) - pypi_audit_parser.add_argument( - "-t", - "--testing", - action="store_true", - help="Use testing pypi, not regular.", - ) - audit_repo = subparsers.add_parser( "audit-repo", description="", @@ -272,9 +254,6 @@ async def _run_cli_async(): # noqa: C901, PLR0912, PLR0915 complex case "pypi": data, sadness = await gh.get_pypi(repo, testing=testing) data = await adpt.transform_pypi_data(data) - case "audit-pypi": - data, sadness = await gh.audit_pypi(repo, testing=testing) - data = await adpt.transform_audit_releases_data(data) case "audit-repo": data, sadness = await gh.audit_rulesets(repo) data = await adpt.transform_audit_rulesets_data(data) From 624bb939148b1a27b6173ef1e62a3e6638befdc2 Mon Sep 17 00:00:00 2001 From: David Angarita Date: Tue, 29 Apr 2025 19:37:54 -0500 Subject: [PATCH 53/63] Add magic method json to Version class --- src/github_helper/api/versions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/github_helper/api/versions.py b/src/github_helper/api/versions.py index f347244..3b3b3d0 100644 --- a/src/github_helper/api/versions.py +++ b/src/github_helper/api/versions.py @@ -211,6 +211,10 @@ def _enumerate_version(self, v: _VersionTypes) -> None: (v.is_prerelease if hasattr(v, "is_prerelease") else bool(v.prerelease)), ) + def __json__(self): + """Convert to json.""" + return self.__str__() + def __str__(self): """Print Version as string.""" return self.__repr__() From 415711cd90a8b140e975b10a08ed312ee233344b Mon Sep 17 00:00:00 2001 From: David Angarita Date: Tue, 29 Apr 2025 19:39:48 -0500 Subject: [PATCH 54/63] Add method json to Tag class --- src/github_helper/api/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 78f8603..b8792f3 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -371,7 +371,7 @@ class Tag: tag: str """Tag""" - version: versions.Version = None + version: versions.Version | None = None """Calculated version.""" def __post_init__(self): @@ -390,6 +390,13 @@ def return_status(self) -> list[str]: td = self.tag_diff() return [td] if td else [] + def __json__(self): + """Convert to json.""" + return { + "tag": self.tag, + "version": self.version.__json__() if self.version else None, + } + async def get_remote_tags(self, repo, count=None) -> RetVal[list[Tag]]: """Return tags ("tag":"name") for a repo.""" _ = await self.get_user() From 29545ab1d9f1b683eb253c1be812980d2b8fd444 Mon Sep 17 00:00:00 2001 From: David Angarita Date: Tue, 29 Apr 2025 19:40:21 -0500 Subject: [PATCH 55/63] Add method json to Release class --- src/github_helper/api/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index b8792f3..ea458ed 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -435,6 +435,17 @@ def return_status(self) -> list[str]: ret.append(ie) return ret + def __json__(self): + """Convert to json.""" + old = GHApi.Tag.__json__(self) + old.update( + { + "prerelease": self.prerelease, + "files": self.files, + }, + ) + return old + async def get_pypi( self, project_name: str, From b856fd4563e597f8a4e75547e76f416fff623611 Mon Sep 17 00:00:00 2001 From: David Angarita Date: Wed, 30 Apr 2025 11:21:53 -0500 Subject: [PATCH 56/63] Remove project-config from integration tests --- integration_test/test.es.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/integration_test/test.es.sh b/integration_test/test.es.sh index a1e0592..0f5a32a 100644 --- a/integration_test/test.es.sh +++ b/integration_test/test.es.sh @@ -35,7 +35,6 @@ run_basic_commands() { local repo_cmds=( "tags -r $REPO" - "project-configs -r $REPO" "releases -r $REPO" "pypi -r $REPO" "audit-repo -r $REPO" @@ -56,7 +55,6 @@ run_pretty_commands() { local repo_cmds=( "tags -r $REPO" - "project-configs -r $REPO" "releases -r $REPO" "pypi -r $REPO" "audit-repo -r $REPO" @@ -79,7 +77,6 @@ run_pretty_json_commands() { local repo_cmds=( "tags -r $REPO" - "project-configs -r $REPO" "releases -r $REPO" "pypi -r $REPO" "audit-repo -r $REPO" From 85624109e7e5c3fdbc4d4dfb86a6a8f9d6ab7181 Mon Sep 17 00:00:00 2001 From: David Angarita Date: Wed, 30 Apr 2025 11:51:41 -0500 Subject: [PATCH 57/63] Add new command called filename --- src/github_helper/_cli.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/github_helper/_cli.py b/src/github_helper/_cli.py index 3cff646..babda17 100644 --- a/src/github_helper/_cli.py +++ b/src/github_helper/_cli.py @@ -119,6 +119,12 @@ def _get_cli_args(): help="Name of repository required.", required=True, ) + configs_parser.add_argument( + "-f", + "--filename", + help="Name of the file required.", + required=True, + ) releases_parser = subparsers.add_parser( "releases", @@ -216,6 +222,7 @@ def run_cli(): async def _run_cli_async(): # noqa: C901, PLR0912, PLR0915 complex parser, cli_args = _get_cli_args() repo = cli_args.pop("repo", None) + filename = cli_args.pop("filename", None) paginate = cli_args.pop("all", False) # Internamente en gh api es un --paginate command = cli_args.pop("command", None) count = cli_args.pop("count", None) @@ -246,7 +253,7 @@ async def _run_cli_async(): # noqa: C901, PLR0912, PLR0915 complex data, sadness = await gh.get_remote_tags(repo) data = await adpt.transform_tags_data(data) case "project-configs": - data, sadness = await gh.get_project_configs(repo) + data, sadness = await gh.get_project_configs(repo, filename) data = await adpt.transform_project_configs_data(data) case "releases": data, sadness = await gh.get_releases(repo) From c7a4cd852f98792500a9edf35d713f8b6c057334 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Wed, 30 Apr 2025 12:57:37 -0500 Subject: [PATCH 58/63] Start basic audit status --- src/github_helper/api/__init__.py | 11 ++++++++++- src/github_helper/api/_pkg_audit/__init__.py | 12 +++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index ea458ed..a32646a 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -446,6 +446,12 @@ def __json__(self): ) return old + def audit_status(self) -> list[str]: + """Get the audit status of the object.""" + if self.audit: + return "\n" + self.audit.status() + return "" + async def get_pypi( self, project_name: str, @@ -609,7 +615,10 @@ class VersionSet: (", ".join(r.gh_tags.return_status()) or ok) if r.gh_tags else "" ), "gh_releases": ( - (", ".join(r.gh_releases.return_status()) or ok) + ( + (", ".join(r.gh_releases.return_status()) or ok) + + r.gh_releases.audit_status() + ) if r.gh_releases else "" ), diff --git a/src/github_helper/api/_pkg_audit/__init__.py b/src/github_helper/api/_pkg_audit/__init__.py index 32aeb4d..891ff6e 100644 --- a/src/github_helper/api/_pkg_audit/__init__.py +++ b/src/github_helper/api/_pkg_audit/__init__.py @@ -28,16 +28,18 @@ class ReleaseAudit: projects: dict """A list of the projects found in this release.""" - def full_print(self): + def status(self): ret = "" + if not self.prerelease_agree: + ret += "Prerelease Disagreement.\n" if self.unknown_files: ret += "Unknown Files: \n " ret += "\n ".join(self.unknown_files) + ret += "\n" if self.ignored_files: - ret += "Ignored Files: \n" - for k, v in self.ignored_files: - ret += f" {k}:\n " - ret += "\n ".join(v) + ret += "Ignored Files:\n" + for k, v in self.ignored_files.items(): + ret += f" {k}: {v!s}\n" return ret def __init__(self, release): From 48041ded1f7deaf98b482cde4a70b04cc301136c Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Wed, 30 Apr 2025 12:58:33 -0500 Subject: [PATCH 59/63] Format status output --- src/github_helper/api/_pkg_audit/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/github_helper/api/_pkg_audit/__init__.py b/src/github_helper/api/_pkg_audit/__init__.py index 891ff6e..7cdc14b 100644 --- a/src/github_helper/api/_pkg_audit/__init__.py +++ b/src/github_helper/api/_pkg_audit/__init__.py @@ -34,7 +34,7 @@ def status(self): ret += "Prerelease Disagreement.\n" if self.unknown_files: ret += "Unknown Files: \n " - ret += "\n ".join(self.unknown_files) + ret += "\n".join(self.unknown_files) ret += "\n" if self.ignored_files: ret += "Ignored Files:\n" From a75813ad072e17d78d8568329a334b0349b9200f Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Wed, 30 Apr 2025 13:29:37 -0500 Subject: [PATCH 60/63] Fix type errors --- src/github_helper/_adapters/to_html/_components.py | 6 ++++-- src/github_helper/api/__init__.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/github_helper/_adapters/to_html/_components.py b/src/github_helper/_adapters/to_html/_components.py index 468e518..0c2b676 100644 --- a/src/github_helper/_adapters/to_html/_components.py +++ b/src/github_helper/_adapters/to_html/_components.py @@ -1,3 +1,5 @@ +from collections.abc import Iterable + import logistro from htmy import Component, Renderer, core, html @@ -31,8 +33,8 @@ def modal( async def render_page( - heads: core.BaseTag, - content: core.BaseTag, + heads: Iterable[core.BaseTag], + content: Iterable[core.BaseTag], ): tailwindcss_cdn = "https://cdn.tailwindcss.com" _logger.debug("Rendering.") diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index a32646a..46780d1 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -346,7 +346,7 @@ async def get_project_configs( ) if not ref: ref = "main" if "main" in await r.list_branches() else "master" - files = [] + files: list[dict] = [] for name in filenames: files.extend(await r.get_files_by_name(name, ref=ref) or []) configs: GHApi.ConfigSet = {} @@ -446,7 +446,7 @@ def __json__(self): ) return old - def audit_status(self) -> list[str]: + def audit_status(self) -> str: """Get the audit status of the object.""" if self.audit: return "\n" + self.audit.status() From f3ca63d604696ffd3dec43cb5aab5eed43192c82 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Thu, 1 May 2025 21:50:40 -0500 Subject: [PATCH 61/63] Implement json for audit --- src/github_helper/api/__init__.py | 58 +++---------------- src/github_helper/api/_pkg_audit/__init__.py | 40 +++++++------ src/github_helper/api/_pkg_audit/_py_audit.py | 21 ++++++- 3 files changed, 50 insertions(+), 69 deletions(-) diff --git a/src/github_helper/api/__init__.py b/src/github_helper/api/__init__.py index 46780d1..6ea66e3 100644 --- a/src/github_helper/api/__init__.py +++ b/src/github_helper/api/__init__.py @@ -378,17 +378,10 @@ def __post_init__(self): """Initialize derivative values.""" self.version = versions.Version(self.tag) - def tag_diff(self) -> str: + def tag_diff(self) -> bool: """Check if our version tag correctly formatted.""" # NOTE: this is biased towards python, semver is different! - if self.tag.removeprefix("v") != str(self.version).removeprefix("v"): - return f"{Fore.yellow}({self.tag}){Style.reset}" - return "" - - def return_status(self) -> list[str]: - """Return all printed status in array.""" - td = self.tag_diff() - return [td] if td else [] + return self.tag.removeprefix("v") != str(self.version).removeprefix("v") def __json__(self): """Convert to json.""" @@ -421,20 +414,6 @@ class Release(Tag): """Files that came with it.""" audit: ReleaseAudit | None = None - def is_empty(self) -> str: - """Check if our release empty.""" - if not self.files: - return f"{Fore.red}Yanked/Empty{Style.reset}" - return "" - - def return_status(self) -> list[str]: - """Return all printed status in array.""" - # can use super() once not subclass - ret = GHApi.Tag.return_status(self) - if ie := self.is_empty(): - ret.append(ie) - return ret - def __json__(self): """Convert to json.""" old = GHApi.Tag.__json__(self) @@ -442,16 +421,11 @@ def __json__(self): { "prerelease": self.prerelease, "files": self.files, + "audit": self.audit.__json__() if self.audit else None, }, ) return old - def audit_status(self) -> str: - """Get the audit status of the object.""" - if self.audit: - return "\n" + self.audit.status() - return "" - async def get_pypi( self, project_name: str, @@ -607,28 +581,14 @@ class VersionSet: return [], 1 all_versions = {v: r} all_versions = dict(sorted(all_versions.items(), reverse=True)) - ok = f"{Fore.green}OK{Style.reset}" result = [ { - "version": str(v), - "gh_tags": ( - (", ".join(r.gh_tags.return_status()) or ok) if r.gh_tags else "" - ), - "gh_releases": ( - ( - (", ".join(r.gh_releases.return_status()) or ok) - + r.gh_releases.audit_status() - ) - if r.gh_releases - else "" - ), - "pypi": ((", ".join(r.pypi.return_status()) or ok) if r.pypi else ""), - "test.pypi": ( - (", ".join(r.test_pypi.return_status()) or ok) - if r.test_pypi - else "" - ), - "validity": v.kind, + "version": v.__json__(), + "gh_tags": r.gh_tags.__json__() if r.gh_tags else None, + "gh_releases": (r.gh_releases.__json__() if r.gh_releases else None), + "pypi": r.pypi.__json__() if r.pypi else None, + "test.pypi": (r.test_pypi.__json__() if r.test_pypi else None), + "validity": str(v.kind), } for v, r in list(all_versions.items())[:count] # count ] diff --git a/src/github_helper/api/_pkg_audit/__init__.py b/src/github_helper/api/_pkg_audit/__init__.py index 7cdc14b..d91c811 100644 --- a/src/github_helper/api/_pkg_audit/__init__.py +++ b/src/github_helper/api/_pkg_audit/__init__.py @@ -19,28 +19,18 @@ class ReleaseAudit: understand them. """ - prerelease_agree: bool - """Does the version agree with the mark about prerelease.""" + tag: str + """The original tag.""" unknown_files: set """Files that couldn't be understood trying to calculate notes.""" ignored_files: dict[str, int] - - projects: dict - """A list of the projects found in this release.""" - - def status(self): - ret = "" - if not self.prerelease_agree: - ret += "Prerelease Disagreement.\n" - if self.unknown_files: - ret += "Unknown Files: \n " - ret += "\n".join(self.unknown_files) - ret += "\n" - if self.ignored_files: - ret += "Ignored Files:\n" - for k, v in self.ignored_files.items(): - ret += f" {k}: {v!s}\n" - return ret + """Files that we don't care about .""" + prerelease_agree: bool + """Does the version agree with the mark about prerelease.""" + version: Version + """The parsed Version.""" + python_audit: PythonAudit + """A python audit.""" def __init__(self, release): """Audit a release object.""" @@ -77,3 +67,15 @@ def process_file(self, filename: str) -> ReturnMessages: elif note := self.python_audit.check_file(filename): return note return {"error": "unrecognized name", "value": filename} + + def __json__(self): + return { + "tag": self.tag, + "version": self.version.__json__(), + "prerelease_agree": self.prerelease_agree, + "ignored_files": self.ignored_files, + "unknown_files": list(self.unknown_files), + "audits": { + "python": self.python_audit.__json__(), + }, + } diff --git a/src/github_helper/api/_pkg_audit/_py_audit.py b/src/github_helper/api/_pkg_audit/_py_audit.py index 765ba2e..302960c 100644 --- a/src/github_helper/api/_pkg_audit/_py_audit.py +++ b/src/github_helper/api/_pkg_audit/_py_audit.py @@ -4,7 +4,6 @@ from typing import Literal import logistro -from colored import Fore, Style from packaging import tags, utils from github_helper.api.versions import Version @@ -89,6 +88,9 @@ def check_file(self, filename) -> Audit | Error | None: return {"name": name, "action": "resolved"} return None + def __json__(self): + return {k: v.__json__() for k, v in self.projects.items()} + @dataclass(slots=True, kw_only=True) class Project: @@ -101,6 +103,23 @@ class Project: files: set[str] = field(default_factory=set) bdist_tree: dict | None = None + def _tags_to_strlist(self, tags): + return [str(t) for t in tags] + + def __json__(self): + return { + "sdist": self.sdist, + "bdist": self.bdist, + "versions_match": not (self.version_weird), + "pure_tags": self._tags_to_strlist(self.pure_tags), + "unknown_tags": self._tags_to_strlist(self.unknown_tags), + "other_tags": self._tags_to_strlist( + self.all_tags - self.pure_tags - self.unknown_tags, + ), + "files": list(self.files), + "compat_tree": self.bdist_tree, + } + def add_sdist(self, file: str): self.sdist = True self.files.add(file) From 1be1562a8a9a393bee81d63d0136a9b823b4a1f9 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Sun, 23 Nov 2025 20:02:52 -0500 Subject: [PATCH 62/63] Fix ROADMAP.md + pre-commit-config.yml --- .pre-commit-config.yaml | 45 +++++++++++++++++++++-------------------- ROADMAP.md | 36 +++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 22 deletions(-) create mode 100644 ROADMAP.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 354549f..b87c200 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,12 +2,12 @@ # See https://pre-commit.com/hooks.html for more hooks %YAML 1.2 --- +exclude: "site/.*" default_install_hook_types: [pre-commit, commit-msg] default_stages: [pre-commit] -exclude: 'site/.*' repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v6.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -18,12 +18,12 @@ repos: - id: check-toml - id: debug-statements - repo: https://github.com/asottile/add-trailing-comma - rev: v3.1.0 + rev: v4.0.0 hooks: - id: add-trailing-comma - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.11.6 + rev: v0.14.4 hooks: # Run the linter. - id: ruff @@ -33,7 +33,7 @@ repos: types_or: [python, pyi] # options: ignore one line things [E701] - repo: https://github.com/adrienverge/yamllint - rev: v1.35.1 + rev: v1.37.1 hooks: - id: yamllint name: yamllint @@ -41,14 +41,12 @@ repos: entry: yamllint language: python types: [file, yaml] - args: ['-d', "{\ - extends: default,\ - rules: {\ - colons: { max-spaces-after: -1 }\ - }\ - }"] + args: [ + '-d', + "{ extends: default, rules: { colons: { max-spaces-after: -1 } } }", + ] - repo: https://github.com/rhysd/actionlint - rev: v1.7.4 + rev: v1.7.8 hooks: - id: actionlint name: Lint GitHub Actions workflow files @@ -69,18 +67,9 @@ repos: args: [--staged, -c, "general.ignore=B6,T3", --msg-filename] stages: [commit-msg] - repo: https://github.com/crate-ci/typos - rev: v1.28.2 + rev: v1 hooks: - id: typos - - repo: https://github.com/markdownlint/markdownlint - rev: v0.13.0 - hooks: - - id: markdownlint - name: Markdownlint - description: Run markdownlint on your Markdown files - entry: mdl --style .markdown.rb - language: ruby - files: \.(md|mdown|markdown)$ - repo: https://github.com/Yelp/detect-secrets rev: v1.5.0 hooks: @@ -89,3 +78,15 @@ repos: language: python entry: detect-secrets-hook args: [''] + - repo: https://github.com/rvben/rumdl-pre-commit + rev: v0.0.173 # Use the latest release tag + hooks: + - id: rumdl + # To only check (default): + # args: [] + # To automatically fix issues: + # args: [--fix] + - repo: https://github.com/RobertCraigie/pyright-python + rev: v1.1.407 # pin a tag; latest as of 2025-10-01 + hooks: + - id: pyright diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..21a4efe --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,36 @@ +# Roadmap + +- [ ] We need to manually close the agent. (new branch)!!! +- [ ] Esta mal + +- [ ] Diff arbol: + - [ ] No imprimir pretty, solo indicar diferencias + - [ ] Manera de "ignorar rutas" + - [ ] Descargar Plantillas + +- [ ] Errores: + - [ ] `get_project_config` esconde errores? + +- [ ] Refactor to V1.1 + - [ ] Auditoria de version: rango de tiempo + - [ ] Reorgnaizer Internos + - service gh + - gh-name + - name + - audit name + - algo FACÍL usar + - [ ] Typer +- [ ] Imprimir: + - [ ] sitio web + - [ ] es espejo? Cómo está "upstream" +- [ ] Imprimir *un* repo o grupo, no solo -a +- [ ] Mostrar info de ramas + +- [ ] Esconder archivo/privado +- [ ] Mejorar filtrar +- [ ] Imprimir enlaces/activación para funciones (discussión) +- [ ] Issues/PRs stats + +- [ ] revisar locales + +- [ ] Seguridad, codeowners, permisos acciones From 04c095d5a1522a1a5169928672d3363d4004e43c Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Wed, 26 Nov 2025 12:27:17 -0500 Subject: [PATCH 63/63] More update to ROADMAP --- ROADMAP.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ROADMAP.md b/ROADMAP.md index 21a4efe..23c5224 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,7 +1,8 @@ # Roadmap +- [ ] BLOCKING: we need to make decision about versions/diff and merge + - [ ] We need to manually close the agent. (new branch)!!! -- [ ] Esta mal - [ ] Diff arbol: - [ ] No imprimir pretty, solo indicar diferencias