From 43bc53c70e794fd8414cca043c52ea5feccc345c Mon Sep 17 00:00:00 2001 From: Antonio Sanchez Date: Mon, 29 Sep 2025 16:40:04 -0700 Subject: [PATCH] Useful scripts related to releases. These scripts use the GitLab API to determine MR/issue information, determine MR/issues related to commits, add labels, and mirror source archives to the generic package registry. Much of the content of these scripts was generated via Gemini, with minor corrections and edits. --- scripts/git_commit_mrs_and_issues.py | 128 +++++++++++++++++ scripts/gitlab_api_deploy_package.py | 136 ++++++++++++++++++ scripts/gitlab_api_issues.py | 174 +++++++++++++++++++++++ scripts/gitlab_api_labeller.py | 122 ++++++++++++++++ scripts/gitlab_api_mrs.py | 200 +++++++++++++++++++++++++++ 5 files changed, 760 insertions(+) create mode 100644 scripts/git_commit_mrs_and_issues.py create mode 100644 scripts/gitlab_api_deploy_package.py create mode 100644 scripts/gitlab_api_issues.py create mode 100644 scripts/gitlab_api_labeller.py create mode 100644 scripts/gitlab_api_mrs.py diff --git a/scripts/git_commit_mrs_and_issues.py b/scripts/git_commit_mrs_and_issues.py new file mode 100644 index 000000000..854ec390e --- /dev/null +++ b/scripts/git_commit_mrs_and_issues.py @@ -0,0 +1,128 @@ +"""Search for MRs and issues related to a list of commits.""" + +import argparse +import json +import sys +import subprocess +import re + + +def find_cherry_pick_source(commit_hash: str): + """ + For a given commit hash, find the original commit it was cherry-picked from. + + Args: + commit_hash: The commit hash to inspect. + + Returns: + The full hash of the original commit if found, otherwise None. + """ + try: + # Use 'git show' to get the full commit message for the given hash. + # The '-s' flag suppresses the diff output. + # The '--format=%B' flag prints only the raw commit body/message. + commit_message = subprocess.check_output( + ["git", "show", "-s", "--format=%B", commit_hash.strip()], + text=True, + stderr=subprocess.PIPE, + ).strip() + + # This regex looks for the specific line Git adds during a cherry-pick. + # It captures the full 40-character SHA-1 hash. + cherry_pick_pattern = re.compile( + r"\(cherry picked from commit ([a-f0-9]{40})\)" + ) + + # Search the entire commit message for the pattern. + match = cherry_pick_pattern.search(commit_message) + + if match: + # If a match is found, return the captured group (the original commit hash). + return match.group(1) + else: + return None + + except subprocess.CalledProcessError as e: + # This error occurs if the git command fails, e.g., for an invalid hash. + print( + f"Error processing commit '{commit_hash.strip()}': {e.stderr.strip()}", + file=sys.stderr, + ) + return None + except FileNotFoundError: + # This error occurs if the 'git' command itself isn't found. + print( + "Error: 'git' command not found. Please ensure Git is installed and in your PATH.", + file=sys.stderr, + ) + sys.exit(1) + + +def main(): + """ + Main function to read commit hashes from stdin and process them. + """ + parser = argparse.ArgumentParser( + description="A script to download all MRs from GitLab matching specified criteria." + ) + parser.add_argument( + "--merge_requests_file", + type=str, + required=True, + help="JSON file containing all the merge request information extracted via the GitLab API.", + ) + + # E.g. git log --pretty=%H 3e819d83bf52abda16bb53565f6801df40d071f1..3.4.1 + parser.add_argument( + "--commits", + required=True, + help="List of commits, '-' for stdin.", + ) + args = parser.parse_args() + + mrs = [] + with open(args.merge_requests_file, "r") as file: + mrs = json.load(file) + mrs_by_commit = {} + + if args.commits == "-": + commit_hashes = sys.stdin.readlines() + else: + with open(args.commits, "r") as file: + commit_hashes = file.readlines() + + # Arrange commits by SHA. + for mr in mrs: + for key in ["sha", "merge_commit_sha", "squash_commit_sha"]: + sha = mr[key] + if sha: + mrs_by_commit[sha] = mr + + # Find the MRs and issues related to each commit. + info = {} + for sha in commit_hashes: + sha = sha.strip() + if not sha: + continue + + # If a cherry-pick, extract the original hash. + sha = find_cherry_pick_source(sha) or sha + mr = mrs_by_commit.get(sha) + + commit_info = {} + if mr: + commit_info["merge_request"] = mr["iid"] + commit_info["related_issues"] = [ + issue["iid"] for issue in mr["related_issues"] + ] + commit_info["closes_issues"] = [ + issue["iid"] for issue in mr["closes_issues"] + ] + + info[sha] = commit_info + + print(json.dumps(info, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/gitlab_api_deploy_package.py b/scripts/gitlab_api_deploy_package.py new file mode 100644 index 000000000..70dc03ff1 --- /dev/null +++ b/scripts/gitlab_api_deploy_package.py @@ -0,0 +1,136 @@ +"""Helper script to download source archives and upload them to the Eigen GitLab generic package registry.""" + +import os +import requests +import hashlib +import argparse +import sys +import tempfile + +EIGEN_PROJECT_ID = 15462818 # Taken from the gitlab project page. + + +def calculate_sha256(filepath: str): + """Calculates the SHA256 checksum of a file.""" + sha256_hash = hashlib.sha256() + with open(filepath, "rb") as f: + # Read and update hash in chunks of 4K + for byte_block in iter(lambda: f.read(4096), b""): + sha256_hash.update(byte_block) + return sha256_hash.hexdigest() + + +def upload_to_generic_registry( + gitlab_private_token: str, package_name: str, package_version: str, filepath: str +): + """Uploads a file to the GitLab generic package registry.""" + headers = {"PRIVATE-TOKEN": gitlab_private_token} + filename = os.path.basename(filepath) + upload_url = f"https://gitlab.com/api/v4/projects/{EIGEN_PROJECT_ID}/packages/generic/{package_name}/{package_version}/{filename}" + + print(f"Uploading {filename} to {upload_url}...") + try: + with open(filepath, "rb") as f: + response = requests.put(upload_url, headers=headers, data=f) + response.raise_for_status() + print(f"Successfully uploaded {filename}.") + return True + except requests.exceptions.RequestException as e: + print(f"Error uploading {filename}: {e}") + if e.response is not None: + print(f"Response content: {e.response.text}") + return False + + +def main(): + """Main function to download archives and upload them to the registry.""" + parser = argparse.ArgumentParser( + description="Download GitLab release archives for Eigen and upload them to the generic package registry." + ) + parser.add_argument( + "--gitlab_private_token", + type=str, + help="GitLab private API token. Defaults to the GITLAB_PRIVATE_TOKEN environment variable if set.", + ) + parser.add_argument( + "--version", + required=True, + help="Specify a single version (tag name) to process.", + ) + parser.add_argument( + "--download-dir", help=f"Directory to store temporary downloads (optional)." + ) + args = parser.parse_args() + + if not args.gitlab_private_token: + args.gitlab_private_token = os.getenv("GITLAB_PRIVATE_TOKEN") + if not args.gitlab_private_token: + print("Could not determine GITLAB_PRIVATE_TOKEN.", file=sys.stderr) + parser.print_usage() + sys.exit(1) + + # Create download directory if it doesn't exist. + cleanup_download_dir = False + if args.download_dir: + if not os.path.exists(args.download_dir): + cleanup_download_dir = True + os.makedirs(args.download_dir) + else: + args.download_dir = tempfile.mkdtemp() + cleanup_download_dir = True + + for ext in ["tar.gz", "tar.bz2", "tar", "zip"]: + archive_filename = f"eigen-{args.version}.{ext}" + archive_url = f"https://gitlab.com/libeigen/eigen/-/archive/{args.version}/{archive_filename}" + archive_filepath = os.path.join(args.download_dir, archive_filename) + + # Download the archive + print(f"Downloading {archive_url}...") + try: + response = requests.get(archive_url, stream=True) + response.raise_for_status() + with open(archive_filepath, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + print(f"Downloaded to {archive_filepath}") + except requests.exceptions.RequestException as e: + print(f"Error downloading {archive_url}: {e}. Skipping.") + continue + + # Calculate SHA256 sum + sha256_sum = calculate_sha256(archive_filepath) + print(f"SHA256 sum: {sha256_sum}") + + # Create SHA256 sum file + sha_filename = f"{archive_filename}.sha256" + sha_filepath = os.path.join(args.download_dir, sha_filename) + with open(sha_filepath, "w") as f: + f.write(f"{sha256_sum} {archive_filename}\n") + print(f"Created SHA256 file: {sha_filepath}") + + # Upload archive to generic registry + if not upload_to_generic_registry( + args.gitlab_private_token, "eigen", args.version, archive_filepath + ): + # If upload fails, clean up and move to the next release + os.remove(archive_filepath) + os.remove(sha_filepath) + continue + + # Upload SHA256 sum file to generic registry + upload_to_generic_registry( + args.gitlab_private_token, "eigen", args.version, sha_filepath + ) + + # Clean up downloaded files + print("Cleaning up local files...") + os.remove(archive_filepath) + os.remove(sha_filepath) + + # Clean up the download directory if it's empty + if cleanup_download_dir and not os.listdir(args.download_dir): + os.rmdir(args.download_dir) + + +if __name__ == "__main__": + main() diff --git a/scripts/gitlab_api_issues.py b/scripts/gitlab_api_issues.py new file mode 100644 index 000000000..1d191fa3e --- /dev/null +++ b/scripts/gitlab_api_issues.py @@ -0,0 +1,174 @@ +"""Downloads all issues from GitLab matching specified criteria.""" + +import argparse +import datetime +import json +import os +import requests +import sys + +EIGEN_PROJECT_ID = 15462818 # Taken from the gitlab project page. + + +def date(date_string: str): + """Convert a date YY-MM-DD string to a datetime object.""" + try: + return datetime.strptime(date_string, "%Y-%m-%d") + except ValueError: + msg = f"Not a valid date: '{date_string}'. Expected format is YYYY-MM-DD." + raise argparse.ArgumentTypeError(msg) + + +def _get_api_query( + gitlab_private_token: str, url: str, params: dict[str, str] | None = None +): + next_page = "1" + if not params: + params = dict() + params["per_page"] = "100" + headers = {"PRIVATE-TOKEN": gitlab_private_token} + out = [] + while next_page: + params["page"] = next_page + try: + resp = requests.head(url=url, params=params, headers=headers) + if resp.status_code != 200: + print("Request failed: ", resp, file=sys.stderr) + break + + next_next_page = resp.headers["x-next-page"] + + resp = requests.get(url=url, params=params, headers=headers) + if resp.status_code != 200: + # Try again. + continue + + out.extend(resp.json()) + # Advance at the end, in case an exception occurs above so we can retry + next_page = next_next_page + except: + # Keep same next_page + continue + return out + + +def get_issues( + gitlab_private_token: str, + author_username: str | None = None, + state: str | None = None, + created_before: datetime.datetime | None = None, + created_after: datetime.datetime | None = None, + updated_after: datetime.datetime | None = None, + updated_before: datetime.datetime | None = None, +): + """Return list of merge requests. + + Args: + gitlab_token: GitLab API token. + author_username: issue author username. + state: issue state (opened, closed). + created_after: datetime start of period. + created_before: datetime end of period. + updated_after: datetime start of period. + updated_before: datetime end of period. + + Returns: + List of merge requests. + """ + url = f"https://gitlab.com/api/v4/projects/{str(EIGEN_PROJECT_ID)}/issues" + params = dict() + if author_username: + params["author_username"] = author_username + + if state: + params["state"] = state + + if created_before: + params["created_before"] = created_before.isoformat() + + if created_after: + params["created_after"] = created_after.isoformat() + + if updated_before: + params["updated_before"] = updated_before.isoformat() + + if updated_after: + params["updated_after"] = updated_after.isoformat() + + params["order_by"] = "created_at" + params["sort"] = "asc" + issues = _get_api_query(gitlab_private_token, url, params) + for issue in issues: + if int(issue["merge_requests_count"]) > 0: + issue_iid = issue["iid"] + issue["related_merge_requests"] = _get_api_query( + gitlab_private_token, f"{url}/{issue_iid}/related_merge_requests" + ) + issue["closed_by_merge_requests"] = _get_api_query( + gitlab_private_token, f"{url}/{issue_iid}/closed_by" + ) + return issues + + +def main(_): + parser = argparse.ArgumentParser( + description="A script to download all issues from GitLab matching specified criteria." + ) + parser.add_argument( + "--gitlab_private_token", + type=str, + help="GitLab private API token. Defaults to the GITLAB_PRIVATE_TOKEN environment variable if set.", + ) + parser.add_argument("--author", type=str, help="The name of the author.") + parser.add_argument( + "--state", + type=str, + choices=["opened", "closed"], + help="The state of the issue.", + ) + parser.add_argument( + "--created_before", + type=date, + help="The created-before date in YYYY-MM-DD format.", + ) + parser.add_argument( + "--created_after", + type=date, + help="The created-after date in YYYY-MM-DD format.", + ) + parser.add_argument( + "--updated_before", + type=date, + help="The updated-before date in YYYY-MM-DD format.", + ) + parser.add_argument( + "--updated_after", + type=date, + help="The updated-after date in YYYY-MM-DD format.", + ) + args = parser.parse_args() + + if not args.gitlab_private_token: + args.gitlab_private_token = os.getenv("GITLAB_PRIVATE_TOKEN") + if not args.gitlab_private_token: + print("Could not determine GITLAB_PRIVATE_TOKEN.", file=sys.stderr) + parser.print_usage() + sys.exit(1) + + # Parse the arguments from the command line + issues = get_issues( + gitlab_private_token=args.gitlab_private_token, + author_username=args.author, + state=args.state, + created_before=args.created_before, + created_after=args.created_after, + updated_before=args.updated_before, + updated_after=args.updated_after, + ) + + issue_str = json.dumps(issues, indent=2) + print(issue_str) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/scripts/gitlab_api_labeller.py b/scripts/gitlab_api_labeller.py new file mode 100644 index 000000000..f5d3530fc --- /dev/null +++ b/scripts/gitlab_api_labeller.py @@ -0,0 +1,122 @@ +"""Adds a label to a GitLab merge requests or issues.""" + +import os +import sys +import argparse +import requests + +EIGEN_PROJECT_ID = 15462818 # Taken from the gitlab project page. + + +def add_label_to_mr(private_token: str, mr_iid: int, label: str): + """ + Adds a label to a specific merge request in a GitLab project. + + Args: + private_token: The user's private GitLab API token. + mr_iid: The internal ID (IID) of the merge request. + label: The label to add. + """ + api_url = ( + f"https://gitlab.com/api/v4/projects/{EIGEN_PROJECT_ID}/merge_requests/{mr_iid}" + ) + headers = {"PRIVATE-TOKEN": private_token} + # Using 'add_labels' ensures we don't overwrite existing labels. + payload = {"add_labels": label} + + try: + response = requests.put(api_url, headers=headers, json=payload) + response.raise_for_status() # Raises an HTTPError for bad responses (4xx or 5xx) + print(f"✅ Successfully added label '{label}' to Merge Request !{mr_iid}.") + except requests.exceptions.RequestException as e: + print(f"❌ Error updating Merge Request !{mr_iid}: {e}", file=sys.stderr) + if hasattr(e, "response") and e.response is not None: + print(f" Response: {e.response.text}", file=sys.stderr) + + +def add_label_to_issue(private_token: str, issue_iid: int, label: str): + """ + Adds a label to a specific issue in a GitLab project. + + Args: + private_token: The user's private GitLab API token. + issue_iid: The internal ID (IID) of the issue. + label: The label to add. + """ + api_url = ( + f"https://gitlab.com/api/v4/projects/{EIGEN_PROJECT_ID}/issues/{issue_iid}" + ) + headers = {"PRIVATE-TOKEN": private_token} + payload = {"add_labels": label} + + try: + response = requests.put(api_url, headers=headers, json=payload) + response.raise_for_status() + print(f"✅ Successfully added label '{label}' to Issue #{issue_iid}.") + except requests.exceptions.RequestException as e: + print(f"❌ Error updating Issue #{issue_iid}: {e}", file=sys.stderr) + if hasattr(e, "response") and e.response is not None: + print(f" Response: {e.response.text}", file=sys.stderr) + + +def main(): + """ + Main function to parse arguments and trigger the labelling process. + """ + parser = argparse.ArgumentParser( + description="Add a label to GitLab merge requests and issues.", + formatter_class=argparse.RawTextHelpFormatter, + ) + parser.add_argument("label", help="The label to add.") + parser.add_argument( + "--mrs", + nargs="+", + type=int, + help="A space-separated list of Merge Request IIDs.", + ) + parser.add_argument( + "--issues", nargs="+", type=int, help="A space-separated list of Issue IIDs." + ) + parser.add_argument( + "--gitlab_private_token", + help="Your GitLab private access token. \n(Best practice is to use the GITLAB_PRIVATE_TOKEN environment variable instead.)", + ) + args = parser.parse_args() + + # Prefer environment variable for the token for better security. + gitlab_private_token = args.gitlab_private_token or os.environ.get( + "GITLAB_PRIVATE_TOKEN" + ) + if not gitlab_private_token: + print("Error: GitLab private token not found.", file=sys.stderr) + print( + "Please provide it using the --token argument or by setting the GITLAB_PRIVATE_TOKEN environment variable.", + file=sys.stderr, + ) + sys.exit(1) + + if not args.mrs and not args.issues: + print( + "Error: You must provide at least one merge request (--mrs) or issue (--issues) ID.", + file=sys.stderr, + ) + sys.exit(1) + + print("-" * 30) + + if args.mrs: + print(f"Processing {len(args.mrs)} merge request(s)...") + for mr_iid in args.mrs: + add_label_to_mr(gitlab_private_token, mr_iid, args.label) + + if args.issues: + print(f"\nProcessing {len(args.issues)} issue(s)...") + for issue_iid in args.issues: + add_label_to_issue(gitlab_private_token, issue_iid, args.label) + + print("-" * 30) + print("Script finished.") + + +if __name__ == "__main__": + main() diff --git a/scripts/gitlab_api_mrs.py b/scripts/gitlab_api_mrs.py new file mode 100644 index 000000000..12eb5f9b5 --- /dev/null +++ b/scripts/gitlab_api_mrs.py @@ -0,0 +1,200 @@ +"""Downloads all MRs from GitLab matching specified criteria.""" + +import argparse +import datetime +import json +import os +import requests +import sys + +EIGEN_PROJECT_ID = 15462818 # Taken from the gitlab project page. + + +def date(date_string: str): + """Convert a date YY-MM-DD string to a datetime object.""" + try: + return datetime.strptime(date_string, "%Y-%m-%d") + except ValueError: + msg = f"Not a valid date: '{date_string}'. Expected format is YYYY-MM-DD." + raise argparse.ArgumentTypeError(msg) + + +def _get_api_query( + gitlab_private_token: str, url: str, params: dict[str, str] | None = None +): + next_page = "1" + if not params: + params = dict() + params["per_page"] = "100" + headers = {"PRIVATE-TOKEN": gitlab_private_token} + out = [] + while next_page: + params["page"] = next_page + try: + resp = requests.head(url=url, params=params, headers=headers) + if resp.status_code != 200: + print("Request failed: ", resp, file=sys.stderr) + break + + next_next_page = resp.headers["x-next-page"] + + resp = requests.get(url=url, params=params, headers=headers) + if resp.status_code != 200: + # Try again. + continue + + out.extend(resp.json()) + # Advance at the end, in case an exception occurs above so we can retry + next_page = next_next_page + except: + # Keep same next_page + continue + return out + + +def get_merge_requests( + gitlab_private_token: str, + author_username: str | None = None, + state: str | None = None, + created_before: datetime.datetime | None = None, + created_after: datetime.datetime | None = None, + updated_after: datetime.datetime | None = None, + updated_before: datetime.datetime | None = None, + related_issues: bool = False, + closes_issues: bool = False, +): + """Return list of merge requests. + + Args: + gitlab_token: GitLab API token. + author_username: MR author username. + state: MR state (merged, opened, closed, locked). + created_after: datetime start of period. + created_before: datetime end of period. + updated_after: datetime start of period. + updated_before: datetime end of period. + + Returns: + List of merge requests. + """ + url = ( + "https://gitlab.com/api/v4/projects/" + + str(EIGEN_PROJECT_ID) + + "/merge_requests" + ) + params = dict() + if author_username: + params["author_username"] = author_username + + if state: + params["state"] = state + + if created_before: + params["created_before"] = created_before.isoformat() + + if created_after: + params["created_after"] = created_after.isoformat() + + if updated_before: + params["updated_before"] = updated_before.isoformat() + + if updated_after: + params["updated_after"] = updated_after.isoformat() + + params["order_by"] = "created_at" + params["sort"] = "asc" + + next_page = "1" + params["per_page"] = "100" + headers = {"PRIVATE-TOKEN": gitlab_private_token} + + mrs = _get_api_query(gitlab_private_token, url, params) + + if related_issues: + for mr in mrs: + mr["related_issues"] = _get_api_query( + gitlab_private_token, f"{url}/{mr['iid']}/related_issues" + ) + + if closes_issues: + for mr in mrs: + mr["closes_issues"] = _get_api_query( + gitlab_private_token, f"{url}/{mr['iid']}/closes_issues" + ) + + return mrs + + +def main(_): + parser = argparse.ArgumentParser( + description="A script to download all MRs from GitLab matching specified criteria." + ) + parser.add_argument( + "--gitlab_private_token", + type=str, + help="GitLab private API token. Defaults to the GITLAB_PRIVATE_TOKEN environment variable if set.", + ) + parser.add_argument("--author", type=str, help="The name of the author.") + parser.add_argument( + "--state", + type=str, + choices=["merged", "opened", "closed", "locked"], + help="The state of the MR.", + ) + parser.add_argument( + "--created_before", + type=date, + help="The created-before date in YYYY-MM-DD format.", + ) + parser.add_argument( + "--created_after", + type=date, + help="The created-after date in YYYY-MM-DD format.", + ) + parser.add_argument( + "--updated_before", + type=date, + help="The updated-before date in YYYY-MM-DD format.", + ) + parser.add_argument( + "--updated_after", + type=date, + help="The updated-after date in YYYY-MM-DD format.", + ) + parser.add_argument( + "--related_issues", action="store_true", help="Query for related issues." + ) + parser.add_argument( + "--closes_issues", + action="store_true", + help="Query for issues closed by the MR.", + ) + + args = parser.parse_args() + + if not args.gitlab_private_token: + args.gitlab_private_token = os.getenv("GITLAB_PRIVATE_TOKEN") + if not args.gitlab_private_token: + print("Could not determine GITLAB_PRIVATE_TOKEN.", file=sys.stderr) + parser.print_usage() + sys.exit(1) + + # Parse the arguments from the command line + mrs = get_merge_requests( + gitlab_private_token=args.gitlab_private_token, + author_username=args.author, + state=args.state, + created_before=args.created_before, + created_after=args.created_after, + updated_before=args.updated_before, + updated_after=args.updated_after, + related_issues=args.related_issues, + closes_issues=args.closes_issues, + ) + + mr_str = json.dumps(mrs, indent=2) + print(mr_str) + + +if __name__ == "__main__": + main(sys.argv) -- GitLab