Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit2cbb98d

Browse files
committed
Small rework of the pypi cleanup script
1 parentd07d4a8 commit2cbb98d

File tree

3 files changed

+134
-113
lines changed

3 files changed

+134
-113
lines changed

‎.github/workflows/cleanup_pypi.yml‎

Lines changed: 47 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,14 @@ on:
66
description:CI environment to run in (pypi-test or pypi-prod-nightly)
77
type:string
88
required:true
9+
subcommand:
10+
description:List or Delete
11+
type:string
12+
default:delete
13+
verbosity:
14+
description:Tool verbosity ("verbose" or "debug")
15+
type:string
16+
default:verbose
917
secrets:
1018
PYPI_CLEANUP_OTP:
1119
description:PyPI OTP
@@ -15,17 +23,29 @@ on:
1523
required:true
1624
workflow_dispatch:
1725
inputs:
18-
dry-run:
19-
description:List packages that would be deleted but don't delete them
20-
type:boolean
21-
default:false
26+
subcommand:
27+
description:List or Delete
28+
type:choice
29+
required:true
30+
options:
31+
-list
32+
-delete
33+
default:list
2234
environment:
2335
description:CI environment to run in
2436
type:choice
2537
required:true
2638
options:
2739
-pypi-prod-nightly
2840
-pypi-test
41+
verbosity:
42+
description:Tool verbosity
43+
type:choice
44+
required:true
45+
options:
46+
-verbose
47+
-debug
48+
default:verbose
2949

3050
jobs:
3151
cleanup_pypi:
@@ -54,22 +74,37 @@ jobs:
5474
with:
5575
version:"0.9.0"
5676

57-
-name:Run Cleanup
77+
-name:Install dependencies
78+
run:uv sync --only-group pypi --no-install-project
79+
80+
-name:List Stale Packages on PyPI
81+
if:inputs.subcommand == 'list'
82+
env:
83+
PYTHON_UNBUFFERED:1
84+
run:|
85+
set -x
86+
uv run --no-sync python -u -m duckdb_packaging.pypi_cleanup \
87+
--${{ inputs.environment == 'pypi-prod-nightly' && 'prod' || 'test' }} \
88+
--max-nightlies ${{ vars.PYPI_MAX_NIGHTLIES }} \
89+
--${{ inputs.verbosity }} \
90+
list 2>&1 | tee cleanup_output
91+
92+
-name:Delete Stale Packages from PyPI
93+
if:inputs.subcommand == 'delete'
5894
env:
5995
PYTHON_UNBUFFERED:1
6096
run:|
6197
set -x
62-
uv sync --only-group pypi --no-install-project
63-
uv run --no-sync python -u -m duckdb_packaging.pypi_cleanup ${{ inputs.dry-run && '--dry' || '' }} \
64-
${{ inputs.environment == 'pypi-prod-nightly' && '--prod' || '--test' }} \
65-
--verbose \
66-
--username "${{ vars.PYPI_CLEANUP_USERNAME }}" \
67-
--max-nightlies ${{ vars.PYPI_MAX_NIGHTLIES }} 2>&1 | tee cleanup_output
98+
uv run --no-sync python -u -m duckdb_packaging.pypi_cleanup \
99+
--${{ inputs.environment == 'pypi-prod-nightly' && 'prod' || 'test' }} \
100+
--max-nightlies ${{ vars.PYPI_MAX_NIGHTLIES }} \
101+
--${{ inputs.verbosity }} \
102+
delete --username "${{ vars.PYPI_CLEANUP_USERNAME }}" 2>&1 | tee cleanup_output
68103
69104
-name:PyPI Cleanup Summary
70105
run :|
71106
echo "## PyPI Cleanup Summary" >> $GITHUB_STEP_SUMMARY
72-
echo "*Dry run: ${{ inputs.dry-run }}" >> $GITHUB_STEP_SUMMARY
107+
echo "*Subcommand: ${{ inputs.subcommand }}" >> $GITHUB_STEP_SUMMARY
73108
echo "* CI Environment: ${{ inputs.environment }}" >> $GITHUB_STEP_SUMMARY
74109
echo "* Output:" >> $GITHUB_STEP_SUMMARY
75110
echo '```' >> $GITHUB_STEP_SUMMARY

‎duckdb_packaging/pypi_cleanup.py‎

Lines changed: 76 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,14 @@
3838

3939
defcreate_argument_parser()->argparse.ArgumentParser:
4040
"""Create and configure the argument parser."""
41+
42+
defmax_nightlies_type(value:int)->int:
43+
"""Validate that --max-nightlies is set to a positive integer."""
44+
ifint(value)<0:
45+
msg=f"max-nightlies must be a positive integer, got{int(value)}"
46+
raiseValueError(msg)
47+
returnint(value)
48+
4149
parser=argparse.ArgumentParser(
4250
description="""
4351
PyPI cleanup script for removing development versions.
@@ -54,23 +62,54 @@ def create_argument_parser() -> argparse.ArgumentParser:
5462
formatter_class=argparse.RawDescriptionHelpFormatter,
5563
)
5664

57-
parser.add_argument("--dry-run",action="store_true",help="Show what would be deleted but don't actually do it")
65+
loglevel_group=parser.add_mutually_exclusive_group(required=False)
66+
loglevel_group.add_argument(
67+
"-d",
68+
"--debug",
69+
help="Show debug logs",
70+
dest="loglevel",
71+
action="store_const",
72+
const=logging.DEBUG,
73+
default=logging.WARNING,
74+
)
75+
loglevel_group.add_argument(
76+
"-v",
77+
"--verbose",
78+
help="Show info logs",
79+
dest="loglevel",
80+
action="store_const",
81+
const=logging.INFO,
82+
)
5883

5984
host_group=parser.add_mutually_exclusive_group(required=True)
60-
host_group.add_argument("--prod",action="store_true",help="Use production PyPI (pypi.org)")
61-
host_group.add_argument("--test",action="store_true",help="Use test PyPI (test.pypi.org)")
85+
host_group.add_argument(
86+
"--prod",help="Use production PyPI (pypi.org)",dest="pypi_url",action="store_const",const=_PYPI_URL_PROD
87+
)
88+
host_group.add_argument(
89+
"--test",help="Use test PyPI (test.pypi.org)",dest="pypi_url",action="store_const",const=_PYPI_URL_TEST
90+
)
6291

6392
parser.add_argument(
6493
"-m",
6594
"--max-nightlies",
66-
type=int,
95+
type=max_nightlies_type,
6796
default=_DEFAULT_MAX_NIGHTLIES,
6897
help=f"Max number of nightlies of unreleased versions (default={_DEFAULT_MAX_NIGHTLIES})",
6998
)
7099

71-
parser.add_argument("-u","--username",type=validate_username,help="PyPI username (required unless --dry-run)")
100+
subparsers=parser.add_subparsers(title="Subcommands")
72101

73-
parser.add_argument("-v","--verbose",action="store_true",help="Enable verbose debug logging")
102+
# Add the "list" subcommand
103+
parser_list=subparsers.add_parser("list",help="List all packages available for deletion")
104+
parser_list.set_defaults(func=lambdaargs:_run(CleanMode.LIST_ONLY,args))
105+
# Add the "delete" subcommand
106+
parser_delete=subparsers.add_parser(
107+
"delete",help="Delete packages that match the given criteria (use with care!"
108+
)
109+
parser_delete.add_argument(
110+
"-u","--username",type=validate_username,help="PyPI username (required)",required=True
111+
)
112+
parser_delete.set_defaults(func=lambdaargs:_run(CleanMode.DELETE,args))
74113

75114
returnparser
76115

@@ -149,17 +188,6 @@ def load_credentials() -> tuple[Optional[str], Optional[str]]:
149188
returnpassword,otp
150189

151190

152-
defvalidate_arguments(args:argparse.Namespace)->None:
153-
"""Validate parsed arguments."""
154-
ifnotargs.dry_runandnotargs.username:
155-
msg="--username is required when not in dry-run mode"
156-
raiseValidationError(msg)
157-
158-
ifargs.max_nightlies<0:
159-
msg="--max-nightlies must be non-negative"
160-
raiseValidationError(msg)
161-
162-
163191
classCsrfParser(HTMLParser):
164192
"""HTML parser to extract CSRF tokens from PyPI forms.
165193
@@ -230,7 +258,7 @@ def run(self) -> int:
230258
ifself._mode==CleanMode.DELETE:
231259
logging.warning("NOT A DRILL: WILL DELETE PACKAGES")
232260
elifself._mode==CleanMode.LIST_ONLY:
233-
logging.info("Running in DRY RUN mode, nothing will be deleted")
261+
logging.debug("Running in DRY RUN mode, nothing will be deleted")
234262
else:
235263
msg="Unexpected mode"
236264
raiseRuntimeError(msg)
@@ -255,20 +283,22 @@ def _execute_cleanup(self, http_session: Session) -> int:
255283
logging.info(f"No releases found for{self._package}")
256284
return0
257285

258-
# Determine versions to delete
286+
# Determineand reportversions to delete
259287
versions_to_delete=self._determine_versions_to_delete(versions)
260-
ifnotversions_to_delete:
261-
logging.info("No versions to delete (no stale rc's or dev releases)")
288+
iflen(versions_to_delete)>0:
289+
print(f"Found the following stale releases on{self._index_host}:")
290+
forversioninsorted(versions_to_delete):
291+
print(f"-{version}")
292+
else:
293+
print(f"No stale releases found on{self._index_host}")
262294
return0
263295

264-
logging.warning(f"Found{len(versions_to_delete)} versions to clean up:")
265-
forversioninsorted(versions_to_delete):
266-
logging.warning(version)
267-
268296
ifself._mode!=CleanMode.DELETE:
269297
logging.info("Dry run complete - no packages were deleted")
270298
return0
271299

300+
logging.warning(f"Will try to delete{len(versions_to_delete)} releases from{self._index_host}")
301+
272302
# Perform authentication and deletion
273303
self._authenticate(http_session)
274304
self._delete_versions(http_session,versions_to_delete)
@@ -527,42 +557,39 @@ def _delete_single_version(self, http_session: Session, version: str) -> None:
527557
delete_response.raise_for_status()
528558

529559

530-
defmain()->int:
531-
"""Main entry point for the script."""
532-
parser=create_argument_parser()
533-
args=parser.parse_args()
534-
535-
# Setup logging
536-
setup_logging((args.verboseandlogging.DEBUG)orlogging.INFO)
537-
560+
def_run(mode:CleanMode,args:argparse.Namespace)->int:
561+
"""Action called by the subcommands after arg parsing."""
562+
setup_logging(args.loglevel)
538563
try:
539-
# Validate arguments
540-
validate_arguments(args)
541-
542-
# Dry run vs delete
543-
password,otp,mode=None,None,CleanMode.LIST_ONLY
544-
ifnotargs.dry_run:
564+
ifmode==CleanMode.DELETE:
545565
password,otp=load_credentials()
546-
mode=CleanMode.DELETE
547-
548-
# Determine PyPI URL
549-
pypi_url=_PYPI_URL_PRODifargs.prodelse_PYPI_URL_TEST
550-
551-
# Create and run cleanup
552-
cleanup=PyPICleanup(pypi_url,mode,args.max_nightlies,username=args.username,password=password,otp=otp)
553-
566+
cleanup=PyPICleanup(
567+
args.pypi_url,mode,args.max_nightlies,username=args.username,password=password,otp=otp
568+
)
569+
elifmode==CleanMode.LIST_ONLY:
570+
cleanup=PyPICleanup(args.pypi_url,mode,args.max_nightlies)
571+
else:
572+
print(f"Unknown mode{mode}. Did nothing.")
573+
return-1
554574
returncleanup.run()
555-
556575
exceptValidationError:
557576
logging.exception("Configuration error")
558577
return2
559578
exceptKeyboardInterrupt:
560579
logging.info("Operation cancelled by user")
561580
return130
562581
exceptException:
563-
logging.exception("Unexpected error",exc_info=args.verbose)
582+
logging.exception("Unexpected error")
564583
return1
565584

566585

586+
defmain()->int:
587+
"""Main entry point for the script."""
588+
parser=create_argument_parser()
589+
args=parser.parse_args()
590+
# call the subcommand's func
591+
returnargs.func(args)
592+
593+
567594
if__name__=="__main__":
568595
sys.exit(main())

‎tests/fast/test_pypi_cleanup.py‎

Lines changed: 11 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
duckdb_packaging=pytest.importorskip("duckdb_packaging")
1616

1717
fromduckdb_packaging.pypi_cleanupimport (# noqa: E402
18+
_DEFAULT_MAX_NIGHTLIES,
19+
_PYPI_URL_PROD,
1820
AuthenticationError,
1921
CleanMode,
2022
CsrfParser,
@@ -26,7 +28,6 @@
2628
main,
2729
session_with_retries,
2830
setup_logging,
29-
validate_arguments,
3031
validate_username,
3132
)
3233

@@ -60,23 +61,6 @@ def test_validate_username_invalid(self):
6061
withpytest.raises(ArgumentTypeError,match="Invalid username format"):
6162
validate_username("invalid-")
6263

63-
deftest_validate_arguments_dry_run(self):
64-
"""Test argument validation for dry run mode."""
65-
args=Mock(dry_run=True,username=None,max_nightlies=2)
66-
validate_arguments(args)# Should not raise
67-
68-
deftest_validate_arguments_live_mode_no_username(self):
69-
"""Test argument validation for live mode without username."""
70-
args=Mock(dry_run=False,username=None,max_nightlies=2)
71-
withpytest.raises(ValidationError,match="username is required"):
72-
validate_arguments(args)
73-
74-
deftest_validate_arguments_negative_nightlies(self):
75-
"""Test argument validation with negative max nightlies."""
76-
args=Mock(dry_run=True,username="test",max_nightlies=-1)
77-
withpytest.raises(ValidationError,match="must be non-negative"):
78-
validate_arguments(args)
79-
8064

8165
classTestCredentials:
8266
"""Test credential loading."""
@@ -465,26 +449,20 @@ def test_argument_parser_creation(self):
465449
parser=create_argument_parser()
466450
assertparser.progisnotNone
467451

468-
deftest_parse_args_prod_dry_run(self):
452+
deftest_parse_args_prod_list(self):
469453
"""Test parsing arguments for production dry run."""
470454
parser=create_argument_parser()
471-
args=parser.parse_args(["--prod","--dry-run"])
455+
args=parser.parse_args(["--prod","list"])
472456

473-
assertargs.prodisTrue
474-
assertargs.testisFalse
475-
assertargs.dry_runisTrue
476-
assertargs.max_nightlies==2
477-
assertargs.verboseisFalse
457+
assertargs.pypi_urlis_PYPI_URL_PROD
458+
assertargs.max_nightlies==_DEFAULT_MAX_NIGHTLIES
459+
assertargs.loglevelislogging.WARN
478460

479-
deftest_parse_args_test_with_username(self):
461+
deftest_parse_args_test_list_with_username(self):
480462
"""Test parsing arguments for test with username."""
481463
parser=create_argument_parser()
482-
args=parser.parse_args(["--test","-u","testuser","--verbose"])
483-
484-
assertargs.testisTrue
485-
assertargs.prodisFalse
486-
assertargs.username=="testuser"
487-
assertargs.verboseisTrue
464+
withpytest.raises(SystemExit):
465+
parser.parse_args(["--test","list","-u","testuser"])
488466

489467
deftest_parse_args_missing_host(self):
490468
"""Test parsing arguments with missing host selection."""
@@ -506,28 +484,9 @@ def test_main_success(self, mock_cleanup_class, mock_setup_logging):
506484
mock_cleanup.run.return_value=0
507485
mock_cleanup_class.return_value=mock_cleanup
508486

509-
withpatch("sys.argv", ["pypi_cleanup.py","--test","-u","testuser"]):
487+
withpatch("sys.argv", ["pypi_cleanup.py","--test","delete","-u","testuser"]):
510488
result=main()
511489

512490
assertresult==0
513491
mock_setup_logging.assert_called_once()
514492
mock_cleanup.run.assert_called_once()
515-
516-
@patch("duckdb_packaging.pypi_cleanup.setup_logging")
517-
deftest_main_validation_error(self,mock_setup_logging):
518-
"""Test main function with validation error."""
519-
withpatch("sys.argv", ["pypi_cleanup.py","--test"]):# Missing username for live mode
520-
result=main()
521-
522-
assertresult==2# Validation error exit code
523-
524-
@patch("duckdb_packaging.pypi_cleanup.setup_logging")
525-
@patch("duckdb_packaging.pypi_cleanup.validate_arguments")
526-
deftest_main_keyboard_interrupt(self,mock_validate,mock_setup_logging):
527-
"""Test main function with keyboard interrupt."""
528-
mock_validate.side_effect=KeyboardInterrupt()
529-
530-
withpatch("sys.argv", ["pypi_cleanup.py","--test","--dry-run"]):
531-
result=main()
532-
533-
assertresult==130# Keyboard interrupt exit code

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp