From ff89ca1d97d62487733bcee32e60c99143f8b37d Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:33:22 +0900 Subject: [PATCH 01/25] create separate session class --- scripts/mediawiki_session.py | 92 ++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 scripts/mediawiki_session.py diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py new file mode 100644 index 00000000000..8b3cc39ddd3 --- /dev/null +++ b/scripts/mediawiki_session.py @@ -0,0 +1,92 @@ +import contextlib +import functools +import http.cookiejar +import os +import time + +import requests + +from typing import Any, Optional + +from deploy_util import HEADER, SLEEP_DURATION + +USER_AGENT = f"GitHub Autodeploy Bot/2.0.0 ({os.getenv('WIKI_UA_EMAIL')})" +WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") +WIKI_USER = os.getenv("WIKI_USER") +WIKI_PASSWORD = os.getenv("WIKI_PASSWORD") +HEADER = { + "User-Agent": USER_AGENT, + "accept": "application/json", + "Accept-Encoding": "gzip", +} + + +class MediaWikiSessionError(IOError): + pass + + +class MediaWikiSession(contextlib.AbstractContextManager): + __cookie_jar: http.cookiejar.FileCookieJar + __session: requests.Session + __wiki: str + + def __init__(self, wiki: str): + self.__wiki = wiki + self.__cookie_jar = self.__read_cookie_jar() + self.__session = requests.session() + self.__session.cookies = self.__cookie_jar + self.__session.headers.update(HEADER) + + def __read_cookie_jar(self) -> http.cookiejar.FileCookieJar: + ckf = f"cookie_{self.__wiki}.ck" + cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) + with contextlib.suppress(OSError): + cookie_jar.load(ignore_discard=True) + return cookie_jar + + @functools.cache + def __get_wiki_api_url(self): + return f"{WIKI_BASE_URL}/{self.__wiki}/api.php" + + def _login(self): + token_response = self.make_action( + "query", params={"meta": "tokens", "type": "login"} + ) + self.make_action( + "login", + data={ + "lgname": WIKI_USER, + "lgpassword": WIKI_PASSWORD, + "lgtoken": token_response["query"]["tokens"]["logintoken"], + }, + ) + self.__cookie_jar.save(ignore_discard=True) + self.cooldown() + + @functools.cached_property + def token(self) -> str: + self._login() + return self.make_action("query", params={"meta": "tokens"})["tokens"]["csrftoken"] + + def make_action( + self, action: str, params: Optional[dict] = None, data: Optional[dict] = None + ) -> dict[str, Any]: + merged_params = {"format": "json", "action": action} + if params is not None: + merged_params = merged_params | params + response: dict = self.__session.post( + self.__get_wiki_api_url(), params=merged_params, data=data + ).json() + if "error" in response.keys(): + raise MediaWikiSessionError(response["error"]["info"]) + return response[action] + + def cooldown(self): + time.sleep(SLEEP_DURATION) + + def close(self): + self.__cookie_jar.save(ignore_discard=True) + self.__session.close() + + def __exit__(self, exc_type, exc_value, traceback): + self.close() From 770e447e87d1ecc7ce417f4104ad7998a8ee8e38 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:36:05 +0900 Subject: [PATCH 02/25] cache get_wikis output --- scripts/deploy_util.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/deploy_util.py b/scripts/deploy_util.py index 32552af2238..7ef4791884d 100644 --- a/scripts/deploy_util.py +++ b/scripts/deploy_util.py @@ -32,14 +32,15 @@ SLEEP_DURATION = 4 -def get_wikis() -> set[str]: +@functools.cache +def get_wikis() -> frozenset[str]: response = requests.get( "https://liquipedia.net/api.php", headers=HEADER, ) wikis = response.json() time.sleep(SLEEP_DURATION) - return set(wikis["allwikis"].keys()) + return frozenset(wikis["allwikis"].keys()) @functools.cache From 2ce81b23af2e00b45b1bf189b8e2daa517764ca5 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:37:47 +0900 Subject: [PATCH 03/25] __all__ --- scripts/mediawiki_session.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 8b3cc39ddd3..8e80c649859 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -10,6 +10,11 @@ from deploy_util import HEADER, SLEEP_DURATION +__all__ = [ + "MediaWikiSession", + "MediaWikiSessionError", +] + USER_AGENT = f"GitHub Autodeploy Bot/2.0.0 ({os.getenv('WIKI_UA_EMAIL')})" WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") WIKI_USER = os.getenv("WIKI_USER") @@ -66,7 +71,9 @@ def _login(self): @functools.cached_property def token(self) -> str: self._login() - return self.make_action("query", params={"meta": "tokens"})["tokens"]["csrftoken"] + return self.make_action("query", params={"meta": "tokens"})["tokens"][ + "csrftoken" + ] def make_action( self, action: str, params: Optional[dict] = None, data: Optional[dict] = None From 61a584418400640f396479cb03c2227e278751ec Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:42:08 +0900 Subject: [PATCH 04/25] make pylance happy --- scripts/mediawiki_session.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 8e80c649859..53eb7b88e5e 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -95,5 +95,8 @@ def close(self): self.__cookie_jar.save(ignore_discard=True) self.__session.close() + def __enter__(self): + return self + def __exit__(self, exc_type, exc_value, traceback): self.close() From 5513cb0b760aa835af971a077523883cfc13b869 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:57:28 +0900 Subject: [PATCH 05/25] refactor resource deploy --- scripts/deploy_res.py | 114 ++++++++++++++++------------------- scripts/mediawiki_session.py | 59 +++++++++++++++++- 2 files changed, 110 insertions(+), 63 deletions(-) diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py index 2b30f76a840..56cb17e3fd3 100644 --- a/scripts/deploy_res.py +++ b/scripts/deploy_res.py @@ -5,66 +5,55 @@ from typing import Iterable -import requests - from deploy_util import ( - HEADER, get_git_deploy_reason, - get_wiki_api_url, - deploy_file_to_wiki, - read_cookie_jar, read_file_from_path, ) -from login_and_get_token import get_token +from mediawiki_session import MediaWikiSession def deploy_resources( - res_type: str, file_paths: Iterable[pathlib.Path], deploy_reason: str + session: MediaWikiSession, + res_type: str, + file_paths: Iterable[pathlib.Path], + deploy_reason: str, ) -> tuple[bool, bool]: all_deployed = True changes_made = False - token = get_token("commons") - with requests.Session() as session: - session.cookies = read_cookie_jar("commons") - for file_path in file_paths: - print(f"::group::Checking {str(file_path)}") - file_content = read_file_from_path(file_path) - page = ( - f"MediaWiki:Common.{'js' if res_type == '.js' else 'css'}/" - + file_path.name - ) - print(f"...page = {page}") - deploy_result = deploy_file_to_wiki( - session, file_path, file_content, "commons", page, token, deploy_reason - ) - all_deployed = all_deployed and deploy_result[0] - changes_made = changes_made or deploy_result[1] - print("::endgroup::") + for file_path in file_paths: + print(f"::group::Checking {str(file_path)}") + file_content = read_file_from_path(file_path) + page = ( + f"MediaWiki:Common.{'js' if res_type == '.js' else 'css'}/" + file_path.name + ) + print(f"...page = {page}") + deploy_result = session.deploy_file( + file_path, file_content, page, deploy_reason + ) + all_deployed = all_deployed and deploy_result[0] + changes_made = changes_made or deploy_result[1] + print("::endgroup::") return (all_deployed, changes_made) -def update_cache(): - with requests.Session() as session: - session.cookies = read_cookie_jar("commons") - cache_result = session.post( - get_wiki_api_url("commons"), - headers=HEADER, - params={"format": "json", "action": "updatelpmwmessageapi"}, - data={ - "messagename": "Resourceloaderarticles-cacheversion", - "value": subprocess.check_output(["git", "log", "-1", "--pretty=%h"]) - .decode() - .strip(), - }, - ).json() - if ( - cache_result["updatelpmwmessageapi"].get("message") - == "Successfully changed the message value" - ): - print("Resource cache version updated succesfully!") - else: - print("::error::Resource cache version unable to be updated!") - exit(1) +def update_cache(session: MediaWikiSession): + cache_result = session.make_action( + "updatelpmwmessageapi", + data={ + "messagename": "Resourceloaderarticles-cacheversion", + "value": subprocess.check_output(["git", "log", "-1", "--pretty=%h"]) + .decode() + .strip(), + }, + ) + if ( + cache_result["updatelpmwmessageapi"].get("message") + == "Successfully changed the message value" + ): + print("Resource cache version updated succesfully!") + else: + print("::error::Resource cache version unable to be updated!") + exit(1) def main(): @@ -82,22 +71,23 @@ def main(): resource_files = [pathlib.Path(arg) for arg in sys.argv[1:]] git_deploy_reason = get_git_deploy_reason() - for res_type, files in itertools.groupby( - sorted(resource_files), lambda path: path.suffix - ): - group_all_deployed, group_changes_made = deploy_resources( - res_type, list(files), git_deploy_reason - ) - all_deployed = all_deployed and group_all_deployed - changes_made = changes_made or group_changes_made + with MediaWikiSession("commons") as commons_session: + for res_type, files in itertools.groupby( + sorted(resource_files), lambda path: path.suffix + ): + group_all_deployed, group_changes_made = deploy_resources( + commons_session, res_type, list(files), git_deploy_reason + ) + all_deployed = all_deployed and group_all_deployed + changes_made = changes_made or group_changes_made - if not all_deployed: - print( - "::error::Some files were not deployed; resource cache version not updated!" - ) - exit(1) - elif changes_made: - update_cache() + if not all_deployed: + print( + "::error::Some files were not deployed; resource cache version not updated!" + ) + exit(1) + elif changes_made: + update_cache(commons_session) if __name__ == "__main__": diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 53eb7b88e5e..380fa17465f 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -2,13 +2,19 @@ import functools import http.cookiejar import os +import pathlib import time import requests from typing import Any, Optional -from deploy_util import HEADER, SLEEP_DURATION +from deploy_util import ( + DEPLOY_TRIGGER, + HEADER, + SLEEP_DURATION, + write_to_github_summary_file, +) __all__ = [ "MediaWikiSession", @@ -91,6 +97,57 @@ def make_action( def cooldown(self): time.sleep(SLEEP_DURATION) + def deploy_file( + self, + file_path: pathlib.Path, + file_content: str, + target_page: str, + deploy_reason: str, + ) -> tuple[bool, bool]: + try: + change_made = False + deployed = True + response = self.make_action( + "edit", + data={ + "title": target_page, + "text": file_content, + "summary": f"Git: {deploy_reason}", + "bot": "true", + "recreate": "true", + "token": self.token, + }, + ) + result = response.get("result") + new_rev_id = response.get("newrevid") + if result == "Success": + if new_rev_id is not None: + change_made = True + if DEPLOY_TRIGGER != "push": + print(f"::warning file={str(file_path)}::File changed") + print(f"...{result}") + print("...done") + write_to_github_summary_file( + f":information_source: {str(file_path)} successfully deployed" + ) + + else: + print(f"::warning file={str(file_path)}::failed to deploy") + write_to_github_summary_file( + f":warning: {str(file_path)} failed to deploy" + ) + deployed = False + time.sleep(SLEEP_DURATION) + return deployed, change_made + except MediaWikiSessionError as e: + print(f"::warning file={str(file_path)}::failed to deploy (API error)") + write_to_github_summary_file( + f":warning: {str(file_path)} failed to deploy due to API error: {str(e)}" + ) + deployed = False + time.sleep(SLEEP_DURATION) + return deployed, change_made + def close(self): self.__cookie_jar.save(ignore_discard=True) self.__session.close() From 4b0eb56575eb9b7d21e321a723824671d6463bde Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:05:43 +0900 Subject: [PATCH 06/25] refactor module deploy --- scripts/deploy.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index 662139e7f3b..dffd1854605 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -6,16 +6,12 @@ from typing import Iterable -import requests - from deploy_util import ( get_git_deploy_reason, - deploy_file_to_wiki, - read_cookie_jar, read_file_from_path, write_to_github_summary_file, ) -from login_and_get_token import get_token +from mediawiki_session import MediaWikiSession HEADER_PATTERN = re.compile( r"\A---\n" r"-- @Liquipedia\n" r"-- page=(?P[^\n]*)\n" @@ -26,9 +22,7 @@ def deploy_all_files_for_wiki( wiki: str, file_paths: Iterable[pathlib.Path], deploy_reason: str ) -> bool: all_modules_deployed = True - token = get_token(wiki) - with requests.Session() as session: - session.cookies = read_cookie_jar(wiki) + with MediaWikiSession(wiki) as session: for file_path in file_paths: print(f"::group::Checking {str(file_path)}") file_content = read_file_from_path(file_path) @@ -40,8 +34,8 @@ def deploy_all_files_for_wiki( page = header_match.groupdict()["pageName"] + ( os.getenv("LUA_DEV_ENV_NAME") or "" ) - module_deployed, _ = deploy_file_to_wiki( - session, file_path, file_content, wiki, page, token, deploy_reason + module_deployed, _ = session.deploy_file( + file_path, file_content, page, deploy_reason ) all_modules_deployed &= module_deployed print("::endgroup::") From 6e0ed0c8d836e3a0e3da936fe1957fb900f9ec27 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:29:26 +0900 Subject: [PATCH 07/25] refactor dev remove --- scripts/mediawiki_session.py | 4 ++ scripts/remove_dev.py | 95 +++++++++++++++--------------------- 2 files changed, 44 insertions(+), 55 deletions(-) diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 380fa17465f..15ab76ab043 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -81,6 +81,10 @@ def token(self) -> str: "csrftoken" ] + @property + def wiki(self) -> str: + return self.__wiki + def make_action( self, action: str, params: Optional[dict] = None, data: Optional[dict] = None ) -> dict[str, Any]: diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py index c8918c96652..4c8fa610b83 100644 --- a/scripts/remove_dev.py +++ b/scripts/remove_dev.py @@ -1,85 +1,70 @@ import os -import time - -import requests from deploy_util import ( - HEADER, - SLEEP_DURATION, - get_wiki_api_url, get_wikis, - read_cookie_jar, write_to_github_summary_file, ) -from login_and_get_token import get_token, login +from mediawiki_session import MediaWikiSession, MediaWikiSessionError LUA_DEV_ENV_NAME = os.getenv("LUA_DEV_ENV_NAME") remove_errors: list[str] = list() -def remove_page(session: requests.Session, page: str, wiki: str): - print(f"deleting {wiki}:{page}") - token = get_token(wiki) - - result = session.post( - get_wiki_api_url(wiki), - headers=HEADER, - params={ - "format": "json", - "action": "delete", - }, - data={ - "title": page, - "reason": f"Remove {LUA_DEV_ENV_NAME}", - "token": token, - }, - ).json() - time.sleep(SLEEP_DURATION) - - if "delete" not in result.keys(): - print(f"::warning::could not delete {page} on {wiki}") - write_to_github_summary_file(f":warning: could not delete {page} on {wiki}") - remove_errors.append(f"{wiki}:{page}") +def remove_page(session: MediaWikiSession, page: str): + print(f"deleting {session.wiki}:{page}") - -def search_and_remove(wiki: str): - with requests.Session() as session: - search_result = session.post( - get_wiki_api_url(wiki), - headers=HEADER, - params={"format": "json", "action": "query"}, + try: + session.post( + "delete", data={ - "list": "search", - "srsearch": f"intitle:{LUA_DEV_ENV_NAME}", - "srnamespace": 828, - "srlimit": 5000, - "srprop": "", + "title": page, + "reason": f"Remove {LUA_DEV_ENV_NAME}", + "token": session.token, }, - ).json() - time.sleep(SLEEP_DURATION) + ) + except MediaWikiSessionError as e: + print(f"::warning::could not delete {page} on {session.wiki}") + write_to_github_summary_file( + f":warning: could not delete {page} on {session.wiki}" + ) + remove_errors.append(f"{session.wiki}:{page}") + finally: + session.cooldown() - # Handle API error responses and missing or empty search results safely. - if "error" in search_result: - error_info = search_result.get("error") - print(f"::warning::search API error on {wiki}: {error_info}") + +def search_and_remove(wiki: str): + with MediaWikiSession(wiki) as session: + search_result: dict + try: + search_result = session.make_action( + "query", + data={ + "list": "search", + "srsearch": f"intitle:{LUA_DEV_ENV_NAME}", + "srnamespace": 828, + "srlimit": 5000, + "srprop": "", + }, + ) + except MediaWikiSessionError as e: + print(f"::warning::search API error on {wiki}: {str(e)}") write_to_github_summary_file( - f":warning: search API error on {wiki}: {error_info}" + f":warning: search API error on {wiki}: {str(e)}" ) return + finally: + session.cooldown() - pages = search_result.get("query", {}).get("search") or [] + pages = search_result.get("search") or [] if len(pages) == 0: return - login(wiki) - session.cookies = read_cookie_jar(wiki) - for page in pages: if os.getenv("INCLUDE_SUB_ENVS") == "true" or page["title"].endswith( LUA_DEV_ENV_NAME ): - remove_page(session, page["title"], wiki) + remove_page(session, page["title"]) def main(): From eabc49a68775cd560568f4316119b6ed605d1fda Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:29:57 +0900 Subject: [PATCH 08/25] remove duplicate code --- scripts/mediawiki_session.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 15ab76ab043..9c5e07de95f 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -141,7 +141,6 @@ def deploy_file( f":warning: {str(file_path)} failed to deploy" ) deployed = False - time.sleep(SLEEP_DURATION) return deployed, change_made except MediaWikiSessionError as e: print(f"::warning file={str(file_path)}::failed to deploy (API error)") @@ -149,8 +148,9 @@ def deploy_file( f":warning: {str(file_path)} failed to deploy due to API error: {str(e)}" ) deployed = False - time.sleep(SLEEP_DURATION) return deployed, change_made + finally: + self.cooldown() def close(self): self.__cookie_jar.save(ignore_discard=True) From 3f794f38b7601a5d41319c6642dcc5fd47bb3a37 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:32:20 +0900 Subject: [PATCH 09/25] cleanup --- scripts/deploy_util.py | 77 +----------------------------------- scripts/mediawiki_session.py | 19 +++------ 2 files changed, 7 insertions(+), 89 deletions(-) diff --git a/scripts/deploy_util.py b/scripts/deploy_util.py index 7ef4791884d..e53e8c800fc 100644 --- a/scripts/deploy_util.py +++ b/scripts/deploy_util.py @@ -1,3 +1,4 @@ +import contextlib import functools import http.cookiejar import os @@ -8,10 +9,8 @@ import requests __all__ = [ - "DEPLOY_TRIGGER", "HEADER", "SLEEP_DURATION", - "deploy_file_to_wiki", "get_git_deploy_reason", "get_wiki_api_url", "get_wikis", @@ -20,7 +19,6 @@ "write_to_github_summary_file", ] -DEPLOY_TRIGGER = os.getenv("DEPLOY_TRIGGER") GITHUB_STEP_SUMMARY_FILE = os.getenv("GITHUB_STEP_SUMMARY") USER_AGENT = f"GitHub Autodeploy Bot/2.0.0 ({os.getenv('WIKI_UA_EMAIL')})" WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") @@ -56,82 +54,11 @@ def get_git_deploy_reason(): ) -def deploy_file_to_wiki( - session: requests.Session, - file_path: pathlib.Path, - file_content: str, - wiki: str, - target_page: str, - token: str, - deploy_reason: str, -) -> tuple[bool, bool]: - change_made = False - deployed = True - response = session.post( - get_wiki_api_url(wiki), - headers=HEADER, - params={"format": "json", "action": "edit"}, - data={ - "title": target_page, - "text": file_content, - "summary": f"Git: {deploy_reason}", - "bot": "true", - "recreate": "true", - "token": token, - }, - ).json() - edit_info = response.get("edit") - error_info = response.get("error") - - # Handle API errors or unexpected response structure - if error_info is not None or edit_info is None: - print(f"::warning file={str(file_path)}::failed to deploy (API error)") - details = "" - if isinstance(error_info, dict): - code = error_info.get("code") - info = error_info.get("info") - detail_parts = [] - if code: - detail_parts.append(f"code={code}") - if info: - detail_parts.append(f"info={info}") - if detail_parts: - details = " (" + ", ".join(detail_parts) + ")" - write_to_github_summary_file( - f":warning: {str(file_path)} failed to deploy due to API error{details}" - ) - deployed = False - time.sleep(SLEEP_DURATION) - return deployed, change_made - - result = edit_info.get("result") - new_rev_id = edit_info.get("newrevid") - if result == "Success": - if new_rev_id is not None: - change_made = True - if DEPLOY_TRIGGER != "push": - print(f"::warning file={str(file_path)}::File changed") - print(f"...{result}") - print("...done") - write_to_github_summary_file( - f":information_source: {str(file_path)} successfully deployed" - ) - - else: - print(f"::warning file={str(file_path)}::failed to deploy") - write_to_github_summary_file(f":warning: {str(file_path)} failed to deploy") - deployed = False - time.sleep(SLEEP_DURATION) - return deployed, change_made - - def read_cookie_jar(wiki: str) -> http.cookiejar.FileCookieJar: ckf = f"cookie_{wiki}.ck" cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) - try: + with contextlib.suppress(OSError): cookie_jar.load(ignore_discard=True) - except OSError: - pass return cookie_jar diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 9c5e07de95f..3240f0a23e6 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -10,9 +10,10 @@ from typing import Any, Optional from deploy_util import ( - DEPLOY_TRIGGER, HEADER, SLEEP_DURATION, + WIKI_BASE_URL, + read_cookie_jar, write_to_github_summary_file, ) @@ -21,15 +22,9 @@ "MediaWikiSessionError", ] -USER_AGENT = f"GitHub Autodeploy Bot/2.0.0 ({os.getenv('WIKI_UA_EMAIL')})" -WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") +DEPLOY_TRIGGER = os.getenv("DEPLOY_TRIGGER") WIKI_USER = os.getenv("WIKI_USER") WIKI_PASSWORD = os.getenv("WIKI_PASSWORD") -HEADER = { - "User-Agent": USER_AGENT, - "accept": "application/json", - "Accept-Encoding": "gzip", -} class MediaWikiSessionError(IOError): @@ -49,15 +44,11 @@ def __init__(self, wiki: str): self.__session.headers.update(HEADER) def __read_cookie_jar(self) -> http.cookiejar.FileCookieJar: - ckf = f"cookie_{self.__wiki}.ck" - cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) - with contextlib.suppress(OSError): - cookie_jar.load(ignore_discard=True) - return cookie_jar + return read_cookie_jar(self.wiki) @functools.cache def __get_wiki_api_url(self): - return f"{WIKI_BASE_URL}/{self.__wiki}/api.php" + return f"{WIKI_BASE_URL}/{self.wiki}/api.php" def _login(self): token_response = self.make_action( From 29a3c3e35b219b73bf8d41307cb803fce1598027 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:09:19 +0900 Subject: [PATCH 10/25] overhaul module protect --- scripts/protect.py | 100 ++++++++++++++++++++++++++-------------- scripts/protect_page.py | 70 +++++++++++++++++++++++++++- 2 files changed, 134 insertions(+), 36 deletions(-) diff --git a/scripts/protect.py b/scripts/protect.py index 562544b6c75..9a0bb9ab4f1 100644 --- a/scripts/protect.py +++ b/scripts/protect.py @@ -1,58 +1,88 @@ +import itertools import os import pathlib import sys -from typing import Iterable - from deploy_util import get_wikis +from mediawiki_session import MediaWikiSession from protect_page import ( - protect_non_existing_page, - protect_existing_page, + protect_non_existing_pages, + protect_existing_pages, handle_protect_errors, ) WIKI_TO_PROTECT = os.getenv("WIKI_TO_PROTECT") -def check_for_local_version(module: str, wiki: str): - if wiki == "commons": - return False - return pathlib.Path(f"./lua/wikis/{wiki}/{module}.lua").exists() +def protect_new_wiki(wiki_to_protect: str): + lua_files = itertools.chain( + pathlib.Path("./lua/wikis/commons/").rglob("*.lua"), + pathlib.Path("./lua/wikis/" + wiki_to_protect + "/").rglob("*.lua"), + ) + + commons_modules: set[str] = set() + local_modules: set[str] = set() + + for file_to_protect in sorted(lua_files): + wiki = file_to_protect.parts[2] + module = "/".join(file_to_protect.parts[3:])[:-4] + page = "Module:" + module + + if wiki == wiki_to_protect: + local_modules.add(page) + elif wiki == "commons": + commons_modules.add(page) + with MediaWikiSession(wiki_to_protect) as session: + protect_non_existing_pages(session, commons_modules - local_modules) + protect_existing_pages(session, local_modules) -def protect_if_has_no_local_version(module: str, wiki: str): - page = "Module:" + module - if not check_for_local_version(module, wiki): - protect_non_existing_page(page, wiki) + handle_protect_errors() def main(): - lua_files: Iterable[pathlib.Path] - if len(sys.argv[1:]) > 0: - lua_files = [pathlib.Path(arg) for arg in sys.argv[1:]] - elif WIKI_TO_PROTECT: - lua_files = pathlib.Path("./lua/wikis/").rglob("*.lua") - else: + if WIKI_TO_PROTECT: + protect_new_wiki(WIKI_TO_PROTECT) + return + elif len(sys.argv[1:]) == 0: print("Nothing to protect") exit(0) - for file_to_protect in sorted(lua_files): - print(f"::group::Checking {str(file_to_protect)}") - wiki = file_to_protect.parts[2] - module = "/".join(file_to_protect.parts[3:])[:-4] - page = "Module:" + module - if WIKI_TO_PROTECT: - if wiki == WIKI_TO_PROTECT: - protect_existing_page(page, wiki) - elif wiki == "commons": - protect_if_has_no_local_version(module, WIKI_TO_PROTECT) - elif wiki != "commons": - protect_existing_page(page, wiki) - else: # commons case - protect_existing_page(page, wiki) - for deploy_wiki in get_wikis() - {"commons"}: - protect_if_has_no_local_version(module, deploy_wiki) - print("::endgroup::") + lua_files = [pathlib.Path(arg) for arg in sys.argv[1:]] + + files_to_protect_by_wiki: dict[str, set[str]] = dict() + + for wiki, files_to_protect in itertools.groupby( + sorted(lua_files), lambda path: path.parts[2] + ): + files_to_protect_by_wiki[wiki] = set( + [ + "Module:" + "/".join(file_to_protect.parts[3:])[:-4] + for file_to_protect in files_to_protect + ] + ) + + new_commons_modules = files_to_protect_by_wiki.get("commons") + + if new_commons_modules: + for wiki in get_wikis(): + with MediaWikiSession(wiki) as session: + if wiki == "commons": + protect_existing_pages(session, new_commons_modules) + else: + new_local_modules = files_to_protect_by_wiki.get(wiki) + if new_local_modules: + protect_existing_pages(session, new_local_modules) + protect_non_existing_pages( + session, new_commons_modules - new_local_modules + ) + else: + protect_non_existing_pages(session, new_commons_modules) + else: + for wiki, new_modules in files_to_protect_by_wiki: + with MediaWikiSession(wiki) as session: + protect_existing_pages(session, new_modules) + handle_protect_errors() diff --git a/scripts/protect_page.py b/scripts/protect_page.py index 3d34c9a21ae..a5de8597e44 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -1,6 +1,6 @@ import time -from typing import Literal +from typing import Iterable, Literal import requests @@ -12,10 +12,13 @@ write_to_github_summary_file, ) from login_and_get_token import get_token +from mediawiki_session import MediaWikiSession, MediaWikiSessionError __all__ = [ "protect_non_existing_page", + "protect_non_existing_pages", "protect_existing_page", + "protect_existing_pages", "handle_protect_errors", ] @@ -64,6 +67,49 @@ def protect_page(page: str, wiki: str, protect_mode: Literal["edit", "create"]): protect_errors.append(f"{protect_mode}:{wiki}:{page}") +def protect_pages( + session: MediaWikiSession, + pages: Iterable[str], + protect_mode: Literal["edit", "create"], +): + protect_options: str + if protect_mode == "edit": + protect_options = "edit=allow-only-sysop|move=allow-only-sysop" + elif protect_mode == "create": + protect_options = "create=allow-only-sysop" + else: + raise ValueError(f"invalid protect mode: {protect_mode}") + print(f"...wiki = {session.wiki}") + for page in pages: + print(f"...page = {page}") + try: + protections = session.make_action( + "protect", + data={ + "title": page, + "protections": protect_options, + "reason": "Git maintained", + "expiry": "infinite", + "bot": "true", + "token": session.token, + }, + ) + for protection in protections: + if protection[protect_mode] == "allow-only-sysop": + return + print( + f"::warning::could not ({protect_mode}) protect {page} on {session.wiki}" + ) + protect_errors.append(f"{protect_mode}:{session.wiki}:{page}") + except MediaWikiSessionError as e: + print( + f"::warning::could not ({protect_mode}) protect {page} on {session.wiki}: {str(e)}" + ) + protect_errors.append(f"{protect_mode}:{session.wiki}:{page}") + finally: + session.cooldown() + + def check_if_page_exists(page: str, wiki: str) -> bool: with requests.Session() as session: session.cookies = read_cookie_jar(wiki) @@ -87,10 +133,32 @@ def protect_non_existing_page(page: str, wiki: str): protect_page(page, wiki, "create") +def protect_non_existing_pages(session: MediaWikiSession, pages: Iterable[str]): + def filter_non_existing_pages(page: str) -> bool: + try: + result = session.post( + "query", + data={"titles": page, "prop": "info"}, + ) + if "-1" in result["pages"]: + return True + print(f"::warning::{page} already exists on {session.wiki}") + protect_errors.append(f"create:{session.wiki}:{page}") + return False + finally: + time.sleep(SLEEP_DURATION) + + protect_pages(session, filter(filter_non_existing_pages, pages), "create") + + def protect_existing_page(page: str, wiki: str): protect_page(page, wiki, "edit") +def protect_existing_pages(session: MediaWikiSession, pages: Iterable[str]): + protect_pages(session, pages, "edit") + + def handle_protect_errors(): if len(protect_errors) == 0: return From 8e50dd657d2aec8782dfb82e79440d1823f55213 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:23:33 +0900 Subject: [PATCH 11/25] overhaul template protect --- scripts/protect_page.py | 76 ------------------------------------ scripts/protect_templates.py | 23 ++++++----- 2 files changed, 14 insertions(+), 85 deletions(-) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index a5de8597e44..7bac8ae638d 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -2,22 +2,15 @@ from typing import Iterable, Literal -import requests from deploy_util import ( - HEADER, SLEEP_DURATION, - get_wiki_api_url, - read_cookie_jar, write_to_github_summary_file, ) -from login_and_get_token import get_token from mediawiki_session import MediaWikiSession, MediaWikiSessionError __all__ = [ - "protect_non_existing_page", "protect_non_existing_pages", - "protect_existing_page", "protect_existing_pages", "handle_protect_errors", ] @@ -25,48 +18,6 @@ protect_errors = list() -def protect_page(page: str, wiki: str, protect_mode: Literal["edit", "create"]): - protect_options: str - if protect_mode == "edit": - protect_options = "edit=allow-only-sysop|move=allow-only-sysop" - elif protect_mode == "create": - protect_options = "create=allow-only-sysop" - else: - raise ValueError(f"invalid protect mode: {protect_mode}") - print(f"...wiki = {wiki}") - print(f"...page = {page}") - token = get_token(wiki) - with requests.Session() as session: - session.cookies = read_cookie_jar(wiki) - response = session.post( - get_wiki_api_url(wiki), - headers=HEADER, - params={"format": "json", "action": "protect"}, - data={ - "title": page, - "protections": protect_options, - "reason": "Git maintained", - "expiry": "infinite", - "bot": "true", - "token": token, - }, - ).json() - - time.sleep(SLEEP_DURATION) - if response.get("error"): - print( - f"::warning::could not ({protect_mode}) protect {page} on {wiki}: {response['error']['info']}" - ) - protect_errors.append(f"{protect_mode}:{wiki}:{page}") - return - protections = response["protect"].get("protections") - for protection in protections: - if protection[protect_mode] == "allow-only-sysop": - return - print(f"::warning::could not ({protect_mode}) protect {page} on {wiki}") - protect_errors.append(f"{protect_mode}:{wiki}:{page}") - - def protect_pages( session: MediaWikiSession, pages: Iterable[str], @@ -110,29 +61,6 @@ def protect_pages( session.cooldown() -def check_if_page_exists(page: str, wiki: str) -> bool: - with requests.Session() as session: - session.cookies = read_cookie_jar(wiki) - - result = session.post( - get_wiki_api_url(wiki), - headers=HEADER, - params={"format": "json", "action": "query"}, - data={"titles": page, "prop": "info"}, - ).json() - - time.sleep(SLEEP_DURATION) - return "-1" not in result["query"]["pages"] - - -def protect_non_existing_page(page: str, wiki: str): - if check_if_page_exists(page, wiki): - print(f"::warning::{page} already exists on {wiki}") - protect_errors.append(f"create:{wiki}:{page}") - else: - protect_page(page, wiki, "create") - - def protect_non_existing_pages(session: MediaWikiSession, pages: Iterable[str]): def filter_non_existing_pages(page: str) -> bool: try: @@ -151,10 +79,6 @@ def filter_non_existing_pages(page: str) -> bool: protect_pages(session, filter(filter_non_existing_pages, pages), "create") -def protect_existing_page(page: str, wiki: str): - protect_page(page, wiki, "edit") - - def protect_existing_pages(session: MediaWikiSession, pages: Iterable[str]): protect_pages(session, pages, "edit") diff --git a/scripts/protect_templates.py b/scripts/protect_templates.py index 488b93be20b..c26fa3fbfe8 100644 --- a/scripts/protect_templates.py +++ b/scripts/protect_templates.py @@ -1,8 +1,9 @@ import os +from mediawiki_session import MediaWikiSession from protect_page import ( - protect_non_existing_page, - protect_existing_page, + protect_non_existing_pages, + protect_existing_pages, handle_protect_errors, ) @@ -11,15 +12,19 @@ def main(): with open("./templates/templatesToProtect", "r") as templates_to_protect: - for template_name in templates_to_protect.read().splitlines(): - if len(template_name.strip()) == 0: - continue - template = "Template:" + template_name - print(f"::group::Checking {WIKI_TO_PROTECT}:{template}") + templates = [ + "Template:" + template_name + for template_name in filter( + lambda template: len(template.strip()) > 0, + templates_to_protect.read().splitlines(), + ) + ] + with MediaWikiSession(WIKI_TO_PROTECT) as session: + print(f"::group::Checking {WIKI_TO_PROTECT}") if WIKI_TO_PROTECT == "commons": - protect_existing_page(template, WIKI_TO_PROTECT) + protect_existing_pages(session, templates) else: - protect_non_existing_page(template, WIKI_TO_PROTECT) + protect_non_existing_pages(session, templates) print("::endgroup::") handle_protect_errors() From f12cdfca409d9bc83b7777306bee5dcea4983368 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:29:38 +0900 Subject: [PATCH 12/25] clean up unused --- scripts/deploy_util.py | 19 +--------- scripts/login_and_get_token.py | 64 ---------------------------------- scripts/mediawiki_session.py | 9 +++-- 3 files changed, 7 insertions(+), 85 deletions(-) delete mode 100644 scripts/login_and_get_token.py diff --git a/scripts/deploy_util.py b/scripts/deploy_util.py index e53e8c800fc..b2e617ec46a 100644 --- a/scripts/deploy_util.py +++ b/scripts/deploy_util.py @@ -1,6 +1,4 @@ -import contextlib import functools -import http.cookiejar import os import pathlib import subprocess @@ -12,16 +10,14 @@ "HEADER", "SLEEP_DURATION", "get_git_deploy_reason", - "get_wiki_api_url", "get_wikis", - "read_cookie_jar", "read_file_from_path", "write_to_github_summary_file", ] GITHUB_STEP_SUMMARY_FILE = os.getenv("GITHUB_STEP_SUMMARY") USER_AGENT = f"GitHub Autodeploy Bot/2.0.0 ({os.getenv('WIKI_UA_EMAIL')})" -WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") + HEADER = { "User-Agent": USER_AGENT, "accept": "application/json", @@ -41,11 +37,6 @@ def get_wikis() -> frozenset[str]: return frozenset(wikis["allwikis"].keys()) -@functools.cache -def get_wiki_api_url(wiki: str) -> str: - return f"{WIKI_BASE_URL}/{wiki}/api.php" - - def get_git_deploy_reason(): return ( subprocess.check_output(["git", "log", "-1", "--pretty='%h %s'"]) @@ -54,14 +45,6 @@ def get_git_deploy_reason(): ) -def read_cookie_jar(wiki: str) -> http.cookiejar.FileCookieJar: - ckf = f"cookie_{wiki}.ck" - cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) - with contextlib.suppress(OSError): - cookie_jar.load(ignore_discard=True) - return cookie_jar - - def read_file_from_path(file_path: pathlib.Path) -> str: with file_path.open("r") as file: return file.read() diff --git a/scripts/login_and_get_token.py b/scripts/login_and_get_token.py deleted file mode 100644 index d91b92afd10..00000000000 --- a/scripts/login_and_get_token.py +++ /dev/null @@ -1,64 +0,0 @@ -import functools -import os -import time - -import requests - -from deploy_util import HEADER, SLEEP_DURATION, get_wiki_api_url, read_cookie_jar - -__all__ = ["get_token"] - -WIKI_USER = os.getenv("WIKI_USER") -WIKI_PASSWORD = os.getenv("WIKI_PASSWORD") - -loggedin: set[str] = set() - - -def login(wiki: str): - if wiki in loggedin: - return - cookie_jar = read_cookie_jar(wiki) - print(f"...logging in on {wiki}") - with requests.Session() as session: - session.cookies = cookie_jar - token_response = session.post( - get_wiki_api_url(wiki), - headers=HEADER, - params={ - "format": "json", - "action": "query", - "meta": "tokens", - "type": "login", - }, - ).json() - session.post( - get_wiki_api_url(wiki), - headers=HEADER, - data={ - "lgname": WIKI_USER, - "lgpassword": WIKI_PASSWORD, - "lgtoken": token_response["query"]["tokens"]["logintoken"], - }, - params={"format": "json", "action": "login"}, - ) - loggedin.add(wiki) - cookie_jar.save(ignore_discard=True) - time.sleep(SLEEP_DURATION) - - -@functools.cache -def get_token(wiki: str) -> str: - login(wiki) - - with requests.Session() as session: - session.cookies = read_cookie_jar(wiki) - token_response = session.post( - get_wiki_api_url(wiki), - headers=HEADER, - params={ - "format": "json", - "action": "query", - "meta": "tokens", - }, - ).json() - return token_response["query"]["tokens"]["csrftoken"] diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 3240f0a23e6..3fd4f0ea835 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -12,8 +12,6 @@ from deploy_util import ( HEADER, SLEEP_DURATION, - WIKI_BASE_URL, - read_cookie_jar, write_to_github_summary_file, ) @@ -23,6 +21,7 @@ ] DEPLOY_TRIGGER = os.getenv("DEPLOY_TRIGGER") +WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") WIKI_USER = os.getenv("WIKI_USER") WIKI_PASSWORD = os.getenv("WIKI_PASSWORD") @@ -44,7 +43,11 @@ def __init__(self, wiki: str): self.__session.headers.update(HEADER) def __read_cookie_jar(self) -> http.cookiejar.FileCookieJar: - return read_cookie_jar(self.wiki) + ckf = f"cookie_{self.wiki}.ck" + cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) + with contextlib.suppress(OSError): + cookie_jar.load(ignore_discard=True) + return cookie_jar @functools.cache def __get_wiki_api_url(self): From dbc385e32e68868f90aa8a0067a5f7f27c63f6f6 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:31:09 +0900 Subject: [PATCH 13/25] fix indexing --- scripts/mediawiki_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 3fd4f0ea835..59a9b101139 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -62,7 +62,7 @@ def _login(self): data={ "lgname": WIKI_USER, "lgpassword": WIKI_PASSWORD, - "lgtoken": token_response["query"]["tokens"]["logintoken"], + "lgtoken": token_response["tokens"]["logintoken"], }, ) self.__cookie_jar.save(ignore_discard=True) From fbd5952cc3e13107d54d459672b371fb9483a081 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:43:18 +0900 Subject: [PATCH 14/25] fix calls --- scripts/protect_page.py | 2 +- scripts/remove_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index 7bac8ae638d..f33244ba17d 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -64,7 +64,7 @@ def protect_pages( def protect_non_existing_pages(session: MediaWikiSession, pages: Iterable[str]): def filter_non_existing_pages(page: str) -> bool: try: - result = session.post( + result = session.make_action( "query", data={"titles": page, "prop": "info"}, ) diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py index 4c8fa610b83..93a74a2c091 100644 --- a/scripts/remove_dev.py +++ b/scripts/remove_dev.py @@ -15,7 +15,7 @@ def remove_page(session: MediaWikiSession, page: str): print(f"deleting {session.wiki}:{page}") try: - session.post( + session.make_action( "delete", data={ "title": page, From 278e534f3a83bdf1192d6881084677f21f3e643e Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:45:21 +0900 Subject: [PATCH 15/25] fix key --- scripts/protect_page.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index f33244ba17d..de67edf9392 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -44,7 +44,8 @@ def protect_pages( "bot": "true", "token": session.token, }, - ) + )["protections"] + for protection in protections: if protection[protect_mode] == "allow-only-sysop": return From d55f4a27cf31cbbb55b1657e703f336c52b6920e Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:48:04 +0900 Subject: [PATCH 16/25] fix incorrect return --- scripts/protect_page.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index de67edf9392..5118e2f0cfe 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -45,14 +45,15 @@ def protect_pages( "token": session.token, }, )["protections"] - + for protection in protections: if protection[protect_mode] == "allow-only-sysop": - return - print( - f"::warning::could not ({protect_mode}) protect {page} on {session.wiki}" - ) - protect_errors.append(f"{protect_mode}:{session.wiki}:{page}") + break + else: + print( + f"::warning::could not ({protect_mode}) protect {page} on {session.wiki}" + ) + protect_errors.append(f"{protect_mode}:{session.wiki}:{page}") except MediaWikiSessionError as e: print( f"::warning::could not ({protect_mode}) protect {page} on {session.wiki}: {str(e)}" From 7c7dec3b32b1c92e48567fb4e029264c9fea38e9 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:59:18 +0900 Subject: [PATCH 17/25] adjust GHA output --- scripts/protect.py | 2 ++ scripts/protect_templates.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/protect.py b/scripts/protect.py index 9a0bb9ab4f1..25a0c3a01e3 100644 --- a/scripts/protect.py +++ b/scripts/protect.py @@ -34,8 +34,10 @@ def protect_new_wiki(wiki_to_protect: str): commons_modules.add(page) with MediaWikiSession(wiki_to_protect) as session: + print(f"::group::Protecting {WIKI_TO_PROTECT}") protect_non_existing_pages(session, commons_modules - local_modules) protect_existing_pages(session, local_modules) + print("::endgroup::") handle_protect_errors() diff --git a/scripts/protect_templates.py b/scripts/protect_templates.py index c26fa3fbfe8..2bf92cc487f 100644 --- a/scripts/protect_templates.py +++ b/scripts/protect_templates.py @@ -20,7 +20,7 @@ def main(): ) ] with MediaWikiSession(WIKI_TO_PROTECT) as session: - print(f"::group::Checking {WIKI_TO_PROTECT}") + print(f"::group::Protecting {WIKI_TO_PROTECT}") if WIKI_TO_PROTECT == "commons": protect_existing_pages(session, templates) else: From b800c6dce9c5bf17fed0f6ccd84735a9908371dd Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:12:22 +0900 Subject: [PATCH 18/25] add GHA debug support --- scripts/mediawiki_session.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 59a9b101139..dacf3bed601 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -20,6 +20,7 @@ "MediaWikiSessionError", ] +ACTIONS_STEP_DEBUG = os.getenv("ACTIONS_STEP_DEBUG") == "true" DEPLOY_TRIGGER = os.getenv("DEPLOY_TRIGGER") WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") WIKI_USER = os.getenv("WIKI_USER") @@ -71,9 +72,12 @@ def _login(self): @functools.cached_property def token(self) -> str: self._login() - return self.make_action("query", params={"meta": "tokens"})["tokens"][ + token = self.make_action("query", params={"meta": "tokens"})["tokens"][ "csrftoken" ] + if ACTIONS_STEP_DEBUG: + print(f"::add-mask::{token}") + return token @property def wiki(self) -> str: @@ -85,12 +89,18 @@ def make_action( merged_params = {"format": "json", "action": action} if params is not None: merged_params = merged_params | params - response: dict = self.__session.post( + response = self.__session.post( self.__get_wiki_api_url(), params=merged_params, data=data - ).json() - if "error" in response.keys(): - raise MediaWikiSessionError(response["error"]["info"]) - return response[action] + ) + if ACTIONS_STEP_DEBUG: + print(f"params: {merged_params}") + print(f"data: {data}") + print(f"HTTP Status: {response.status_code}") + print(f"Raw response: \"{response}\"") + parsed_response: dict[str, Any] = response.json() + if "error" in parsed_response.keys(): + raise MediaWikiSessionError(parsed_response["error"]["info"]) + return parsed_response[action] def cooldown(self): time.sleep(SLEEP_DURATION) From b5dbf00a8b910b1c739e3f596c53450964da3876 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:16:11 +0900 Subject: [PATCH 19/25] shorthand --- scripts/mediawiki_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index dacf3bed601..9133af71591 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -88,7 +88,7 @@ def make_action( ) -> dict[str, Any]: merged_params = {"format": "json", "action": action} if params is not None: - merged_params = merged_params | params + merged_params |= params response = self.__session.post( self.__get_wiki_api_url(), params=merged_params, data=data ) From a18c779d8ff7393ff552488ef606d8186bb9bba6 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:17:37 +0900 Subject: [PATCH 20/25] use cooldown --- scripts/protect_page.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index 5118e2f0cfe..3dd5299b9b0 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -3,10 +3,7 @@ from typing import Iterable, Literal -from deploy_util import ( - SLEEP_DURATION, - write_to_github_summary_file, -) +from deploy_util import write_to_github_summary_file from mediawiki_session import MediaWikiSession, MediaWikiSessionError __all__ = [ @@ -76,7 +73,7 @@ def filter_non_existing_pages(page: str) -> bool: protect_errors.append(f"create:{session.wiki}:{page}") return False finally: - time.sleep(SLEEP_DURATION) + session.cooldown() protect_pages(session, filter(filter_non_existing_pages, pages), "create") From 266504d9abbde3c2268765a993642369ef3e4d3d Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:17:45 +0900 Subject: [PATCH 21/25] lint --- scripts/mediawiki_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 9133af71591..b7b80ab6e5a 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -96,7 +96,7 @@ def make_action( print(f"params: {merged_params}") print(f"data: {data}") print(f"HTTP Status: {response.status_code}") - print(f"Raw response: \"{response}\"") + print(f'Raw response: "{response}"') parsed_response: dict[str, Any] = response.json() if "error" in parsed_response.keys(): raise MediaWikiSessionError(parsed_response["error"]["info"]) From 7c88fe12413dc79ff282dfdb9cbd934064e2f45a Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:21:17 +0900 Subject: [PATCH 22/25] lint --- scripts/protect_page.py | 2 -- scripts/remove_dev.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index 3dd5299b9b0..e915cbe436d 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -1,5 +1,3 @@ -import time - from typing import Iterable, Literal diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py index 93a74a2c091..8deec4dba4b 100644 --- a/scripts/remove_dev.py +++ b/scripts/remove_dev.py @@ -23,7 +23,7 @@ def remove_page(session: MediaWikiSession, page: str): "token": session.token, }, ) - except MediaWikiSessionError as e: + except MediaWikiSessionError: print(f"::warning::could not delete {page} on {session.wiki}") write_to_github_summary_file( f":warning: could not delete {page} on {session.wiki}" From d914c9ea64cce8d6b0da9c164bd2f609bb223625 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Mon, 9 Mar 2026 18:56:32 +0900 Subject: [PATCH 23/25] sort --- scripts/protect_page.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index e915cbe436d..55e031dde71 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -26,7 +26,7 @@ def protect_pages( else: raise ValueError(f"invalid protect mode: {protect_mode}") print(f"...wiki = {session.wiki}") - for page in pages: + for page in sorted(pages): print(f"...page = {page}") try: protections = session.make_action( From a80f0c803a3663e759c80da54250de415a18b1b7 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Mon, 9 Mar 2026 19:12:48 +0900 Subject: [PATCH 24/25] group messages --- scripts/protect_page.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index 55e031dde71..a7ff00b9584 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -25,7 +25,7 @@ def protect_pages( protect_options = "create=allow-only-sysop" else: raise ValueError(f"invalid protect mode: {protect_mode}") - print(f"...wiki = {session.wiki}") + print(f"::group::Protecting {session.wiki}") for page in sorted(pages): print(f"...page = {page}") try: @@ -56,6 +56,7 @@ def protect_pages( protect_errors.append(f"{protect_mode}:{session.wiki}:{page}") finally: session.cooldown() + print("::endgroup::") def protect_non_existing_pages(session: MediaWikiSession, pages: Iterable[str]): From ea561508d6f19c227c3b5f6313c1b4c96bf722bf Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Mon, 9 Mar 2026 19:13:01 +0900 Subject: [PATCH 25/25] sort --- scripts/protect.py | 2 +- scripts/remove_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/protect.py b/scripts/protect.py index 25a0c3a01e3..fa2455abf21 100644 --- a/scripts/protect.py +++ b/scripts/protect.py @@ -67,7 +67,7 @@ def main(): new_commons_modules = files_to_protect_by_wiki.get("commons") if new_commons_modules: - for wiki in get_wikis(): + for wiki in sorted(get_wikis()): with MediaWikiSession(wiki) as session: if wiki == "commons": protect_existing_pages(session, new_commons_modules) diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py index 8deec4dba4b..8b0a80ecd95 100644 --- a/scripts/remove_dev.py +++ b/scripts/remove_dev.py @@ -68,7 +68,7 @@ def search_and_remove(wiki: str): def main(): - for wiki in get_wikis(): + for wiki in sorted(get_wikis()): if wiki == "commons" and os.getenv("INCLUDE_COMMONS") != "true": continue search_and_remove(wiki)