fixup! Move rewriting stuff from warc2zim to zimscraperlib

Author: benoit74

.github/workflows/Publish.yaml

@@ -1,4 +1,4 @@
-name: Build and publish to PyPI / NPM
+name: Build and publish to PyPI
 
 on:
   release:

.pre-commit-config.yaml

@@ -3,7 +3,7 @@
 exclude: ^tests/files # these are raw test files, no need to mess with them
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -12,11 +12,11 @@ repos:
     hooks:
       - id: black
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.9
+    rev: v0.7.0
     hooks:
       - id: ruff
   - repo: https://github.com/RobertCraigie/pyright-python
-    rev: v1.1.368
+    rev: v1.1.385
     hooks:
       - id: pyright
         name: pyright (system)

openzim.toml

@@ -9,7 +9,14 @@ action="get_file"
 source="https://cdn.jsdelivr.net/npm/@webrecorder/wombat@3.8.2/dist/wombat.js"
 target_file="wombat.js"
 
-[files.assets.actions."wombatSetup.js"] # fallback if this script has not been properly built
+# wombatSetup.js is supposed to be built locally from files in javascript folder.
+# Should someone not have proper skills / tooling / knowledge, or simply install from
+# sdist / Github repo directly, without any advanced knowledge of this specificity, the
+# configuration below ensures that wombatSetup.js is downloaded from dev.kiwix.org,
+# where we have the latest version from `main` branch. wheel contains the wombatSetup.js
+# which was built at the same time than the wheel. (reminder: get_file action does not
+# overwrite a file which already exists)
+[files.assets.actions."wombatSetup.js"]
 action="get_file"
 source="https://dev.kiwix.org/zimscraperlib/wombatSetup.js"
 target_file="wombatSetup.js"

pyproject.toml

@@ -52,20 +52,20 @@ scripts = [
 ]
 lint = [
   "black==24.10.0",
-  "ruff==0.4.9",
+  "ruff==0.7.0",
 ]
 check = [
-  "pyright==1.1.368",
-  "pytest==8.2.2",
+  "pyright==1.1.385",
+  "pytest==8.3.3",
 ]
 test = [
-  "pytest==8.2.2",
+  "pytest==8.3.3",
   "pytest-mock==3.14.0",
   "coverage==7.5.3",
 ]
 dev = [
   "ipython==8.25.0",
-  "pre-commit==3.7.1",
+  "pre-commit==4.0.1",
   "zimscraperlib[scripts]",
   "zimscraperlib[lint]",
   "zimscraperlib[test]",

rules/rules.yaml

@@ -1,12 +1,13 @@
 # This file comes from an adaptation of rules present in
-# https://github.com/webrecorder/wabac.js/blame/main/src/fuzzymatcher.js
+# https://github.com/webrecorder/wabac.js/blob/main/src/fuzzymatcher.ts
 #
 # Syncing rules is done manually, based on expert knowledge, especially because in
 # scraperlib we are not really fuzzy matching (searching the best entry among existing
 # ones) but just rewriting to proper path.
 #
-# This file is in sync with content at commit 879018d5b96962df82340a9a57570bbc0fc67815
-# from June 9, 2024
+# This file is in sync with content at commit
+# https://github.com/webrecorder/wabac.js/commit/1c3acfce39e0dc127acf455b04237e9a82062730
+# from October 17, 2024
 #
 # This file should be updated at every release of scraperlib
 #
@@ -27,12 +28,12 @@ fuzzyRules:
       - raw_url: foobargooglevideo.com/videoplayback?some=thing&id=1576&key=value
         fuzzified_url: youtube.fuzzy.replayweb.page/videoplayback?id=1576
       - raw_url: foobargooglevideo.com/videoplaybackandfoo?some=thing&id=1576&key=value
-        unchanged: true  # videoplayback is not followed by `?`
+        unchanged: true # videoplayback is not followed by `?`
       - raw_url: foobargoogle_video.com/videoplaybackandfoo?some=thing&id=1576&key=value
-        unchanged: true  # No googlevideo.com in url
+        unchanged: true # No googlevideo.com in url
   - name: youtube_video_info
     pattern: (?:www\.)?youtube(?:-nocookie)?\.com/(get_video_info\?).*(video_id=[^&]+).*
-    replace : youtube.fuzzy.replayweb.page/\1\2
+    replace: youtube.fuzzy.replayweb.page/\1\2
     tests:
       - raw_url: www.youtube.com/get_video_info?video_id=123ah
         fuzzified_url: youtube.fuzzy.replayweb.page/get_video_info?video_id=123ah
@@ -52,7 +53,7 @@ fuzzyRules:
         unchanged: true # improper hostname
   - name: youtube_thumbnails
     pattern: i\.ytimg\.com\/vi\/(.*?)\/.*?\.(\w*?)(?:\?.*|$)
-    replace : i.ytimg.com.fuzzy.replayweb.page/vi/\1/thumbnail.\2
+    replace: i.ytimg.com.fuzzy.replayweb.page/vi/\1/thumbnail.\2
     tests:
       - raw_url: i.ytimg.com/vi/-KpLmsAR23I/maxresdefault.jpg?sqp=-oaymwEmCIAKENAF8quKqQMa8AEB-AH-CYAC0AWKAgwIABABGHIgTyg-MA8=&rs=AOn4CLDr-FmDmP3aCsD84l48ygBmkwHg-g
         fuzzified_url: i.ytimg.com.fuzzy.replayweb.page/vi/-KpLmsAR23I/thumbnail.jpg
@@ -64,7 +65,7 @@ fuzzyRules:
         fuzzified_url: i.ytimg.com.fuzzy.replayweb.page/vi/-KpLmsAR23I/thumbnail.jpg
   - name: trim_digits_only
     pattern: ([^?]+)\?[\d]+$
-    replace : \1
+    replace: \1
     tests:
       - raw_url: www.example.com/page?1234
         fuzzified_url: www.example.com/page
@@ -80,7 +81,7 @@ fuzzyRules:
         unchanged: true
   - name: youtubei
     pattern: (?:www\.)?youtube(?:-nocookie)?\.com\/(youtubei\/[^?]+).*(videoId[^&]+).*
-    replace : youtube.fuzzy.replayweb.page/\1?\2
+    replace: youtube.fuzzy.replayweb.page/\1?\2
     tests:
       - raw_url: www.youtube-nocookie.com/youtubei/page/?videoId=123ah
         fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/?videoId=123ah
@@ -104,7 +105,7 @@ fuzzyRules:
         unchanged: true
   - name: youtube_embed
     pattern: (?:www\.)?youtube(?:-nocookie)?\.com/embed/([^?]+).*
-    replace : youtube.fuzzy.replayweb.page/embed/\1
+    replace: youtube.fuzzy.replayweb.page/embed/\1
     tests:
       - raw_url: www.youtube-nocookie.com/embed/foo
         fuzzified_url: youtube.fuzzy.replayweb.page/embed/foo
@@ -123,7 +124,7 @@ fuzzyRules:
 
   - name: vimeo_cdn_fix # custom warc2zim rule intended to fix Vimeo support
     pattern: .*(?:gcs-vimeo|vod|vod-progressive|vod-adaptive)\.akamaized\.net.*\/(.+?.mp4)\?.*range=(.*?)(?:&.*|$)
-    replace : vimeo-cdn.fuzzy.replayweb.page/\1?range=\2
+    replace: vimeo-cdn.fuzzy.replayweb.page/\1?range=\2
     tests:
       - raw_url: gcs-vimeo.akamaized.net/123.mp4?range=123-456
         fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/123.mp4?range=123-456
@@ -151,7 +152,7 @@ fuzzyRules:
         unchanged: true
   - name: vimeo_cdn
     pattern: .*(?:gcs-vimeo|vod|vod-progressive)\.akamaized\.net.*?\/([\d/]+.mp4)$
-    replace : vimeo-cdn.fuzzy.replayweb.page/\1
+    replace: vimeo-cdn.fuzzy.replayweb.page/\1
     tests:
       - raw_url: vod.akamaized.net/23.mp4
         fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/23.mp4
@@ -161,7 +162,7 @@ fuzzyRules:
         fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/01/4423/13/347119375/1398505169.mp4
   - name: vimeo_player
     pattern: .*player.vimeo.com\/(video\/[\d]+)\?.*
-    replace : vimeo.fuzzy.replayweb.page/\1
+    replace: vimeo.fuzzy.replayweb.page/\1
     tests:
       - raw_url: player.vimeo.com/video/1234?foo=bar
         fuzzified_url: vimeo.fuzzy.replayweb.page/video/1234
@@ -177,15 +178,15 @@ fuzzyRules:
         unchanged: true
   - name: i_vimeo_cdn
     pattern: .*i\.vimeocdn\.com\/(.*)\?.*
-    replace : i.vimeocdn.fuzzy.replayweb.page/\1
+    replace: i.vimeocdn.fuzzy.replayweb.page/\1
     tests:
       - raw_url: i.vimeocdn.com/image/1234?foo=bar
         fuzzified_url: i.vimeocdn.fuzzy.replayweb.page/image/1234
       - raw_url: i.vimeocdn.com/something/a456?foo
         fuzzified_url: i.vimeocdn.fuzzy.replayweb.page/something/a456
   - name: cheatography_com
     pattern: cheatography\.com\/scripts\/(.*).js.*[?&](v=[^&]+).*
-    replace : cheatography.com.fuzzy.replayweb.page/scripts/\1.js?\2
+    replace: cheatography.com.fuzzy.replayweb.page/scripts/\1.js?\2
     tests:
       - raw_url: cheatography.com/scripts/useful.min.js?v=2&q=1719438924
         fuzzified_url: cheatography.com.fuzzy.replayweb.page/scripts/useful.min.js?v=2