Merge pull request #40 from freelw/wangli_dev_20250617_1

update tools

Author: freelw

.gitignore

@@ -14,3 +14,9 @@ vocab/fra_vocab_builder/tgt_vocab.txt
 vocab/fra_vocab_builder/src_vocab.txt
 backends/gpu/metal/metal-cpp/.DS_Store
 handwritten_recognition
+tools/vocab_builder/vocab.txt
+tools/vocab_builder/builder.dSYM/Contents/Resources/DWARF/builder
+tools/vocab_builder/builder.dSYM/Contents/Resources/Relocations/aarch64/builder.yml
+tools/vocab_builder/builder.dSYM/Contents/Info.plist
+tools/vocab_builder/builder
+tools/vocab_builder/builder.dSYM

tools/pre_process/preprocess.py

@@ -0,0 +1,14 @@
+import re
+
+def get_timemachine():
+    with open("./timemachine.txt") as f:
+        return f.read()
+
+def preprocess(text):
+    """Defined in :numref:`sec_text-sequence`"""
+    return re.sub('[^A-Za-z]+', ' ', text).lower()
+
+if '__main__' == __name__:
+
+    with open("./timemachine_preprocessed.txt", "w") as f:
+        f.write(preprocess(get_timemachine()))