From 93f1ee8a1c846a257c3b33d3998268e23b87adfd Mon Sep 17 00:00:00 2001 From: Google Team Member Date: Wed, 18 Mar 2026 03:06:00 -0700 Subject: [PATCH] No public description PiperOrigin-RevId: 885483365 --- README.md | 6 +++++- google/genai/local_tokenizer.py | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e409383c8..2e4034846 100644 --- a/README.md +++ b/README.md @@ -1132,13 +1132,17 @@ print(response) #### Local Count Tokens ```python -tokenizer = genai.LocalTokenizer(model_name='gemini-2.5-flash') +from google.genai.local_tokenizer import LocalTokenizer + +tokenizer = LocalTokenizer(model_name='gemini-2.5-flash') result = tokenizer.count_tokens("What is your name?") ``` #### Local Compute Tokens ```python +from google.genai.local_tokenizer import LocalTokenizer + tokenizer = genai.LocalTokenizer(model_name='gemini-2.5-flash') result = tokenizer.compute_tokens("What is your name?") ``` diff --git a/google/genai/local_tokenizer.py b/google/genai/local_tokenizer.py index 5db6d6922..ff65ba263 100644 --- a/google/genai/local_tokenizer.py +++ b/google/genai/local_tokenizer.py @@ -317,8 +317,8 @@ def count_tokens( .. code-block:: python - from google import genai - tokenizer = genai.LocalTokenizer(model_name='gemini-2.0-flash-001') + from google.genai.local_tokenizer import LocalTokenizer + tokenizer = LocalTokenizer(model_name='gemini-2.0-flash-001') result = tokenizer.count_tokens("What is your name?") print(result) # total_tokens=5 @@ -358,8 +358,8 @@ def compute_tokens( .. code-block:: python - from google import genai - tokenizer = genai.LocalTokenizer(model_name='gemini-2.0-flash-001') + from google.genai.local_tokenizer import LocalTokenizer + tokenizer = LocalTokenizer(model_name='gemini-2.0-flash-001') result = tokenizer.compute_tokens("What is your name?") print(result) # tokens_info=[TokensInfo(token_ids=[279, 329, 1313, 2508, 13], tokens=[b' What', b' is', b' your', b' name', b'?'], role='user')]