Skip to content

Commit 5fdd7ee

Browse files
[pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
1 parent 856a601 commit 5fdd7ee

File tree

1 file changed

+9
-2
lines changed

1 file changed

+9
-2
lines changed

computer_vision/vision_transformer.py

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,10 @@ def add_positional_encoding(
158158

159159

160160
def attention_mechanism(
161-
query: np.ndarray, key: np.ndarray, value: np.ndarray, mask: Optional[np.ndarray] = None
161+
query: np.ndarray,
162+
key: np.ndarray,
163+
value: np.ndarray,
164+
mask: Optional[np.ndarray] = None,
162165
) -> tuple[np.ndarray, np.ndarray]:
163166
"""
164167
Compute scaled dot-product attention.
@@ -265,7 +268,11 @@ def feedforward_network(x: np.ndarray, hidden_dim: int = 3072) -> np.ndarray:
265268
hidden = x @ w1 + b1
266269

267270
# GELU activation (approximation)
268-
hidden = 0.5 * hidden * (1 + np.tanh(np.sqrt(2 / np.pi) * (hidden + 0.044715 * hidden**3)))
271+
hidden = (
272+
0.5
273+
* hidden
274+
* (1 + np.tanh(np.sqrt(2 / np.pi) * (hidden + 0.044715 * hidden**3)))
275+
)
269276

270277
# Second linear layer
271278
w2 = np.random.randn(hidden_dim, embedding_dim) * 0.02

0 commit comments

Comments
 (0)