From 1c39b2fe132ecbcdca4f0a8d1cb2c3d80ea1f8b7 Mon Sep 17 00:00:00 2001 From: Arsalan Uddin Date: Thu, 9 Mar 2023 13:15:33 +0000 Subject: [PATCH 01/11] adding packed bert from optimum-main --- packed-bert/models/__init__.py | 0 packed-bert/models/modeling_bert_packed.py | 279 ++++ ...BERT_multi_label_text_classification.ipynb | 1234 +++++++++++++++ .../packedBERT_question_answering.ipynb | 1133 ++++++++++++++ ...ERT_single_label_text_classification.ipynb | 1375 +++++++++++++++++ packed-bert/utils/__init__.py | 0 packed-bert/utils/packing/__init__.py | 0 packed-bert/utils/packing/algorithms.py | 225 +++ packed-bert/utils/packing/dataset_creator.py | 299 ++++ .../utils/packing/dataset_templates.py | 100 ++ packed-bert/utils/packing/qa_utils.py | 243 +++ 11 files changed, 4888 insertions(+) create mode 100644 packed-bert/models/__init__.py create mode 100644 packed-bert/models/modeling_bert_packed.py create mode 100644 packed-bert/packedBERT_multi_label_text_classification.ipynb create mode 100644 packed-bert/packedBERT_question_answering.ipynb create mode 100644 packed-bert/packedBERT_single_label_text_classification.ipynb create mode 100644 packed-bert/utils/__init__.py create mode 100644 packed-bert/utils/packing/__init__.py create mode 100644 packed-bert/utils/packing/algorithms.py create mode 100644 packed-bert/utils/packing/dataset_creator.py create mode 100644 packed-bert/utils/packing/dataset_templates.py create mode 100644 packed-bert/utils/packing/qa_utils.py diff --git a/packed-bert/models/__init__.py b/packed-bert/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packed-bert/models/modeling_bert_packed.py b/packed-bert/models/modeling_bert_packed.py new file mode 100644 index 0000000..023b634 --- /dev/null +++ b/packed-bert/models/modeling_bert_packed.py @@ -0,0 +1,279 @@ +# Copyright (c) 2023 Graphcore Ltd. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Tuple, Union + +import torch +import torch.nn as nn + +import poptorch +from optimum.graphcore.models.bert.modeling_bert import BertPipelineMixin +from transformers import BertForQuestionAnswering, BertForSequenceClassification +from transformers.modeling_outputs import QuestionAnsweringModelOutput + + +class PackedBertPooler(nn.Module): + def __init__(self, config): + super().__init__() + self.max_seq_per_pack = config.max_sequences_per_pack + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + """ + We "pool" the model by simply taking the hidden states corresponding + to the last max_sequences_per_pack tokens. Note that the [CLS] tokens + are always located at the end of the pack. When the actual number of + sequences is lower than max_sequences_per_pack, we still slice out + the last max_sequences_per_pack tokens, but we will not use all of + them during loss calculation. + """ + sh = hidden_states.shape + last_tokens_tensors = hidden_states[:, -self.max_seq_per_pack :] + last_reshape = last_tokens_tensors.reshape(sh[0] * self.max_seq_per_pack, sh[2]) + # output size: [bs x max_sequences_per_pack, hidden_size] + output = self.dense(last_reshape) + output = self.activation(output) + + return output + + +class PackedBertOutputsForMultiLabel(nn.Module): + """ + This class handles the custom model output phase for multi-label sequence classification. + """ + + def __init__(self, config): + super().__init__() + self.max_seq_per_pack = config.max_sequences_per_pack + self.multi_loss = torch.nn.BCEWithLogitsLoss(reduction="none") + + def forward( + self, + outputs: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor], + batch_dim: int, + labels: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor]: + max_labels = torch.max(attention_mask[:, : -self.max_seq_per_pack], dim=-1).values.unsqueeze(1) + + # Create a mask corresponding to actual number of seqs in pack, to mask padding + label_mask = torch.arange(0, self.max_seq_per_pack).unsqueeze(0).repeat(batch_dim, 1) + label_mask = torch.where( + label_mask < max_labels, + torch.ones(batch_dim, self.max_seq_per_pack), + torch.zeros(batch_dim, self.max_seq_per_pack), + ) + label_mask = label_mask.view(-1).unsqueeze(1) + + # Adjust logits to rule out padding + logits = label_mask * outputs.logits + + loss = None + if labels is not None: + # Flatten and adjust labels to rule out padding + labels = labels.view(-1, *(labels.size()[2:])).to(torch.float32) + labels = label_mask * labels + + # Adjust the loss to rule out the padding and CLS logits + loss = self.multi_loss(logits, labels) + loss *= label_mask + + # Take mean over each multi-class pred + loss = torch.sum(loss) / (torch.sum(max_labels) * labels.shape[-1]) + loss = poptorch.identity_loss(loss, reduction="none") + + logits = logits.reshape([batch_dim, self.max_seq_per_pack, logits.shape[-1]]) + + return (loss, logits) + else: + return logits + + +class PipelinedPackedBertForSequenceClassification(BertForSequenceClassification, BertPipelineMixin): + """ + This class supports doing single-label/multi-label sequence-classification tasks with custom outputs. + The problem_type must be passed to differentiate the two methods - multi_label_classification or single_label_classification. Multi-label requires a custom loss implementation to mask labels and logits, unlike single-label. + + In both cases: + * The logits need to be reshaped at output to revert them from the 'unpacked' batch dimension to a batch dimension equivalent to that of the labels passed to the model in order for Optimum's trainer class to perform evaluation. + + * The attention mask is reshaped from the 'packed' attention mask to an equivalent binary 3D "extended" attention mask for BERT to recognise the sequences within a single packed input as unrelated sequences. + """ + + def __init__(self, config): + super().__init__(config) + self.max_seq_per_pack = config.max_sequences_per_pack + self.problem_type = config.problem_type + self.num_labels = config.num_labels + + self.bert.pooler = PackedBertPooler(config) + self.multi_label_outputs = PackedBertOutputsForMultiLabel(config) + + def parallelize(self): + super().parallelize() + last_ipu = self.ipu_config.ipus_per_replica - 1 + self.classifier = poptorch.BeginBlock(self.classifier, "Classifier Output", ipu_id=last_ipu) + return self + + def forward( + self, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + token_type_ids: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + labels: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Tuple[torch.Tensor]: + bs = input_ids.shape[0] + seq_len = input_ids.shape[1] + + attention_mask_3d = attention_mask[:, None, :].repeat(1, seq_len, 1) + attention_mask_3d = (attention_mask_3d == attention_mask_3d.transpose(1, 2)) * (attention_mask_3d != 0) + + # Manual masking of logits and loss only needed for multi-label, single-label loss allows ignore_index + output = super().forward( + input_ids=input_ids, + attention_mask=attention_mask_3d, + token_type_ids=token_type_ids, + position_ids=position_ids, + labels=labels if labels is not None and self.problem_type == "single_label_classification" else None, + ) + + if self.problem_type == "single_label_classification": + if labels is not None: + logits = output.logits.reshape([-1, self.max_seq_per_pack, self.num_labels]) + output.logits = logits + + else: + output = self.multi_label_outputs( + outputs=output, attention_mask=attention_mask, batch_dim=bs, labels=labels + ) + + return output + + +class PackedBertOutputsForQA(nn.Module): + """ + This class handles the custom output phase for a question-answering task. + """ + + def __init__(self, config): + super().__init__() + # Use the default QA model output formatting class to return outputs in the same form as the base model. + self.output = QuestionAnsweringModelOutput + self.max_sequences_per_pack = config.max_sequences_per_pack + + def forward( + self, + final_layer_output: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + start_positions: Optional[torch.Tensor] = None, + end_positions: Optional[torch.Tensor] = None, + ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: + # Create unpacking mask to separate packed logits out into sequence-specific logits only + unpacking_mask = attention_mask[:, None, :].repeat(1, self.max_sequences_per_pack, 1) + pack_seq_ids = torch.arange(1, self.max_sequences_per_pack + 1).view(self.max_sequences_per_pack, 1) + + unpacking_mask = unpacking_mask == pack_seq_ids + + # Expand start logits using mask to isolate logits for each internal sequence in the pack + unpacked_start_logits = final_layer_output.start_logits[:, None, :] * unpacking_mask + unpacked_end_logits = final_layer_output.end_logits[:, None, :] * unpacking_mask + + # Calculate loss on logits/labels with initial [bs, mspp, ...] dims collapsed into one [bs*mspp, ...] + total_loss = None + if start_positions is not None and end_positions is not None: + start_positions = start_positions.view(-1) + end_positions = end_positions.view(-1) + + unpacked_start_logits = unpacked_start_logits.contiguous() + unpacked_end_logits = unpacked_end_logits.contiguous() + + unpacked_start_logits = unpacked_start_logits.view(-1, unpacked_start_logits.shape[-1]) + unpacked_end_logits = unpacked_end_logits.view(-1, unpacked_end_logits.shape[-1]) + + loss_fct = nn.CrossEntropyLoss() + start_loss = loss_fct(unpacked_start_logits, start_positions) + end_loss = loss_fct(unpacked_end_logits, end_positions) + + total_loss = (start_loss + end_loss) / 2 + + return self.output( + loss=total_loss, + start_logits=unpacked_start_logits, + end_logits=unpacked_end_logits, + hidden_states=final_layer_output.hidden_states, + attentions=final_layer_output.attentions, + ) + + +class PipelinedPackedBertForQuestionAnswering(BertForQuestionAnswering, BertPipelineMixin): + """ + This class extends BertForQuestionAnswering with some differences required for packing. The 'packed' attention mask must be extended to a 3D binary "extended" attention mask for BERT to recognise the sequences within a single packed input as unrelated sequences. The output is extended to enable masking for padded labels, and then 'unpacking' the packed hidden state output before performing the loss calculation. + """ + + def __init__(self, config): + super().__init__(config) + self.max_seq_per_pack = self.config.max_sequences_per_pack + self.packed_outputs = PackedBertOutputsForQA(config) + + def parallelize(self): + super().parallelize() + last_ipu = self.ipu_config.ipus_per_replica - 1 + self.qa_outputs = poptorch.BeginBlock(self.qa_outputs, "QA Outputs", ipu_id=last_ipu) + return self + + def forward( + self, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + token_type_ids: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + start_positions: Optional[torch.Tensor] = None, + end_positions: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Tuple[torch.Tensor]: + # Create 3D attention mask for sequence specific attention in pack + seq_len = input_ids.shape[1] + packed_attention_mask = attention_mask[:, None, :].repeat(1, seq_len, 1) + packed_attention_mask = (packed_attention_mask == packed_attention_mask.transpose(1, 2)) * ( + packed_attention_mask != 0 + ) + + # Run forwards pass through model without labels + final_layer_output = super().forward( + input_ids, attention_mask=packed_attention_mask, token_type_ids=token_type_ids, position_ids=position_ids + ) + + # Custom PackedBert for SQuAD output, redirect from before loss function in transformers model class. + output = self.packed_outputs( + final_layer_output, + attention_mask=attention_mask, + start_positions=start_positions, + end_positions=end_positions, + ) + + if start_positions is not None and end_positions is not None: + return poptorch.identity_loss(output.loss, reduction="mean"), output.start_logits, output.end_logits + else: + return output.start_logits, output.end_logits diff --git a/packed-bert/packedBERT_multi_label_text_classification.ipynb b/packed-bert/packedBERT_multi_label_text_classification.ipynb new file mode 100644 index 0000000..1a27389 --- /dev/null +++ b/packed-bert/packedBERT_multi_label_text_classification.ipynb @@ -0,0 +1,1234 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "X4cRE8IbIrIV" + }, + "source": [ + "First of all, make sure your environment has installed the latest version of [πŸ€— Optimum Graphcore](https://github.com/huggingface/optimum-graphcore)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "MOsHUjgdIrIW", + "outputId": "f84a093e-147f-470e-aad9-80fb51193c8e", + "scrolled": false + }, + "outputs": [], + "source": [ + "%pip install git+https://github.com/huggingface/optimum-graphcore.git" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Also make sure all the packages required for this notebook are installed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "%pip install scikit-learn;\n", + "%pip install datasets\n", + "%pip install evaluate\n", + "%pip install tokenizers\n", + "%pip install matplotlib\n", + "%pip install scipy\n", + "%pip install --force-reinstall huggingface_hub==0.11.1;" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's start by importing and printing out the versions of `Transformers` and `Optimum Graphcore`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import transformers\n", + "import optimum.graphcore\n", + "\n", + "print(transformers.__version__)\n", + "print(optimum.graphcore.__version__)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "At the end of this notebook, to be able to share your model with the community and easily access it through HuggingFace, there are some short set-up steps you must follow to enable uploading your checkpoint to the HuggingFace Hub.\n", + "\n", + "First you have to store your authentication token from the Hugging Face website ([sign up here](https://huggingface.co/join) if you haven't already!) then execute the following cell and input your username and password:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from huggingface_hub import notebook_login\n", + "\n", + "notebook_login()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Git-lfs must also be installed to enable large file storage when pushing to the hub:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!apt install git-lfs" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rEJBSTyZIrIb" + }, + "source": [ + "# Faster multi-label text classification with PackedBERT" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook builds on the process of fine-tuning BERT on a [text classification task](text_classification.ipynb) showing how to implement packing for BERT for multi-label classification. [Packing](https://www.graphcore.ai/posts/introducing-packed-bert-for-2x-faster-training-in-natural-language-processing) is an optimisation method originally used for 2x faster BERT pre-training, which can now also provide massive throughput increases for **fine-tuning** and **batched inference**! \n", + "\n", + "**So, what *is* packing?** The basic idea of 'packing' a dataset is to utilise the requirement for constant-shaped inputs into a model. Instead of padding it with empty, unused space, we can recycle this unused space and fill it with more inputs! The architecture of transformer models like BERT supports this, and lets us optimally use this space to process multiple sequences within one input.\n", + "\n", + "**And here is why you might want to use it:** Having a single input contain multiple sequences leads to multiple sequences being processed in parallel in a single pass within a single iteration inside a batch, increasing the 'effective' batch size of the model by a considerable factor in many cases, and most importantly, increasing model throughput for training and batched inference significantly.\n", + "\n", + "The [GoEmotions](https://ai.googleblog.com/2021/10/goemotions-dataset-for-fine-grained.html) dataset will be fine-tuned using packing. This notebook outlines how to easily enable packing for BERT when performing fine-tuning/inference on a text-classification task in πŸ€— Optimum, resulting in an impressive 5-9x faster training and inference run-time for the dataset. \n", + "\n", + "You can read more about packing in the original [paper](https://arxiv.org/abs/2107.02027)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "![GoEmotions dataset (Source: GoogleBlog)](../images/go_emotions.png)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The dataset consists of 58k comments labelled for 27 different emotion categories (and a 28th \"neutral\" category). This dataset is used for multi-label, multi-class classification. The dataset format and categories can be viewed on the [Huggingface Hub](https://huggingface.co/datasets/go_emotions)." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's initialise our training configurations. \n", + "\n", + "In this notebook, we are using both data parallelism and pipeline parallelism (see this [tutorial](https://github.com/graphcore/tutorials/tree/master/tutorials/pytorch/tut2_efficient_data_loading) for more). Therefore the global batch size, which is the actual number of samples used for the weight update, is determined using four factors:\n", + "\n", + " global batch size = micro_batch_size * gradient accumulation steps * device iterations * replication factor\n", + "\n", + "and replication factor is determined by pod_type, which will be used as a key to select the replication factor from a dictionary defined in the IPU config file. For example, the dictionary in the IPU config file Graphcore/roberta-base-ipu looks like this:\n", + "\n", + " \"replication_factor\": {\"pod4\": 1, \"pod8\": 2, \"pod16\": 4, \"pod32\": 8, \"pod64\": 16, \"default\": 1}\n", + "\n", + "Depending on your model and the pod machine you are using, you might need to adjust these three batch-size-related arguments.\n", + "\n", + "By default this notebook is configured to run on 4 IPUs.\n", + "\n", + "Finally, `max_seq_length` is the maximum length a sequence can be, and all sequences will be padded to this length, so it should not be larger than the maximum length of the model. \n", + "\n", + "Given the small size of the sequences in go-emotions, we can reduce the model maximum input size to `max_seq_length = 256`. Set these parameters and the rest of the notebook should run smoothly:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "zVvslsfMIrIh" + }, + "outputs": [], + "source": [ + "model_checkpoint = \"bert-base-uncased\" # Default uncased pre-trained BERT checkpoint\n", + "ipu_config_name = \"Graphcore/bert-base-uncased\" # Default Graphcore IPU config initialisation for pre-trained BERT\n", + "max_seq_length = 256 # The maximum sequence length allowed for sequences in the model.\n", + "micro_batch_size = 2 \n", + "gradient_accumulation_steps = 39\n", + "device_iterations = 32\n", + "model_task = 'go_emotions'\n", + "num_labels = 28" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Gradients are not calculated during validation, so gradient accumulation is not applicable, and the global batch size for validation can be defined separately as:\n", + "```\n", + "global_validation_batch_size=device_iterations*replication_factor*max_seq_per_pack\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Values for machine size and cache directories can be configured through environment variables or directly in the notebook:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "pod_type = os.getenv(\"GRAPHCORE_POD_TYPE\", \"pod4\")\n", + "executable_cache_dir = os.getenv(\"POPLAR_EXECUTABLE_CACHE_DIR\", \"./exe_cache/\") + \"/packed_bert_mlseqcls/\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "whPRbBNbIrIl" + }, + "source": [ + "## Loading the dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "W7QYTpxXIrIl" + }, + "source": [ + "We will use the [πŸ€— Datasets](https://github.com/huggingface/datasets) library to download the data and get the metric we need to use for evaluation (to compare our model to the benchmark). This can be easily done with the functions `load_dataset` and `load_metric`. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "IreSlFmlIrIm" + }, + "outputs": [], + "source": [ + "from datasets import load_dataset\n", + "import evaluate" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataset = load_dataset(model_task)\n", + "metric = evaluate.load(\"roc_auc\", \"multilabel\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RzfPtOMoIrIu" + }, + "source": [ + "The `dataset` object itself is [`DatasetDict`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasetdict), which contains one key for the training, validation and test set (with more keys for the mismatched validation and test set in the special case of `mnli`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GWiVUF0jIrIv", + "outputId": "35e3ea43-f397-4a54-c90c-f2cf8d36873e" + }, + "outputs": [], + "source": [ + "dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "u3EtYfeHIrIz" + }, + "source": [ + "To access an actual element, you need to select a split first, then give an index:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "X6HrpprwIrIz", + "outputId": "d7670bc0-42e4-4c09-8a6a-5c018ded7d95" + }, + "outputs": [], + "source": [ + "dataset[\"train\"][0]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WHUmphG3IrI3" + }, + "source": [ + "To get a sense of what the data looks like, the following function will show some examples picked randomly in the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "i3j8APAoIrI3" + }, + "outputs": [], + "source": [ + "import datasets\n", + "import random\n", + "import pandas as pd\n", + "from IPython.display import display, HTML\n", + "\n", + "def show_random_elements(dataset, num_examples=10):\n", + " assert num_examples <= len(dataset), \"Can't pick more elements than there are in the dataset.\"\n", + " picks = []\n", + " for _ in range(num_examples):\n", + " pick = random.randint(0, len(dataset)-1)\n", + " while pick in picks:\n", + " pick = random.randint(0, len(dataset)-1)\n", + " picks.append(pick)\n", + " \n", + " df = pd.DataFrame(dataset[picks])\n", + " for column, typ in dataset.features.items():\n", + " if isinstance(typ, datasets.ClassLabel):\n", + " df[column] = df[column].transform(lambda i: typ.names[i])\n", + " display(HTML(df.to_html()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "SZy5tRB_IrI7", + "outputId": "ba8f2124-e485-488f-8c0c-254f34f24f13" + }, + "outputs": [], + "source": [ + "show_random_elements(dataset[\"train\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lnjDIuQ3IrI-" + }, + "source": [ + "The metric is an instance of [`datasets.Metric`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Metric):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5o4rUteaIrI_", + "outputId": "18038ef5-554c-45c5-e00a-133b02ec10f1" + }, + "outputs": [], + "source": [ + "metric" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jAWdqcUBIrJC" + }, + "source": [ + "You can call its `compute` method with your predictions and labels directly and it will return a dictionary with the metric(s) value:" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "n9qywopnIrJH" + }, + "source": [ + "## Preprocessing the data" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "YVx71GdAIrJH" + }, + "source": [ + "Before we can feed the texts to our model, we need to preprocess them. This is done by a πŸ€— Transformers `Tokenizer` which will (as the name indicates) tokenize the inputs (including converting the tokens to their corresponding IDs in the pretrained vocabulary), putting them into a format the model expects, as well as generate the other inputs that model requires.\n", + "\n", + "To do all of this, we instantiate our tokenizer with the `AutoTokenizer.from_pretrained` method, which will ensure:\n", + "\n", + "- we get a tokenizer that corresponds to the model architecture we want to use,\n", + "- we download the vocabulary used when pretraining this specific checkpoint.\n", + "\n", + "That vocabulary will be cached, so it's not downloaded again the next time we run the cell." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "eXNLu_-nIrJI" + }, + "outputs": [], + "source": [ + "from transformers import AutoTokenizer\n", + " \n", + "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Vl6IidfdIrJK" + }, + "source": [ + "We pass along `use_fast=True` to the call above to use one of the fast tokenizers (backed by Rust) from the πŸ€— Tokenizers library. Those fast tokenizers are available for almost all models, but if you got an error with the previous call, remove that argument." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rowT4iCLIrJK" + }, + "source": [ + "You can directly call this tokenizer on one sentence or a pair of sentences:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "a5hBlsrHIrJL", + "outputId": "acdaa98a-a8cd-4a20-89b8-cc26437bbe90" + }, + "outputs": [], + "source": [ + "tokenizer(\"Hello, this one sentence!\", \"And this sentence goes with it.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qo_0B1M2IrJM" + }, + "source": [ + "Depending on the model you selected, you will see different keys in the dictionary returned by the cell above. They don't matter much for what we're doing here (just know they are required by the model we will instantiate later), you can learn more about them in [this tutorial](https://huggingface.co/transformers/preprocessing.html) if you're interested.\n", + "\n", + "To preprocess our dataset, we will need the names of the columns containing the sentence(s). In this case, the column is called `'text'` and it is indexed as such in the tokenization function." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2C0hcmp9IrJQ" + }, + "source": [ + "We can then write the function that will preprocess our samples. We just feed them to the `tokenizer` with the three arguments.`truncation=True` will ensure that an input longer than maximum length will be truncated to the maximum length. `max_length=max_seq_length` sets the maximum length of a sequence.\n", + "\n", + "**Note that since we use packing later, we don't set any padding in the tokenizer.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vc0BSBLIIrJQ" + }, + "outputs": [], + "source": [ + "# no padding for packing\n", + "def preprocess_function(examples):\n", + " return tokenizer(examples['text'], truncation=True, max_length=max_seq_length)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For multi-label classification, we also need to convert our labels from integer values indicating a category to an N-hot binary format (where N is the maximum number of labels. This makes sure we have constant sized labels, and all of our labels (one input can have multiple target labels) are present for training. The conversion looks something like this:\n", + "\n", + "```python\n", + "unprocessed_labels = [3,21] # Where 3 and 21 are label categories\n", + "preprocessed_labels = id_to_N_hot([3,21])\n", + "preprocessed_labels = [0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0]\n", + "```\n", + "\n", + "\n", + "The following function processes one example and converts it to N-hot - the `.map()` functionality available in the `datasets` library allows the function to be applied easily to the entire dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "def id_to_N_hot(example):\n", + " indexes = example['labels']\n", + " label = np.zeros((num_labels,), dtype=int)\n", + " for idx in indexes:\n", + " label[idx] = 1\n", + " example['labels'] = label\n", + " return example" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zS-6iXTkIrJT" + }, + "source": [ + "To apply this function on all the sentences (or pairs of sentences) in our dataset, we just use the `map` method of our `dataset` object we created earlier. This will apply the function on all the elements of all the splits in `dataset`, so our training, validation and testing data will be preprocessed in one single command." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DDtsaJeVIrJT", + "outputId": "aa4734bf-4ef5-4437-9948-2c16363da719" + }, + "outputs": [], + "source": [ + "encoded_dataset = dataset.map(id_to_N_hot)\n", + "encoded_dataset = encoded_dataset.map(preprocess_function, batched=True)\n", + "\n", + "len(encoded_dataset['validation'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "voWiw8C7IrJV" + }, + "source": [ + "Even better, the results are automatically cached by the πŸ€— Datasets library to avoid spending time on this step the next time you run your notebook. The πŸ€— Datasets library is normally smart enough to detect when the function you pass to map has changed (and thus requires to not use the cache data). For instance, it will properly detect if you change the task in the first cell and rerun the notebook. πŸ€— Datasets warns you when it uses cached files, you can pass `load_from_cache_file=False` in the call to `map` to not use the cached files and force the preprocessing to be applied again.\n", + "\n", + "Note that we passed `batched=True` to encode the texts by batches together. This is to leverage the full benefit of the fast tokenizer we loaded earlier, which will use multi-threading to treat the texts in a batch concurrently." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Packing the dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To implement packing, we need to pack our dataset first. Each new element will be a \"pack\" containing at most `max_seq_per_pack` sequences." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "max_seq_per_pack = 6" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The problem type for this task is multi_label_classification, this also needs to be defined for the packed model to work." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "problem_type = 'multi_label_classification'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Packing algorithm" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to pack efficiently, we will use a histogram-based algorithm: shortest-pack-first histogram packing (SPFHP) presented in the [blog post](https://www.graphcore.ai/posts/introducing-packed-bert-for-2x-faster-training-in-natural-language-processing) adapted from the [blog code](https://github.com/graphcore/tutorials/tree/master/blogs_code/packedBERT). The full process of packing the dataset consists of four steps:\n", + "\n", + "1. Create a histogram of the sequence lengths of the dataset.\n", + "2. Generate the 'strategy' for the dataset using one of the state-of-the-art packing algorithms, which maps out the order and indices of the sequences that need to be packed together.\n", + "3. Use this strategy to create the actual dataset, concatenating the tokenized features together for each column in the dataset, including the labels.\n", + "4. Finally, pass these new columns into a custom PyTorch dataset, ready to be passed to the PopTorch dataloader!\n", + "\n", + "These steps have been simplified through the easy-to-use `packing_utils` available in Graphcore Optimum. You can simply generate the packed dataset after the usual tokenization and preprocessing by passing all necessary packing configuration to the `PackedDatasetCreator` class, and generate the ready-to-use PyTorch dataset with `.create()`.\n", + "\n", + "Within the function, there are some column names used by default. The expected default columns for text classification include:\n", + "* `input_ids`\n", + "* `attention_mask`\n", + "* `token_type_ids`\n", + "* `labels`\n", + "\n", + "These should all be generated automatically when tokenizing any classification dataset for BERT. However, the labels key, as it is not encoded, may have a different name. For this dataset, the column key for the labels for this dataset is `labels`, we can pass this to the argument `custom_label_key`, so the class can find our labels. \n", + "\n", + "The `PackedDatasetCreator` requires different instantiations for different datasets, so it must be called separately for each of our dataset splits. We can set either `training`, `validation` or `inference` to `True` as needed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from utils.packing.dataset_creator import PackedDatasetCreator\n", + "\n", + "train_data_packer = PackedDatasetCreator(\n", + " tokenized_dataset = encoded_dataset['train'],\n", + " max_sequence_length = max_seq_length,\n", + " max_sequences_per_pack = max_seq_per_pack,\n", + " training = True,\n", + " num_labels = num_labels,\n", + " problem_type = problem_type,\n", + " algorithm = 'SPFHP',\n", + " custom_label_key = 'labels'\n", + ")\n", + "\n", + "val_data_packer = PackedDatasetCreator(\n", + " tokenized_dataset = encoded_dataset['validation'],\n", + " max_sequence_length = max_seq_length,\n", + " max_sequences_per_pack = max_seq_per_pack,\n", + " validation = True,\n", + " num_labels = num_labels,\n", + " problem_type = problem_type,\n", + " algorithm = 'SPFHP',\n", + " custom_label_key = 'labels'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This will create the strategy and initialise the necessary parameters for packing the dataset. We can see that the ideal speed-up we have achieved is approximately 5.7x the original dataset, which corresponds directly to the average packing factor: the average number of sequences within one pack.\n", + "\n", + "The `PackedDatasetCreator` class also has some other features we do not use here for training, such as `pad_to_global_batch_size`, a feature useful for performing batched inference on a large samples when we do not want to lose any of the samples when creating data iterators using the `poptorch.Dataloader`, it applies 'vertical' padding to the dataset, adding filler rows to bring the dataset up to a value divisible by the global batch size, and allows for the largest possible batch sizes to be used without any loss of data.\n", + "\n", + "You can also view the histogram generated in the first step of the packing process, to observe whether the distribution of sequence lengths in the dataset will benefit from packing - as a general rule, as long as the average length of the sequences in the dataset is 50% or less of the maximum sequence length, packing will offer at least a 2x throughput benefit, in other words: `throughput_increase β‰ˆ max_seq_len/mean_seq_len`\n", + "\n", + "Many datasets have distributions with much smaller average lengths, and will benefit much more. We can easily observe this distribution by retrieving and plotting the histogram from the data class:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "train_histogram = train_data_packer.histogram\n", + "\n", + "plt.hist(train_histogram, bins = [k for k in range(0,max_seq_length,10)]) \n", + "plt.title(\"Sequence length histogram\") \n", + "plt.xlabel('Sequence lengths')\n", + "plt.ylabel('Frequency')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we need to create the actual packed dataset, this is the 3rd step of the packing process outlined above.\n", + "\n", + "In this stage, we take the strategy for mapping the sequences by size into 'packs' that was generated by the packing algorithm, and use this to extract the sequences from the tokenized dataset, inserting them into packs for each column in the dataset. Any remaining space in a pack after the sequences have been concatenated is padded to bring all sequences up to the maximum sequence length.\n", + "\n", + "Some key features unique to packed datasets are worth mentioning here:\n", + "\n", + "- A specific `attention_mask` is generated: It contains a unique index for each sequence of the pack and `0` for the remaining padding tokens. This, essentially, tells the model where to \"look\" from the perspective of a single token, ignoring any encoded information (such as a different sequence) that is not relevant to that token.\n", + " - Example of 3 sequences: `attention_mask = [1,1,1,1,1,1,2,2,2,2,2,3,3,3,3,3,0,...,0,1,2,3]`\n", + "\n", + "\n", + "- The [CLS] tokens of each sequence must be moved to the end of the pack.\n", + " - For instance: `[CLS,a,b,c] + [CLS, d,e,f] + [CLS, g,h,i] -> [a,b,c,d,e,f,g,h,i,...,CLS,CLS,CLS]`\n", + " \n", + "\n", + "- The `position_ids` of a pack contain the concatenated `position_ids` of each sequences \n", + " - For instance given 3 sequences: `[0,1,2,3,4] + [0,1,2,3] + [0,1,2] -> [1,2,3,4,1,2,3,1,2,...,0,0,0]` (note: the CLS tokens position id '0' are also moved the end of the pack)\n", + " \n", + "- `labels` and `token_type_ids` are also packed to correspond to the `input_ids` pack.\n", + "\n", + "\n", + "To create a dataloader-ready packed dataset, all you need to do is call the `create()` method:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "packed_train_dataset = train_data_packer.create()\n", + "packed_val_dataset = val_data_packer.create()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's visualize one sample of the new `packed_train_dataset`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "packed_train_dataset[133]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Fine-tuning the model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that our data is ready, we can download the pretrained model and fine-tune it." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Implementing Packed BERT\n", + "\n", + "A few model modifications are required to make packing work with BERT.\n", + "We extend the existing class `BertForSequenceClassification` to `PipelinedPackedBertForSequenceClassification` which incorporates the required changes to the pooler and the model output. The crux of these changes is to modify the generic sequence classification model to handle 'unpacking' multiple sequences in the output stage, treating them as a larger batch size for classification, as well as masking any padding created by packing.\n", + "\n", + "First let's load a default BERT configuration using `AutoConfig`. The config includes a new parameter we must set, `max_sequences_per_pack`, this informs the model of the maximum number of sequences it will need to 'unpack' in the model output. It also allows us to clearly define the `num_labels` and `problem_type` for this model.\n", + "\n", + "The problem type is essential to define here, as switching between methods used by different types of classification requires it within the custom model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoConfig\n", + "\n", + "config = AutoConfig.from_pretrained(model_checkpoint)\n", + "config.max_sequences_per_pack = max_seq_per_pack\n", + "config.num_labels = num_labels\n", + "config.problem_type = problem_type" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can instantiate the model class with the config, loading the weights from the model checkpoint." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "import torch\n", + "import numpy as np\n", + "torch.manual_seed(43)\n", + "np.random.seed(43)\n", + "\n", + "from models.modeling_bert_packed import PipelinedPackedBertForSequenceClassification\n", + "\n", + "\n", + "model = PipelinedPackedBertForSequenceClassification(config).from_pretrained(\n", + " model_checkpoint, config=config)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The warning is telling us we are throwing away some weights and randomly initializing others. This is absolutely normal in this case, because we are removing the head used to pretrain the model on a masked language modeling objective and replacing it with a new head for sequence classification, which we don't have pretrained weights, so the library warns us we should fine-tune this model before using it for inference, which is exactly what we are going to do.\n", + "\n", + "We can first test the model on CPU and observe that the output logits have the size `[batch_size, max_seq_per_pack, 2] = [1, 6, 28]` with this notebook's default values, and the 28 labels for the dataset. The logits are reshaped into this form in the model output, to be the same shape as the labels, for ease of postprocessing." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# test the model on CPU\n", + "from transformers.data.data_collator import default_data_collator\n", + "\n", + "loader = torch.utils.data.DataLoader(packed_train_dataset,\n", + " batch_size=1,\n", + " shuffle=True,\n", + " drop_last=True,\n", + " collate_fn=default_data_collator)\n", + "data = next(iter(loader))\n", + "labels = data['labels']\n", + "\n", + "print('labels: ', labels.shape)\n", + "o = model(**data)\n", + "print('outputs (loss, logits): ', o[0], o[1].shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, let's prepare the model for IPU" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, we set the model in half precision:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "model.half()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For validation, we need to define a function to compute the metrics from the predictions, which will just use the `metric` we loaded earlier, preprocessing here involves a step to mask the labels and predictions we are not using, set to a `-100` value when creating the dataset, with a boolean mask. Then, the predictions are passed into a softmax function to determine the probabilities of each class, as this is a multi-label task. \n", + "\n", + "These predictions and labels are passed into the metric function to compute the accuracy during evaluation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "model_name = model_checkpoint.split(\"/\")[-1]\n", + "from scipy.special import softmax\n", + "from tqdm import tqdm\n", + "def compute_metrics(eval_pred):\n", + " predictions, labels = eval_pred\n", + " \n", + " labels = labels.reshape(-1, labels.shape[-1])\n", + " predictions = predictions.reshape(-1, predictions.shape[-1])\n", + " \n", + " # Remove the padding labels\n", + " mask = (labels != -100)[:,0]\n", + " \n", + " labels = labels[mask,:]\n", + " predictions = predictions[mask,:]\n", + " pred_scores = softmax(predictions.astype(\"float32\"), axis=1) \n", + "\n", + " auc = metric.compute(\n", + " prediction_scores=pred_scores, references=labels, multi_class=\"ovr\"\n", + " )[\"roc_auc\"]\n", + "\n", + " return {\"roc_auc\": auc}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we need to define the `IPUConfig`, which is a class that specifies attributes and configuration parameters to compile and put the model on the device. We initialize it with one config name or path, which we set earlier. Then we use it to set the mode attribute `model.ipu_config` " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from optimum.graphcore import IPUConfig, IPUTrainer, IPUTrainingArguments\n", + "\n", + "ipu_config = IPUConfig.from_pretrained(\n", + " ipu_config_name,\n", + " executable_cache_dir = executable_cache_dir,\n", + " gradient_accumulation_steps=gradient_accumulation_steps,\n", + " device_iterations = device_iterations,\n", + " replication_factor=1,\n", + " inference_device_iterations = 64,\n", + " inference_replication_factor = 1\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The IPUTrainingArguments define any custom parameter modification we want to do, such as the initial learning rate for the model. It also allows other options, such as dataloader parameters, micro batch sizes and an automatic push to the Huggingface Hub (if credentials were set up earlier) to happen at given intervals.\n", + "\n", + "These arguments are passed to the `IPUTrainer` which wraps the model training and evaluation process into a simple single-line process, doing all of the heavy lifting for us regarding training and evaluation loops, device assignment, optimiser definition, dataloading etc.\n", + "\n", + "Note that only some arbitrary hyperparameter tuning was performed for this task. Other tasks and datasets may require further tuning to get the most optimal results." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "from transformers import default_data_collator\n", + "metric_name = \"roc_auc\"\n", + "\n", + "args = IPUTrainingArguments(\n", + " \"./\"+f\"{model_name}-{model_task}\",\n", + " per_device_train_batch_size=micro_batch_size,\n", + " per_device_eval_batch_size=4,\n", + " num_train_epochs=5,\n", + " learning_rate=2e-4,\n", + " adam_epsilon=1e-6,\n", + " loss_scaling=16.0,\n", + " warmup_ratio=0.1,\n", + " weight_decay=0,\n", + " lr_scheduler_type = \"cosine\",\n", + " metric_for_best_model=metric_name,\n", + " dataloader_drop_last=True,\n", + " dataloader_mode=\"async_rebatched\",\n", + " logging_steps=1,\n", + " pod_type=pod_type,\n", + " gradient_accumulation_steps=gradient_accumulation_steps,\n", + " push_to_hub=True \n", + ")\n", + "\n", + "trainer = IPUTrainer(\n", + " model,\n", + " ipu_config,\n", + " args,\n", + " train_dataset=packed_train_dataset,\n", + " eval_dataset=packed_val_dataset,\n", + " data_collator=default_data_collator,\n", + " compute_metrics=compute_metrics\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then, to train the model we can simply call the `train()` method:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "trainer.train()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***About the performance:*** `IPUTrainer` doesn't take into account that we have packed data samples when computing the speed metrics. It treats a 'sample' as a single input to the model, i.e. one **pack**.\n", + "\n", + "So the actual throughput estimation can be obtained by multiplying the `samples_per_second` by the average packing factor (the average number of samples per pack) of the dataset. These were obtained in the `packing_algorithm` section: `5.68` for the `go-emotions` training set and `5.83` for validation set.\n", + "\n", + "\n", + "Next, we can evaluate the model by simply calling the `evaluate()` method:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "trainer.evaluate()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can now upload the result of the training to the Hub if you successfully logged in at the beginning of this notebook, just execute this instruction:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "trainer.push_to_hub()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also save the model locally:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "trainer.save_model(\"./\"+f\"{model_name}-{model_task}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You have now successfully fine-tuned and evaluated your speed-optimised model for text classification using packing!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Faster inference" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This section demonstrates how to perform faster, batched inference with a large number of samples using packing." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When training, the packing factor affects the convergence and hyperparameters in a similar way to a large increase in batch size. However, for inference-only runs, we are free to use a bigger packing factor to speed it up. Let's try it on GoEmotions with `max_seq_per_pack = 12`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "max_seq_per_pack = 12" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataset = load_dataset(\"go_emotions\")\n", + "\n", + "encoded_dataset = dataset.map(id_to_N_hot)\n", + "encoded_dataset = encoded_dataset.map(preprocess_function, batched=True)\n", + "inference_dataset = encoded_dataset['train'] #Lets use the train dataset to have more features\n", + "\n", + "# The dataset initialisation and .create() can be executed in one line\n", + "inference_packed_dataset = PackedDatasetCreator(\n", + " tokenized_dataset = encoded_dataset['train'],\n", + " max_sequence_length = max_seq_length,\n", + " max_sequences_per_pack = max_seq_per_pack,\n", + " inference = True,\n", + " problem_type = problem_type).create()\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see that the average packing factor has improved from 5.68 to 9.14, allowing an approximate **9 times** throughput speed-up from the base unpacked model.\n", + "\n", + "Let's also modify the configuration of the model for inference. For speed up, we can replicate a one-IPU run (`ipus_per_replica`) over four IPUs by changing the `replication_factor`. After this, we can re-initialise the model and the `IPUTrainer` with the existing arguments." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ipu_config.layers_per_ipu = [12]\n", + "ipu_config.inference_device_iterations = 32\n", + "ipu_config.inference_replication_factor = 4\n", + "ipu_config.ipus_per_replica = 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's load the checkpoint we saved earlier to run the inference on:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_checkpoint = f\"./{model_name}_{model_task}\"\n", + "\n", + "# Load from Huggingface Hub instead:\n", + "# model_checkpoint = '/{model_name}-{model_task}'\n", + "\n", + "model = PipelinedPackedBertForSequenceClassification.from_pretrained(model_checkpoint, config=config)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "args = IPUTrainingArguments(\n", + " \"/tmp/\"+f\"{model_name}-{model_task}-fast-inf\",\n", + " per_device_eval_batch_size=8,\n", + " dataloader_mode=\"async_rebatched\",\n", + " dataloader_drop_last=True,\n", + " logging_steps=10,\n", + " pod_type=pod_type\n", + ")\n", + "\n", + "trainer = IPUTrainer(\n", + " model,\n", + " ipu_config,\n", + " args,\n", + " eval_dataset=inference_packed_dataset,\n", + " compute_metrics=compute_metrics\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "trainer.evaluate(inference_packed_dataset)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using these simple optimisations and the increase in maximum sequences per pack, we can see a massive throughput increase to approximately **25k sequences per second** - remember that to obtain the actual throughput we multiply the packed samples/s by the average packing factor - highlighting the benefits of using packing! " + ] + } + ], + "metadata": { + "colab": { + "name": "Text Classification on GLUE", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + }, + "vscode": { + "interpreter": { + "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + } + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/packed-bert/packedBERT_question_answering.ipynb b/packed-bert/packedBERT_question_answering.ipynb new file mode 100644 index 0000000..fa97ddc --- /dev/null +++ b/packed-bert/packedBERT_question_answering.ipynb @@ -0,0 +1,1133 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "51b0ba30", + "metadata": {}, + "source": [ + "First of all, ensure your environment has the latest version of [πŸ€— Optimum Graphcore](https://github.com/huggingface/optimum-graphcore) installed:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4ad38948", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install git+https://github.com/huggingface/optimum-graphcore.git" + ] + }, + { + "cell_type": "markdown", + "id": "b26df2b4", + "metadata": {}, + "source": [ + "Next, ensure all required packages for this notebook are installed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e98ec027", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "%pip install datasets\n", + "%pip install evaluate\n", + "%pip install tokenizers\n", + "%pip install matplotlib\n", + "%pip install scipy\n", + "%pip install --force-reinstall huggingface_hub==0.11.1;" + ] + }, + { + "cell_type": "markdown", + "id": "df689f96", + "metadata": {}, + "source": [ + "Let's start by importing the `transformers` and `optimum.graphcore` libraries, and printing the versions we are using." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa8d39f7", + "metadata": {}, + "outputs": [], + "source": [ + "import transformers\n", + "import optimum.graphcore\n", + "\n", + "print(transformers.__version__)\n", + "print(optimum.graphcore.__version__)" + ] + }, + { + "cell_type": "markdown", + "id": "802f03f9", + "metadata": {}, + "source": [ + "At the end of this notebook, to be able to share your model with the community and easily access it through HuggingFace, there are some short set-up steps you must follow to enable uploading your checkpoint to the HuggingFace Hub.\n", + "\n", + "First you have to store your authentication token from the Hugging Face website ([sign up here](https://huggingface.co/join) if you haven't already!) then execute the following cell and input your username and password:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4c81945", + "metadata": {}, + "outputs": [], + "source": [ + "from huggingface_hub import notebook_login\n", + "\n", + "notebook_login()" + ] + }, + { + "cell_type": "markdown", + "id": "2b3d049e", + "metadata": {}, + "source": [ + "Git-lfs must also be installed to enable large file storage when pushing to the hub:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "afa6ac5a", + "metadata": {}, + "outputs": [], + "source": [ + "! apt install git-lfs" + ] + }, + { + "cell_type": "markdown", + "id": "156ed6e4", + "metadata": {}, + "source": [ + "# Faster question-answering with SQuAD using PackedBERT\n", + "\n", + "This notebook describes how to fine-tune BERT from [πŸ€— Transformers](https://github.com/huggingface/transformers) for question-answering using the SQuAD(v1) dataset using [packing](https://towardsdatascience.com/introducing-packed-bert-for-2x-faster-training-in-natural-language-processing-eadb749962b1), an optimisation method originally used for 2x faster BERT pre-training, which can now also provide massive throughput increases for fine-tuning and batched inference! \n", + "\n", + "**So, what *is* packing?** The basic idea of 'packing' a dataset is to utilise the requirement for constant-shaped inputs into a model. Instead of padding it with empty, unused space, we can recycle this unused space and fill it with more inputs! The architecture of transformer models like BERT supports this, and lets us optimally use this space to process multiple sequences within one input.\n", + "\n", + "**And here is why you might want to use it:** Having a single input contain multiple sequences leads to multiple sequences being processed in parallel in a single pass within a single iteration inside a batch, increasing the 'effective' batch size of the model by a considerable factor in many cases, and most importantly, increasing model throughput for training and batched inference significantly.\n", + "\n", + "The process of training and validating the `BertForQuestionAnswering` model requires some adaptations to accommodate a packed dataset, and this notebook aims to introduce these on top of the [existing process](https://github.com/huggingface/optimum-graphcore/blob/main/notebooks/question_answering.ipynb) for fine-tuning the SQuAD dataset with BERT using an unmodified dataset." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "89898522", + "metadata": {}, + "source": [ + "Let's initialise our training configurations. \n", + "\n", + "Note here that we define a 'micro' batch size, which is the local batch size that would be passed into the model on the CPU. In this notebook, we are using both data parallelism and pipeline parallelism (see this [tutorial](https://github.com/graphcore/tutorials/tree/master/tutorials/pytorch/efficient_data_loading)), so the 'global' batch size, i.e. the number of data elements passed for one gradient calculation on the IPU, is calculated using the `device_iterations`, `gradient_accumulation_steps`, `replication_factor` and `max_seq_per_pack` (maximum sequences in a pack) for training, such that:\n", + "\n", + "```\n", + "global_training_batch_size = micro_batch_size * device_iterations * gradient_accumulation_steps * replication_factor\n", + "```\n", + "\n", + "Depending on you model and the pod machine you are using, you might need to adjust these three batch-size-related arguments.\n", + "\n", + "`max_seq_per_pack` highlights the benefit of packing multiple sequences into one input sequence given there is enough space for them. It shows that multiple sequences are processed effectively in parallel within the model, using up space that would essentially be padding if one sequence were passed at a time. This is a much more efficient way to send inputs into the model, and improves the global batch size to a best-case-scenario of:\n", + "\n", + "```\n", + "global_training_batch_size = micro_batch_size * device_iterations * gradient_accumulation_steps * replication_factor * max_seq_per_pack\n", + "```\n", + "\n", + "Realistically, the global batch size will not always be multiplied by the *maximum* number of sequences in a packed sequence, but rather the *average* number of sequences in a packed sequence, and will depend on the sequence length distribution within any given dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ad1b478", + "metadata": {}, + "outputs": [], + "source": [ + "model_checkpoint=\"bert-base-uncased\" # Default uncased pre-trained BERT checkpoint\n", + "ipu_config_name=\"Graphcore/bert-base-uncased\" # Default Graphcore IPU config initialisation for pre-trained BERT\n", + "max_seq_length=384 # The maximum sequence length allowed for sequences in the model.\n", + "gradient_accumulation_steps=32 # Gradient accumulation steps for training the model on the IPU.\n", + "device_iterations = 32\n", + "micro_batch_size=2\n", + "model_task=\"squad\" " + ] + }, + { + "cell_type": "markdown", + "id": "77dde875", + "metadata": {}, + "source": [ + "Gradients are not calculated during validation, so gradient accumulation is not applicable, and the global batch size for validation can be defined separately as:\n", + "\n", + "```\n", + "global_validation_batch_size=micro_batch_size*device_iterations*replication_factor*max_seq_per_pack\n", + "```\n", + "\n", + "In Optimum, we can define inference-specific `device iterations` and `replication factor`, which can be adjusted to create larger batches to complensate for the lack of a gradient accumulation factor." + ] + }, + { + "cell_type": "markdown", + "id": "92ae1cca", + "metadata": {}, + "source": [ + "Values for machine size and cache directories can be configured through environment variables or directly in the notebook:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b882a5b3", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "pod_type = os.getenv(\"GRAPHCORE_POD_TYPE\", \"pod4\")\n", + "executable_cache_dir = os.getenv(\"POPLAR_EXECUTABLE_CACHE_DIR\", \"./exe_cache/\") + \"/packed_bert_squad/\"" + ] + }, + { + "cell_type": "markdown", + "id": "33597c71", + "metadata": {}, + "source": [ + "## Loading the dataset\n", + "\n", + "The next step is to use the [πŸ€— Datasets](https://github.com/huggingface/datasets) library to download the dataset from the hub, and to use the [πŸ€— Evaluate](https://github.com/huggingface/evaluate) library to load the evaluation metrics for the SQuAD model. This will allow easy performance metric analysis during validation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b37cb293", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "from datasets import load_dataset, load_metric\n", + "import evaluate\n", + "\n", + "\n", + "\n", + "dataset = load_dataset(model_task) # Load dataset\n", + "metric = evaluate.load(model_task) # Load metric for dataset" + ] + }, + { + "cell_type": "markdown", + "id": "6dac6eca", + "metadata": {}, + "source": [ + "The `dataset` object itself is [`DatasetDict`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasetdict), which contains one key for the training, validation and test set:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2115928b", + "metadata": {}, + "outputs": [], + "source": [ + "dataset" + ] + }, + { + "cell_type": "markdown", + "id": "23d3f421", + "metadata": {}, + "source": [ + "To access an actual element, you need to select a split first, then provide an index:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "311b8b73", + "metadata": {}, + "outputs": [], + "source": [ + "dataset[\"train\"][0]" + ] + }, + { + "cell_type": "markdown", + "id": "3702f2a3", + "metadata": {}, + "source": [ + "In the SQuAD dataset, we have a `question`, its `context` i.e., an excerpt of text which includes the answer as well as surrounding context, and the `answer` key, which holds the start position of the answer in the context, as well as the answer itself. For a different or custom question-answering dataset, these fields may have different names but serve the same purpose, so pre-defining them is useful.\n", + "\n", + "We have a configuration describing these necessary keys in the dataset containing the raw data that needs to be pre-processed or tokenised before being passed into the model. These generic keys may change for custom datasets, but the usage of them generally stays the same for a similar fine-tuning task." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "628bc41f", + "metadata": {}, + "outputs": [], + "source": [ + "question_key=\"question\"\n", + "context_key=\"context\"\n", + "answer_key=\"answers\"\n", + "train = True\n", + "validate = True" + ] + }, + { + "cell_type": "markdown", + "id": "793dcd19", + "metadata": {}, + "source": [ + "## Preprocessing the data\n", + "\n", + "Before we can feed those texts to our model, we need to preprocess them. This is done by a πŸ€— Transformers `Tokenizer` which will (as the name indicates) tokenize the inputs (including converting the tokens to their corresponding IDs in the pretrained vocabulary) and put it in a format the model expects, as well as generate the other inputs that model requires.\n", + "\n", + "To do all of this, we instantiate our tokenizer with the `AutoTokenizer.from_pretrained` method, which will ensure:\n", + "\n", + "- we get a tokenizer that corresponds to the model architecture we want to use,\n", + "- we download the vocabulary used when pretraining this specific checkpoint.\n", + "\n", + "That vocabulary will be cached, so it's not downloaded again the next time we run the cell.\n", + "\n", + "The `Dataset` method is also imported, which will allow us to convert our modified and tokenized columns in dictionary form to a dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aab94819", + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoTokenizer\n", + "from datasets import Dataset \n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True)" + ] + }, + { + "cell_type": "markdown", + "id": "a47ea927", + "metadata": {}, + "source": [ + "For SQuAD, we define a custom function to handle the overflows and offset mapping created by generating tokenised inputs from sequences, as well as the start and end positions of the answers which need to be translated from positions of characters to positions of tokens.\n", + "\n", + "The first step is to tokenize the dataset using the tokenizer. Note here that for packing, it is important to **not** pad the dataset, so `padding` should be set to `False`. If we pad, we will have to un-pad when packing sequences into a packed sequence, which is inefficient.\n", + "\n", + "The preprocessing function is outlined in [the original (unpacked) question-answering notebook](question_answering.ipynb) for more information on it. In this case, we can import the preprocessing directly from `utils.packing`, ready *without* padding for PackedBERT." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2263dfef", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "from utils.packing.qa_utils import preprocess_packed_qa\n", + "\n", + "raw_train_dataset = dataset['train']\n", + "\n", + "tokenized_training_dataset = preprocess_packed_qa(\n", + " dataset=raw_train_dataset,\n", + " tokenizer=tokenizer,\n", + " question_key=question_key,\n", + " context_key=context_key,\n", + " answer_key=answer_key,\n", + " sequence_length=max_seq_length,\n", + " padding=False,\n", + " train=True\n", + ")\n", + "\n", + "\n", + "raw_validation_dataset = dataset['validation']\n", + "\n", + "tokenized_validation_dataset = preprocess_packed_qa(\n", + " dataset=raw_validation_dataset,\n", + " tokenizer=tokenizer,\n", + " question_key=question_key,\n", + " context_key=context_key,\n", + " answer_key=answer_key,\n", + " sequence_length=max_seq_length,\n", + " padding=False,\n", + " train=False\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "f57906e8", + "metadata": {}, + "source": [ + "## Packing the dataset\n", + "\n", + "To implement packing, we need to pack our dataset first. Each new element will be a \"pack\" containing at most `max_seq_per_pack` sequences." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6bdd1b9e", + "metadata": {}, + "outputs": [], + "source": [ + "max_seq_per_pack = 6" + ] + }, + { + "cell_type": "markdown", + "id": "51c17c9b", + "metadata": {}, + "source": [ + "We also define the number of labels in our dataset. For SQuAD, this means the number of outputs, i.e. positions returned by the model - since it is not a classification task, so this is set to 2, to correspond to start and end positions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bfda406f", + "metadata": {}, + "outputs": [], + "source": [ + "num_labels = 2\n", + "problem_type = 'question_answering'" + ] + }, + { + "cell_type": "markdown", + "id": "c39316ea", + "metadata": {}, + "source": [ + "### Packing algorithm" + ] + }, + { + "cell_type": "markdown", + "id": "d9d24ab5", + "metadata": {}, + "source": [ + "In order to pack efficiently, we will use a histogram-based algorithm: shortest-pack-first histogram packing (SPFHP) presented in the [blog post](https://www.graphcore.ai/posts/introducing-packed-bert-for-2x-faster-training-in-natural-language-processing) adapted from the [blog code](https://github.com/graphcore/tutorials/tree/master/blogs_code/packedBERT). The full process of packing the dataset consists of four steps:\n", + "\n", + "1. Create a histogram of the sequence lengths of the dataset.\n", + "2. Generate the 'strategy' for the dataset using one of the state-of-the-art packing algorithms, which maps out the order and indices of the sequences that need to be packed together.\n", + "3. Use this strategy to create the actual dataset, concatenating the tokenized features together for each column in the dataset, including the labels.\n", + "4. Finally, pass these new columns into a custom PyTorch dataset, ready to be passed to the PopTorch dataloader!\n", + "\n", + "These steps have been simplified through the easy-to-use `utils.packing` available in Graphcore Optimum. You can simply generate the packed dataset after the usual tokenization and preprocessing by passing all necessary packing configuration to the `PackedDatasetCreator` class, and generate the ready-to-use PyTorch dataset with `.create()`.\n", + "\n", + "Within the function, there are some column names used by default. The expected default columns for question-answering include:\n", + "* `input_ids`\n", + "* `attention_mask`\n", + "* `token_type_ids`\n", + "* `start_positions`\n", + "* `end_positions`\n", + "\n", + "These should all be generated automatically when tokenizing the SQuAD dataset for BERT.\n", + "\n", + "The `PackedDatasetCreator` requires different instantiations for different datasets, so it must be called separately for each of our dataset splits. We can set either `training`, `validation` or `inference` to `True` as needed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e66ed06d", + "metadata": {}, + "outputs": [], + "source": [ + "from utils.packing.dataset_creator import PackedDatasetCreator\n", + "\n", + "train_data_packer = PackedDatasetCreator(\n", + " tokenized_dataset = tokenized_training_dataset,\n", + " max_sequence_length = max_seq_length,\n", + " max_sequences_per_pack = max_seq_per_pack,\n", + " training = True,\n", + " num_labels = num_labels,\n", + " problem_type = problem_type,\n", + " algorithm = 'SPFHP'\n", + ")\n", + "\n", + "val_data_packer = PackedDatasetCreator(\n", + " tokenized_dataset = tokenized_validation_dataset,\n", + " max_sequence_length = max_seq_length,\n", + " max_sequences_per_pack = max_seq_per_pack,\n", + " validation = True,\n", + " num_labels = num_labels,\n", + " problem_type = problem_type,\n", + " algorithm = 'SPFHP'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "720ea314", + "metadata": {}, + "source": [ + "This will create the strategy and initialise the necessary parameters for packing the dataset. We can see that the ideal speed-up we have achieved is approximately 2.2x the original dataset, which corresponds directly to the average packing factor: the average number of sequences within one pack.\n", + "\n", + "The `PackedDatasetCreator` class also has some other features we do not use here for training, such as `pad_to_global_batch_size`, a feature useful for performing batched inference on a large samples when we do not want to lose any of the samples when creating data iterators using the `poptorch.Dataloader`, it applies 'vertical' padding to the dataset, adding filler rows to bring the dataset up to a value divisible by the global batch size, and allows for the largest possible batch sizes to be used without any loss of data." + ] + }, + { + "cell_type": "markdown", + "id": "46319488", + "metadata": {}, + "source": [ + "You can also view the histogram generated in the first step of the packing process, to observe whether the distribution of sequence lengths in the dataset will benefit from packing - as a general rule, as long as the average length of the sequences in the dataset is 50% or less of the maximum sequence length, packing will offer at least a 2x throughput benefit, in other words: `throughput_increase β‰ˆ max_seq_len/mean_seq_len`\n", + "\n", + "Many datasets have distributions with much smaller average lengths, and will benefit much more. We can easily observe this distribution by retrieving and plotting the histogram from the data class:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "113b58f4", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "train_histogram = train_data_packer.histogram\n", + "\n", + "plt.hist(train_histogram, bins = [k for k in range(0,max_seq_length,10)]) \n", + "plt.title(\"Sequence length histogram\") \n", + "plt.xlabel('Sequence lengths')\n", + "plt.ylabel('Frequency')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "1d077b97", + "metadata": {}, + "source": [ + "Now we need to create the actual packed dataset, this is the 3rd step of the packing process outlined above.\n", + "\n", + "In this stage, we take the strategy for mapping the sequences by size into 'packs' that was generated by the packing algorithm, and use this to extract the sequences from the tokenized dataset, inserting them into packs for each column in the dataset. Any remaining space in a pack after the sequences have been concatenated is padded to bring all sequences up to the maximum sequence length.\n", + "\n", + "**Some key features unique to packed datasets are worth mentioning here**:\n", + "\n", + "- A specific `attention_mask` is generated: It contains a unique index for each sequence of the pack and `0` for the remaining padding tokens. This, essentially, tells the model where to \"look\" from the perspective of a single token, ignoring any encoded information (such as a different sequence) that is not relevant to that token.\n", + " - Example of 3 sequences in a pack: `attention_mask = [1,1,1,1,1,1,2,2,2,2,2,3,3,3,3,3,0,0,0]`\n", + " - Compared to a single sequence in an unpacked input `attention_mask = [1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0]`\n", + " \n", + "\n", + "- The `position_ids` of a pack contain the concatenated `position_ids` of each sequences \n", + " - For instance given 3 sequences: `[0,1,2,3,4] + [0,1,2,3] + [0,1,2] -> [1,2,3,4,1,2,3,1,2,...,0,0,0]` (note: the CLS tokens position id '0' are also moved the end of the pack)\n", + " \n", + " \n", + "- For SQuAD, during training, answers are determined using a start position and end position within the sequence. During preprocessing, these were converted from character positions to token positions. Now, during packing, as tokenized sequences are effectively being concatenated along the same dimension, the positions of the answer will change for any sequence that is not starting at index 0 within a pack. For example, in a pack with 2 sequences:\n", + " - Answer positions before packing:\n", + " ```\n", + " Length of sequence 1: 100 tokens (index 0 to 99) , start position: 30, end position: 35\n", + " Length of sequence 2: 120 tokens (index 0 to 119) , start position: 15, end position: 25\n", + " ```\n", + " - Answer positions after packing:\n", + " ```\n", + " Length of sequence 1 in pack 1: 100 tokens (index 0 to 99) , start position: 30, end position: 35\n", + " Length of sequence 2 in pack 1: 120 tokens (index 100 to 219), start position: 115, end position: 125 \n", + " ```\n", + "\n", + " - The positions have been shifted by the total length of preceding sequences in the pack, We call this the `positions_offset`.\n", + "\n", + "\n", + "To create a dataloader-ready packed dataset, all you need to do is call the `create()` method:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bdcc161d", + "metadata": {}, + "outputs": [], + "source": [ + "packed_train_dataset = train_data_packer.create()\n", + "packed_val_dataset = val_data_packer.create()" + ] + }, + { + "cell_type": "markdown", + "id": "ce443c8f", + "metadata": {}, + "source": [ + "Let's visualize one sample of the new `packed_train_dataset`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c966cd9a", + "metadata": {}, + "outputs": [], + "source": [ + "packed_train_dataset[133]" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "a1d8ce9c", + "metadata": {}, + "source": [ + "## Fine-tuning the model\n", + "\n", + "Now that our data is ready, we can download the pretrained model and fine-tune it.\n", + "\n", + "### Implement Packed BERT\n", + "\n", + "Some model modifications are required to make packing work with BERT. For SQuAD, we create a custom output class to separate the logits according to each of the sequences within the pack and calculate the loss. The existing class `BertForQuestionAnswering` is extended to `PipelinedPackedBertForQuestionAnswering` which incorporates the required modifications to the model. The crux of these changes is to introduce the new attention mask, and modify the hidden layer output of the model to mask any padded inputs from the logits.\n", + "\n", + "First let's load a default BERT configuration using `AutoConfig`. The config includes a new parameter we must set, `max_sequences_per_pack`, this informs the model of the maximum number of sequences it will need to 'unpack' in the model output. It also allows us to clearly define the `num_labels` and `problem_type` for this model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "254a0f83", + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoConfig\n", + "\n", + "config = AutoConfig.from_pretrained(model_checkpoint)\n", + "config.max_sequences_per_pack = max_seq_per_pack\n", + "config.num_labels = num_labels\n", + "config.problem_type = problem_type" + ] + }, + { + "cell_type": "markdown", + "id": "0b7dae37", + "metadata": {}, + "source": [ + "Now we can instantiate the model class with the config, loading the weights from the model checkpoint. For SQuAD, we can determine the number of \"labels\" as the two output types that will determine whether answers are correct or not, i.e., the start and end position." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3285aaa3", + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "import torch\n", + "import numpy as np\n", + "torch.manual_seed(43)\n", + "np.random.seed(43)\n", + " \n", + "from models.modeling_bert_packed import PipelinedPackedBertForQuestionAnswering\n", + "\n", + "model = PipelinedPackedBertForQuestionAnswering.from_pretrained(model_checkpoint, config=config)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "d6070000", + "metadata": {}, + "source": [ + "The warning is telling us we are throwing away some weights and randomly initializing others. This is absolutely normal in this case, because we are removing the head used to pretrain the model on a masked language modeling objective and replacing it with a new head for question answering, for which we don't have pretrained weights, so the library warns us we should fine-tune this model before using it for inference, which is exactly what we are going to do.\n", + "\n", + "We can first test the model on CPU." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02aac4e1", + "metadata": {}, + "outputs": [], + "source": [ + "# test the model on CPU\n", + "from transformers.data.data_collator import default_data_collator\n", + "\n", + "loader = torch.utils.data.DataLoader(packed_train_dataset,\n", + " batch_size=2,\n", + " shuffle=True,\n", + " drop_last=True,\n", + " collate_fn=default_data_collator)\n", + "data = next(iter(loader))\n", + "o = model(**data)\n", + "print(\"Logits shape:\", o)" + ] + }, + { + "cell_type": "markdown", + "id": "5c9922fa", + "metadata": {}, + "source": [ + "Now, let's prepare the model for IPU.\n", + "\n", + "First, we set the model in half precision:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11502853", + "metadata": {}, + "outputs": [], + "source": [ + "model.half()" + ] + }, + { + "cell_type": "markdown", + "id": "7a9ab06f", + "metadata": {}, + "source": [ + "### Define validation metrics for SQuAD\n", + "\n", + "Before training and evaluating, a custom postprocessing function needs to be defined for SQuAD. This is because we need to map the predictions of the model back to parts of the context in terms of the character positions in the original untokenized samples. The model predicts logits for the start and end token position of the answer.\n", + "\n", + "The purpose of the function is to identify each of the tokenized features according to their `example_ids` and map the start and end token positions for the output, taking the top-*n* logit indices and discarding all invalid solutions. It then uses the `offset_mapping` to map the start and end token-level positions back to character-level positions within the context, and generates a text answer using the original context. This text prediction can then be used to calculate accuracy metrics and compared to the target answer present in the dataset.\n", + "\n", + "The `postprocess_qa_predictions()` function is adapted for packing, taken directly from the existing [tutorial for SQuAD finetuning for the IPU](https://github.com/huggingface/optimum-graphcore/blob/main/notebooks/question_answering.ipynb) for an unpacked dataset. The full description for the use of this function is described in that tutorial. \n", + "\n", + "The main changes to the function for packing include: \n", + "* Instead of iterating through all the features in the tokenized dataset, and obtaining the `example_id` field created during tokenization of the validation dataset, this function iterates through each feature within each pack, obtaining the corresponding `example_id` for each feature within the pack. \n", + "\n", + "* It saves the index of the pack in the dataset, **as well as the index of the feature within the pack**, to allow the function to easily and linearly obtain the features to perform validation on.\n", + "\n", + "This postprocessing is available ready-to-use from the packing utils: `utils.packing`, and can simply be initialised." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f4c6e4d0", + "metadata": {}, + "outputs": [], + "source": [ + "from utils.packing.qa_utils import postprocess_packed_qa_predictions" + ] + }, + { + "cell_type": "markdown", + "id": "38c76b0b", + "metadata": {}, + "source": [ + "Finally, a `compute_validation_metrics` function is created to take in the postprocessed predictions. This obtains the answers from the dataset, maps them according to the `example_id` to the corresponding prediction, and uses `metric` from the πŸ€— Evaluate library to compute the relevant metrics for SQuAD, including an \"exact match\" accuracy, as well as F1 score, for each answer. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5420ef6a", + "metadata": {}, + "outputs": [], + "source": [ + "def compute_validation_metrics(predictions, raw_validation_dataset, packed_validation_dataset_unformatted, metric):\n", + " \n", + " target_answers = [\n", + " {\"id\": ex[\"id\"], \"answers\": ex[\"answers\"]} for ex in raw_validation_dataset\n", + " ]\n", + " \n", + " final_predictions = postprocess_packed_qa_predictions(\n", + " raw_validation_dataset, packed_validation_dataset_unformatted, predictions\n", + " )\n", + "\n", + " formatted_predictions = [\n", + " {\"id\": k, \"prediction_text\": v} for k, v in final_predictions.items()\n", + " ]\n", + "\n", + " metrics = metric.compute(predictions=formatted_predictions, references=target_answers)\n", + " \n", + " return metrics\n" + ] + }, + { + "cell_type": "markdown", + "id": "acdde982", + "metadata": {}, + "source": [ + "### Train and validate the model using the πŸ€— Optimum Graphcore `Trainer`\n", + "\n", + "Now let's prepare the model for IPU, instantiate the options and machine configurations and create an IPU Trainer to efficiently and easily perform training on the IPU in just a few lines.\n", + "\n", + "We need to define the `IPUConfig`, which is a class that specifies attributes and configuration parameters to compile and put the model on the device. We initialize it with one config name or path, which we set earlier. Then we use it to set the mode attribute `model.ipu_config` " + ] + }, + { + "cell_type": "markdown", + "id": "e2073fcd", + "metadata": {}, + "source": [ + "As we are using a pre-trained checkpoint, we can use the existing IPU configuration for `\"Graphcore/bert-base-uncased\"`for the custom model. This should require no changes as even though the model has been modified to be compatible with a packed dataset, the pipelining stages and IPU options will remain the same. \n", + "\n", + "Some of the options have been specified when defining the `ipu_config` to highlight the global batch size. This uses the configurations defined at the beginning of this script. Note that we can also define inference specific device iterations and replication factors for performing validation on the model, to modify the validation global batch size." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8b0452e1", + "metadata": {}, + "outputs": [], + "source": [ + "from optimum.graphcore import IPUConfig, IPUTrainer, IPUTrainingArguments\n", + "\n", + "ipu_config = IPUConfig.from_pretrained(\n", + " ipu_config_name,\n", + " executable_cache_dir = executable_cache_dir,\n", + " gradient_accumulation_steps=gradient_accumulation_steps,\n", + " device_iterations=device_iterations,\n", + " replication_factor=1,\n", + " embedding_serialization_factor=1,\n", + " inference_device_iterations= 64,\n", + " inference_replication_factor=1,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "0a6b8635", + "metadata": {}, + "source": [ + "To instantiate an `IPUTrainer`, we will need to define `IPUTrainingArguments`, which is a class that contains all the attributes to customize the training. It requires one folder name, which will be used to save the checkpoints of the model, and all other arguments are optional:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "141a2e2d", + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "training_args = IPUTrainingArguments(\n", + " output_dir=f\"./{model_checkpoint}-{model_task}\",\n", + " per_device_train_batch_size=micro_batch_size,\n", + " per_device_eval_batch_size=8,\n", + " num_train_epochs=3,\n", + " learning_rate=9e-05,\n", + " loss_scaling=64.0,\n", + " weight_decay=0.01,\n", + " warmup_ratio=0.25,\n", + " lr_scheduler_type='cosine',\n", + " pod_type=pod_type,\n", + " gradient_accumulation_steps=gradient_accumulation_steps,\n", + " dataloader_drop_last=True,\n", + " dataloader_num_workers=64,\n", + " logging_steps=5\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "c5e150ed", + "metadata": {}, + "source": [ + "**Note that we do not set evaluation to be performed during the training process for SQuAD**. This is due to the custom postprocessing steps required to extract text-level answers for SQuAD, for which the logits cannot be easily modified without multiple function inputs, such as the tokenized and raw datasets, while the `preprocess_logits_for_metrics` argument provided in `IPUTrainingArguments` can only utilise logits alone. Therefore, validation is done after training." + ] + }, + { + "cell_type": "markdown", + "id": "eeb965d0", + "metadata": {}, + "source": [ + "We will need a data collator that will batch our processed examples together, here we will use the default data collator imported from the Transformers library. This is passed to the `IPUTrainer` class. \n", + "\n", + "Then we just need to pass all of this along with our datasets to the IPUTrainer:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "561a41ca", + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "from transformers import default_data_collator\n", + "\n", + "trainer = IPUTrainer(\n", + " model=model,\n", + " ipu_config=ipu_config,\n", + " args=training_args,\n", + " train_dataset=packed_train_dataset,\n", + " data_collator=default_data_collator\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "id": "07b0933f", + "metadata": {}, + "source": [ + "We can now finetune our model by just calling the train method:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4cbe563", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "train_run_metrics = trainer.train()" + ] + }, + { + "cell_type": "markdown", + "id": "c31ad4dc", + "metadata": {}, + "source": [ + "You can now upload the result of the training to the Hub if you successfully logged in at the beginning of this notebook, just execute this instruction:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b9e60061", + "metadata": {}, + "outputs": [], + "source": [ + "trainer.push_to_hub()" + ] + }, + { + "cell_type": "markdown", + "id": "a6d0fc29", + "metadata": {}, + "source": [ + "Then save the model with the model checkpoint name." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "625847dc", + "metadata": {}, + "outputs": [], + "source": [ + "trainer.save_model(f\"./{model_checkpoint}-{model_task}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c93fb854", + "metadata": {}, + "source": [ + "We can then perform the evaluation by using the `IPUTrainer`'s `predict` functionality. This provides all of the raw predictions for the packed inputs for validation. This will, be default, use the global batch size defined specifically for inference in the `IPUTrainingArguments`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c65f6830", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "raw_predictions = trainer.predict(packed_val_dataset)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "7553b34d", + "metadata": {}, + "source": [ + "Once the predictions have been obtained, the validation metrics can be computed by passing them into the `compute_validation_metrics` function. This, as described previously, performs the necessary postprocessing on the logits and obtains text answers, then computes the accuracy metrics (exact match and F1 score) for SQuAD finetuning." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "825dd9a8", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "val_metrics = compute_validation_metrics(\n", + " raw_predictions.predictions, raw_validation_dataset, packed_val_dataset, metric)\n", + "\n", + "print(val_metrics)" + ] + }, + { + "cell_type": "markdown", + "id": "50eb0d90", + "metadata": {}, + "source": [ + "## Faster Inference" + ] + }, + { + "cell_type": "markdown", + "id": "6bda303c", + "metadata": {}, + "source": [ + "When training, the packing factor affects the convergence and hyperparameters in a similar way to a large increase in batch size. However, for inference-only runs, we are free to use a bigger packing factor to speed it up. Let's try it on SQuAD with max_seq_per_pack = 12, and sequence length set to 512." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "50cd4b0e", + "metadata": {}, + "outputs": [], + "source": [ + "max_seq_per_pack = 12" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c64fcd44", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = load_dataset(\"squad\")\n", + "raw_train_dataset = dataset['train']\n", + "max_seq_length = 512\n", + "\n", + "# Lets use the train dataset to have more features to infer over\n", + "tokenized_inference_dataset = preprocess_packed_qa(\n", + " dataset=raw_train_dataset,\n", + " tokenizer=tokenizer,\n", + " question_key=question_key,\n", + " context_key=context_key,\n", + " answer_key=answer_key,\n", + " sequence_length=max_seq_length,\n", + " padding=False,\n", + " train=False\n", + ")\n", + "\n", + "packed_inference_dataset = PackedDatasetCreator(\n", + " tokenized_dataset = tokenized_inference_dataset,\n", + " max_sequence_length = max_seq_length,\n", + " max_sequences_per_pack = max_seq_per_pack,\n", + " inference=True,\n", + " problem_type = problem_type,\n", + ").create()" + ] + }, + { + "cell_type": "markdown", + "id": "42d720aa", + "metadata": {}, + "source": [ + "We can see that the average packing factor has improved from 2.2 to 2.95, allowing an approximate 3x throughput speed-up from the base unpacked model. This is not nearly as much as the maximum sequences per pack limit, due to the larger sequence lengths in the SQuAD dataset, but still allows a 3x speedup for inference!\n", + "\n", + "Let's also modify the configuration of the model for inference. For speed up, we can replicate a one-IPU run (`ipus_per_replica`) over four IPUs by changing the `replication_factor`. After this, we can re-initialise the model and the `IPUTrainer` with the existing arguments." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2b7f4fca", + "metadata": {}, + "outputs": [], + "source": [ + "ipu_config.layers_per_ipu = [12]\n", + "ipu_config.inference_device_iterations = 32\n", + "ipu_config.inference_replication_factor = 4\n", + "ipu_config.ipus_per_replica = 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a11a6f8f", + "metadata": {}, + "outputs": [], + "source": [ + "model = PipelinedPackedBertForQuestionAnswering.from_pretrained(\n", + " f\"./{model_checkpoint}-{model_task}\", config=config)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1debe64e", + "metadata": {}, + "outputs": [], + "source": [ + "args = IPUTrainingArguments(\n", + " \"/tmp/\"+f\"{model_checkpoint}-{model_task}-fast-inf\",\n", + " per_device_eval_batch_size=8,\n", + " dataloader_mode=\"async_rebatched\",\n", + " dataloader_drop_last=True,\n", + " logging_steps=10,\n", + " pod_type=pod_type\n", + ")\n", + "\n", + "trainer = IPUTrainer(\n", + " model,\n", + " ipu_config,\n", + " args,\n", + " eval_dataset=packed_inference_dataset\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "81969920", + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "trainer.evaluate()" + ] + }, + { + "cell_type": "markdown", + "id": "3c77e5ac", + "metadata": {}, + "source": [ + "Using these simple optimisations and the increase in maximum sequences per pack, we can see a throughput increase to approximately **8000 sequences per second** - remember that to obtain the actual throughput we multiply the packed samples/s by the average packing factor - highlighting the benefits of using packing! " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + }, + "vscode": { + "interpreter": { + "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/packed-bert/packedBERT_single_label_text_classification.ipynb b/packed-bert/packedBERT_single_label_text_classification.ipynb new file mode 100644 index 0000000..cb37161 --- /dev/null +++ b/packed-bert/packedBERT_single_label_text_classification.ipynb @@ -0,0 +1,1375 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "X4cRE8IbIrIV" + }, + "source": [ + "First of all, make sure your environment has the latest version of [πŸ€— Optimum Graphcore](https://github.com/huggingface/optimum-graphcore) installed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "%pip install git+https://github.com/huggingface/optimum-graphcore.git" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Also make sure all the packages required for this notebook are installed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "%pip install scikit-learn;\n", + "%pip install datasets\n", + "%pip install evaluate\n", + "%pip install tokenizers\n", + "%pip install matplotlib\n", + "%pip install scipy\n", + "%pip install --force-reinstall huggingface_hub==0.11.1;" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's start by importing and printing out the versions of `Transformers` and `Optimum Graphcore`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import transformers\n", + "import optimum.graphcore\n", + "\n", + "print(transformers.__version__)\n", + "print(optimum.graphcore.__version__)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "At the end of this notebook, to be able to share your model with the community and easily access it through HuggingFace, there are some short set-up steps you must follow to enable uploading your checkpoint to the HuggingFace Hub.\n", + "\n", + "First you have to store your authentication token from the Hugging Face website ([sign up here](https://huggingface.co/join) if you haven't already!) then execute the following cell and input your username and password:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from huggingface_hub import notebook_login\n", + "\n", + "notebook_login()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Git-lfs must also be installed to enable large file storage when pushing to the hub:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! apt install git-lfs" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rEJBSTyZIrIb" + }, + "source": [ + "# Faster single-label text classification with PackedBERT \n", + "\n", + "This notebook builds on the process of [fine-tuning BERT on a text classification task](text_classification.ipynb), using [packing](https://www.graphcore.ai/posts/introducing-packed-bert-for-2x-faster-training-in-natural-language-processing), an optimisation method originally used for 2x faster BERT pre-training, which can now also provide massive throughput increases for fine-tuning and batched inference! \n", + "\n", + "**So, what *is* packing?** The basic idea of 'packing' a dataset is to utilise the requirement for constant-shaped inputs into a model. Instead of padding it with empty, unused space, we can recycle this unused space and fill it with more inputs! The architecture of transformer models like BERT supports this, and lets us optimally use this space to process multiple sequences within one input.\n", + "\n", + "**And here is why you might want to use it:** Having a single input contain multiple sequences leads to multiple sequences being processed in parallel in a single pass within a single iteration inside a batch, increasing the 'effective' batch size of the model by a considerable factor in many cases, and most importantly, increasing model throughput for training and batched inference significantly.\n", + "\n", + "This notebook outlines how to easily enable packing for BERT when performing fine-tuning/inference on a text-classification task in πŸ€— Optimum, resulting in an impressive 5-6x faster training and inference run-time on the `GLUE/sst2` dataset. \n", + "\n", + "You can read more about packing in the original [paper](https://arxiv.org/abs/2107.02027)." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "kTCFado4IrIc" + }, + "source": [ + "In this notebook, we will see how to fine-tune BERT, a [πŸ€— Transformers](https://github.com/huggingface/transformers) model to a text classification task of the [GLUE Benchmark](https://gluebenchmark.com/).\n", + "\n", + "The GLUE Benchmark is a group of nine classification tasks on sentences or pairs of sentences, which are:\n", + "\n", + "- [CoLA](https://nyu-mll.github.io/CoLA/) (Corpus of Linguistic Acceptability) Determine if a sentence is grammatically correct or not.is a dataset containing sentences labeled grammatically correct or not.\n", + "- [MNLI](https://arxiv.org/abs/1704.05426) (Multi-Genre Natural Language Inference) Determine if a sentence entails, contradicts or is unrelated to a given hypothesis. (This dataset has two versions, one with the validation and test set coming from the same distribution, another called mismatched where the validation and test use out-of-domain data.)\n", + "- [MRPC](https://www.microsoft.com/en-us/download/details.aspx?id=52398) (Microsoft Research Paraphrase Corpus) Determine if two sentences are paraphrases from one another or not.\n", + "- [QNLI](https://rajpurkar.github.io/SQuAD-explorer/) (Question-answering Natural Language Inference) Determine if the answer to a question is in the second sentence or not. (This dataset is built from the SQuAD dataset.)\n", + "- [QQP](https://data.quora.com/First-Quora-Dataset-Release-Question-Pairs) (Quora Question Pairs2) Determine if two questions are semantically equivalent or not.\n", + "- [RTE](https://aclweb.org/aclwiki/Recognizing_Textual_Entailment) (Recognizing Textual Entailment) Determine if a sentence entails a given hypothesis or not.\n", + "- [SST-2](https://nlp.stanford.edu/sentiment/index.html) (Stanford Sentiment Treebank) Determine if the sentence has a positive or negative sentiment.\n", + "- [STS-B](http://ixa2.si.ehu.es/stswiki/index.php/STSbenchmark) (Semantic Textual Similarity Benchmark) Determine the similarity of two sentences with a score from 1 to 5.\n", + "- [WNLI](https://cs.nyu.edu/faculty/davise/papers/WinogradSchemas/WS.html) (Winograd Natural Language Inference) Determine if a sentence with an anonymous pronoun and a sentence with this pronoun replaced are entailed or not. (This dataset is built from the Winograd Schema Challenge dataset.)\n", + "\n", + "We will see how to easily load the dataset for these tasks and use BERT with packing to fine-tune a model on SST-2. Each task is named using an acronym, with `mnli-mm` standing for the 'mis-matched' version of MNLI (so it is the same training set as `mnli` but with different validation and test sets):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "YZbiBDuGIrId" + }, + "outputs": [], + "source": [ + "GLUE_TASKS = [\"cola\", \"mnli\", \"mnli-mm\", \"mrpc\", \"qnli\", \"qqp\", \"rte\", \"sst2\", \"stsb\", \"wnli\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**For this Packed BERT demo, we cover (single-label) sequence classification on the `sst2` dataset. The `task` can be changed to run the other `GLUE` tasks. However, training hyperparameters may need further tuning for these other tasks.**" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "4RRkXuteIrIh" + }, + "source": [ + "Let's initialise our training configurations. \n", + "\n", + "In this notebook, we are using both data parallelism and pipeline parallelism (see this [tutorial](https://github.com/graphcore/tutorials/tree/master/tutorials/pytorch/tut2_efficient_data_loading) for more). Therefore the global batch size, which is the actual number of samples used for the weight update, is determined using four factors:\n", + "\n", + " global batch size = micro_batch_size * gradient accumulation steps * device iterations * replication factor\n", + "\n", + "Replication factor is determined by `pod_type`, which will be used as a key to select the replication factor from a dictionary defined in the IPU config file. For example, the dictionary in the IPU config file [Graphcore/roberta-base-ipu](https://huggingface.co/Graphcore/roberta-base-ipu/blob/main/ipu_config.json) looks like this.:\n", + "\n", + " \"replication_factor\": {\"pod4\": 1, \"pod8\": 2, \"pod16\": 4, \"pod32\": 8, \"pod64\": 16, \"default\": 1}\n", + "\n", + "Depending on your model and the pod machine you are using, you might need to adjust these three batch-size-related arguments.\n", + "\n", + "By default this notebook is configured to run on 4 IPUs.\n", + "\n", + "Finally, `max_seq_length` is the maximum length a sequence can be, and all sequences will be padded to this length, so it should not be larger than the maximum length of the model. Set these parameters and the rest of the notebook should run smoothly:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Given the small size of the sequences in `sst2`, we can reduce the model maximum input size to `max_seq_length = 256`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "zVvslsfMIrIh" + }, + "outputs": [], + "source": [ + "task = \"sst2\"\n", + "model_checkpoint = \"bert-base-uncased\"\n", + "ipu_config_name = \"Graphcore/bert-base-uncased\"\n", + "micro_batch_size = 2\n", + "gradient_accumulation_steps = 32\n", + "device_iterations = 32\n", + "max_seq_length = 256" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Gradients are not calculated during validation, so gradient accumulation is not applicable, and the global batch size for validation can be defined separately as:\n", + "\n", + "```\n", + "global_validation_batch_size=device_iterations*replication_factor*max_seq_per_pack\n", + "```\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "Values for machine size and cache directories can be configured through environment variables or directly in the notebook:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "pod_type = os.getenv(\"GRAPHCORE_POD_TYPE\", \"pod4\")\n", + "executable_cache_dir = os.getenv(\"POPLAR_EXECUTABLE_CACHE_DIR\", \"./exe_cache/\") + \"/packed_bert_slseqcls/\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "whPRbBNbIrIl" + }, + "source": [ + "## Loading the dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "W7QYTpxXIrIl" + }, + "source": [ + "We will use the [πŸ€— Datasets](https://github.com/huggingface/datasets) library and the [πŸ€— Evaluate](https://github.com/huggingface/evaluate) library to download the data and get the metric we need to use for evaluation (to compare our model to the benchmark). This can be easily done with the functions `load_dataset` and `evaluate.load()`. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "IreSlFmlIrIm" + }, + "outputs": [], + "source": [ + "from datasets import load_dataset\n", + "import evaluate" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CKx2zKs5IrIq" + }, + "source": [ + "Apart from `mnli-mm` being a special code, we can directly pass our task name to those functions. `load_dataset` will cache the dataset to avoid downloading it again the next time you run this cell." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "actual_task = \"mnli\" if task == \"mnli-mm\" else task\n", + "dataset = load_dataset(\"glue\", actual_task)\n", + "metric = evaluate.load('glue', actual_task)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RzfPtOMoIrIu" + }, + "source": [ + "The `dataset` object itself is [`DatasetDict`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasetdict), which contains one key for the training, validation and test set (with more keys for the mismatched validation and test set in the special case of `mnli`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GWiVUF0jIrIv", + "outputId": "35e3ea43-f397-4a54-c90c-f2cf8d36873e" + }, + "outputs": [], + "source": [ + "dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "u3EtYfeHIrIz" + }, + "source": [ + "To access an actual element, you need to select a split first, then give an index:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "X6HrpprwIrIz", + "outputId": "d7670bc0-42e4-4c09-8a6a-5c018ded7d95" + }, + "outputs": [], + "source": [ + "dataset[\"train\"][0]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WHUmphG3IrI3" + }, + "source": [ + "To get a sense of what the data looks like, the following function will show some examples picked randomly in the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "i3j8APAoIrI3" + }, + "outputs": [], + "source": [ + "import datasets\n", + "import random\n", + "import pandas as pd\n", + "from IPython.display import display, HTML\n", + "\n", + "def show_random_elements(dataset, num_examples=10):\n", + " assert num_examples <= len(dataset), \"Can't pick more elements than there are in the dataset.\"\n", + " picks = []\n", + " for _ in range(num_examples):\n", + " pick = random.randint(0, len(dataset)-1)\n", + " while pick in picks:\n", + " pick = random.randint(0, len(dataset)-1)\n", + " picks.append(pick)\n", + " \n", + " df = pd.DataFrame(dataset[picks])\n", + " for column, typ in dataset.features.items():\n", + " if isinstance(typ, datasets.ClassLabel):\n", + " df[column] = df[column].transform(lambda i: typ.names[i])\n", + " display(HTML(df.to_html()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "SZy5tRB_IrI7", + "outputId": "ba8f2124-e485-488f-8c0c-254f34f24f13" + }, + "outputs": [], + "source": [ + "show_random_elements(dataset[\"train\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lnjDIuQ3IrI-" + }, + "source": [ + "The metric is an instance of [`datasets.Metric`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Metric):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5o4rUteaIrI_", + "outputId": "18038ef5-554c-45c5-e00a-133b02ec10f1" + }, + "outputs": [], + "source": [ + "metric" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jAWdqcUBIrJC" + }, + "source": [ + "You can call its `compute` method with your predictions and labels directly and it will return a dictionary with the metric(s) value:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6XN1Rq0aIrJC", + "outputId": "a4405435-a8a9-41ff-9f79-a13077b587c7" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "fake_preds = np.random.randint(0, 2, size=(64,))\n", + "fake_labels = np.random.randint(0, 2, size=(64,))\n", + "metric.compute(predictions=fake_preds, references=fake_labels)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YOCrQwPoIrJG" + }, + "source": [ + "Note that `load_metric` has loaded the proper metric associated to your task, which is:\n", + "\n", + "- for CoLA: [Matthews Correlation Coefficient](https://en.wikipedia.org/wiki/Matthews_correlation_coefficient)\n", + "- for MNLI (matched or mismatched): Accuracy\n", + "- for MRPC: Accuracy and [F1 score](https://en.wikipedia.org/wiki/F1_score)\n", + "- for QNLI: Accuracy\n", + "- for QQP: Accuracy and [F1 score](https://en.wikipedia.org/wiki/F1_score)\n", + "- for RTE: Accuracy\n", + "- for SST-2: Accuracy\n", + "- for STS-B: [Pearson Correlation Coefficient](https://en.wikipedia.org/wiki/Pearson_correlation_coefficient) and [Spearman's_Rank_Correlation_Coefficient](https://en.wikipedia.org/wiki/Spearman%27s_rank_correlation_coefficient)\n", + "- for WNLI: Accuracy\n", + "\n", + "so the metric object only computes the one(s) needed for your task." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "n9qywopnIrJH" + }, + "source": [ + "## Preprocessing the data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YVx71GdAIrJH" + }, + "source": [ + "Before we can feed the texts to our model, we need to preprocess them. This is done by a πŸ€— Transformers `Tokenizer` which will (as the name indicates) tokenize the inputs (including converting the tokens to their corresponding IDs in the pretrained vocabulary) and put it in a format the model expects, as well as generate the other inputs that model requires.\n", + "\n", + "To do all of this, we instantiate our tokenizer with the `AutoTokenizer.from_pretrained` method, which will ensure:\n", + "\n", + "- we get a tokenizer that corresponds to the model architecture we want to use,\n", + "- we download the vocabulary used when pretraining this specific checkpoint.\n", + "\n", + "That vocabulary will be cached, so it's not downloaded again the next time we run the cell." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "eXNLu_-nIrJI" + }, + "outputs": [], + "source": [ + "from transformers import AutoTokenizer\n", + " \n", + "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Vl6IidfdIrJK" + }, + "source": [ + "We pass along `use_fast=True` to the call above to use one of the fast tokenizers (backed by Rust) from the πŸ€— Tokenizers library. Those fast tokenizers are available for almost all models, but if you got an error with the previous call, remove that argument." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rowT4iCLIrJK" + }, + "source": [ + "You can directly call this tokenizer on one sentence or a pair of sentences:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "a5hBlsrHIrJL", + "outputId": "acdaa98a-a8cd-4a20-89b8-cc26437bbe90" + }, + "outputs": [], + "source": [ + "tokenizer(\"Hello, this is one sentence!\", \"And this sentence goes with it.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qo_0B1M2IrJM" + }, + "source": [ + "Depending on the model you selected, you will see different keys in the dictionary returned by the cell above. They don't matter much for what we're doing here (just know they are required by the model we will instantiate later), you can learn more about them in [this tutorial](https://huggingface.co/transformers/preprocessing.html) if you're interested.\n", + "\n", + "To preprocess our dataset, we will thus need the names of the columns containing the sentence(s). The following dictionary keeps track of the correspondence task to column names:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "fyGdtK9oIrJM" + }, + "outputs": [], + "source": [ + "task_to_keys = {\n", + " \"cola\": (\"sentence\", None),\n", + " \"mnli\": (\"premise\", \"hypothesis\"),\n", + " \"mnli-mm\": (\"premise\", \"hypothesis\"),\n", + " \"mrpc\": (\"sentence1\", \"sentence2\"),\n", + " \"qnli\": (\"question\", \"sentence\"),\n", + " \"qqp\": (\"question1\", \"question2\"),\n", + " \"rte\": (\"sentence1\", \"sentence2\"),\n", + " \"sst2\": (\"sentence\", None),\n", + " \"stsb\": (\"sentence1\", \"sentence2\"),\n", + " \"wnli\": (\"sentence1\", \"sentence2\"),\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xbqtC4MrIrJO" + }, + "source": [ + "We can double check it does work on our current dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "19GG646uIrJO", + "outputId": "0cb4a520-817e-4f92-8de8-bb45df367657" + }, + "outputs": [], + "source": [ + "sentence1_key, sentence2_key = task_to_keys[task]\n", + "\n", + "if sentence2_key is None:\n", + " print(f\"Sentence: {dataset['train'][0][sentence1_key]}\")\n", + "else:\n", + " print(f\"Sentence 1: {dataset['train'][0][sentence1_key]}\")\n", + " print(f\"Sentence 2: {dataset['train'][0][sentence2_key]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2C0hcmp9IrJQ" + }, + "source": [ + "We can then write the function that will preprocess our samples. We just feed them to the `tokenizer` with the three arguments.`truncation=True` will ensure that an input longer than maximum length will be truncated to the maximum length. `max_length=max_seq_length` sets the maximum length of a sequence.\n", + "\n", + "**Important: since we will use packing later, we don't want to perform any padding in the tokenizer.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vc0BSBLIIrJQ" + }, + "outputs": [], + "source": [ + "# no padding for packing\n", + "def preprocess_function(examples):\n", + " if sentence2_key is None:\n", + " return tokenizer(examples[sentence1_key], truncation=True, max_length=max_seq_length)\n", + " \n", + " return tokenizer(examples[sentence1_key], examples[sentence2_key], truncation=True, max_length=max_seq_length)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0lm8ozrJIrJR" + }, + "source": [ + "This function works with one or several examples. In the case of several examples, the tokenizer will return a list of lists for each key:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-b70jh26IrJS", + "outputId": "acd3a42d-985b-44ee-9daa-af5d944ce1d9" + }, + "outputs": [], + "source": [ + "preprocess_function(dataset['train'][:5])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zS-6iXTkIrJT" + }, + "source": [ + "To apply this function on all the sentences (or pairs of sentences) in our dataset, we just use the `map` method of our `dataset` object we created earlier. This will apply the function on all the elements of all the splits in `dataset`, so our training, validation and testing data will be preprocessed in one single command." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DDtsaJeVIrJT", + "outputId": "aa4734bf-4ef5-4437-9948-2c16363da719" + }, + "outputs": [], + "source": [ + "encoded_dataset = dataset.map(preprocess_function, batched=True)\n", + "len(encoded_dataset['train'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "voWiw8C7IrJV" + }, + "source": [ + "Even better, the results are automatically cached by the πŸ€— Datasets library to avoid spending time on this step the next time you run your notebook. The πŸ€— Datasets library is normally smart enough to detect when the function you pass to map has changed (and thus requires to not use the cache data). For instance, it will properly detect if you change the task in the first cell and rerun the notebook. πŸ€— Datasets warns you when it uses cached files, you can pass `load_from_cache_file=False` in the call to `map` to not use the cached files and force the preprocessing to be applied again.\n", + "\n", + "Note that we passed `batched=True` to encode the texts by batches together. This is to leverage the full benefit of the fast tokenizer we loaded earlier, which will use multi-threading to treat the texts in a batch concurrently." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Packing the dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To implement packing, we need to pack our dataset first. Each new element will be a \"pack\" containing at most `max_seq_per_pack` sequences." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "max_seq_per_pack = 6" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We also define the number of labels in our dataset, `sst2` is a single_label task: it will contain one true class for each example." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "num_labels = 3 if task.startswith(\"mnli\") else 1 if task==\"stsb\" else 2\n", + "problem_type = 'single_label_classification'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Packing algorithm" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to pack efficiently, we will use a histogram-based algorithm: shortest-pack-first histogram packing (SPFHP) presented in the [blog post](https://www.graphcore.ai/posts/introducing-packed-bert-for-2x-faster-training-in-natural-language-processing) adapted from the [blog code](https://github.com/graphcore/tutorials/tree/master/blogs_code/packedBERT). The full process of packing the dataset consists of four steps:\n", + "\n", + "1. Create a histogram of the sequence lengths of the dataset.\n", + "2. Generate the 'strategy' for the dataset using one of the state-of-the-art packing algorithms, which maps out the order and indices of the sequences that need to be packed together.\n", + "3. Use this strategy to create the actual dataset, concatenating the tokenized features together for each column in the dataset, including the labels.\n", + "4. Finally, pass these new columns into a custom PyTorch dataset, ready to be passed to the PopTorch dataloader!\n", + "\n", + "These steps have been simplified through the easy-to-use `utils.packing` available in Graphcore Optimum. You can simply generate the packed dataset after the usual tokenization and preprocessing by passing all necessary packing configuration to the `PackedDatasetCreator` class, and generate the ready-to-use PyTorch dataset with `.create()`.\n", + "\n", + "Within the function, there are some column names used by default. The expected default columns for text classification include:\n", + "* `input_ids`\n", + "* `attention_mask`\n", + "* `token_type_ids`\n", + "* `labels`\n", + "\n", + "These should all be generated automatically when tokenizing any classification dataset for BERT. However, the labels key, as it is not encoded, may have a different name. For this dataset, the column key for the labels for this dataset is `label`, since the dataset creator expects `labels`, we can pass this to the argument `custom_label_key`, so the class can find our labels. \n", + "\n", + "The `PackedDatasetCreator` requires different instantiations for different datasets, so it must be called separately for each of our dataset splits. We can set either `training`, `validation` or `inference` to `True` as needed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from utils.packing.dataset_creator import PackedDatasetCreator\n", + "\n", + "train_data_packer = PackedDatasetCreator(\n", + " tokenized_dataset = encoded_dataset['train'],\n", + " max_sequence_length = max_seq_length,\n", + " max_sequences_per_pack = max_seq_per_pack,\n", + " training = True,\n", + " num_labels = num_labels,\n", + " problem_type = problem_type,\n", + " algorithm = 'SPFHP',\n", + " custom_label_key = 'label'\n", + ")\n", + "\n", + "val_data_packer = PackedDatasetCreator(\n", + " tokenized_dataset = encoded_dataset['validation'],\n", + " max_sequence_length = max_seq_length,\n", + " max_sequences_per_pack = max_seq_per_pack,\n", + " validation = True,\n", + " num_labels = num_labels,\n", + " problem_type = problem_type,\n", + " algorithm = 'SPFHP',\n", + " custom_label_key = 'label'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This will create the strategy and initialise the necessary parameters for packing the dataset. We can see that the ideal speed-up we have achieved is approximately 5.15x the original dataset, which corresponds directly to the average packing factor: the average number of sequences within one pack.\n", + "\n", + "The `PackedDatasetCreator` class also has some other features we do not use here for training, such as `pad_to_global_batch_size`, a feature useful for performing batched inference on a large samples when we do not want to lose any of the samples when creating data iterators using the `poptorch.Dataloader`, it applies 'vertical' padding to the dataset, adding filler rows to bring the dataset up to a value divisible by the global batch size, and allows for the largest possible batch sizes to be used without any loss of data." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also view the histogram generated in the first step of the packing process, to observe whether the distribution of sequence lengths in the dataset will benefit from packing - as a general rule, as long as the average length of the sequences in the dataset is 50% or less of the maximum sequence length, packing will offer at least a 2x throughput benefit, in other words: `throughput_increase β‰ˆ max_seq_len/mean_seq_len`\n", + "\n", + "Many datasets have distributions with much smaller average lengths, and will benefit much more. We can easily observe this distribution by retrieving and plotting the histogram from the data class:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "train_histogram = train_data_packer.histogram\n", + "\n", + "plt.hist(train_histogram, bins = [k for k in range(0,max_seq_length,10)]) \n", + "plt.title(\"Sequence length histogram\") \n", + "plt.xlabel('Sequence lengths')\n", + "plt.ylabel('Frequency')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we need to create the actual packed dataset, this is the 3rd step of the packing process outlined above.\n", + "\n", + "In this stage, we take the strategy for mapping the sequences by size into 'packs' that was generated by the packing algorithm, and use this to extract the sequences from the tokenized dataset, inserting them into packs for each column in the dataset. Any remaining space in a pack after the sequences have been concatenated is padded to bring all sequences up to the maximum sequence length.\n", + "\n", + "Some key features unique to packed datasets are worth mentioning here:\n", + "\n", + "- A specific `attention_mask` is generated: It contains a unique index for each sequence of the pack and `0` for the remaining padding tokens. This, essentially, tells the model where to \"look\" from the perspective of a single token, ignoring any encoded information (such as a different sequence) that is not relevant to that token.\n", + " - Example of 3 sequences: `attention_mask = [1,1,1,1,1,1,2,2,2,2,2,3,3,3,3,3,0,...,0,1,2,3]`\n", + "\n", + "\n", + "- The [CLS] tokens of each sequence must be moved to the end of the pack.\n", + " - For instance: `[CLS,a,b,c] + [CLS, d,e,f] + [CLS, g,h,i] -> [a,b,c,d,e,f,g,h,i,...,CLS,CLS,CLS]`\n", + " \n", + "\n", + "- The `position_ids` of a pack contain the concatenated `position_ids` of each sequences \n", + " - For instance given 3 sequences: `[0,1,2,3,4] + [0,1,2,3] + [0,1,2] -> [1,2,3,4,1,2,3,1,2,...,0,0,0]` (note: the CLS tokens position id '0' are also moved the end of the pack)\n", + " \n", + "- `labels` and `token_type_ids` are also packed to correspond to the `input_ids` pack.\n", + "\n", + "\n", + "To create a dataloader-ready packed dataset, all you need to do is call the `create()` method:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "packed_train_dataset = train_data_packer.create()\n", + "packed_val_dataset = val_data_packer.create()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's visualize one sample of the new `packed_train_dataset`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "packed_train_dataset[133]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "545PP3o8IrJV" + }, + "source": [ + "## Fine-tuning the model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FBiW8UpKIrJW" + }, + "source": [ + "Now that our data is ready, we can download the pretrained model and fine-tune it." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Implement Packed BERT\n", + "\n", + "A few model modifications are required to make packing work with BERT.\n", + "We extend the existing class `BertForSequenceClassification` to `PipelinedPackedBertForSequenceClassification` which incorporates the required changes to the pooler and the model output. The crux of these changes is to modify the generic sequence classification model to handle 'unpacking' multiple sequences in the output stage, treating them as a larger batch size for classification, as well as masking any padding created by packing.\n", + "\n", + "First let's load a default BERT configuration using `AutoConfig`. The config includes a new parameter we must set, `max_sequences_per_pack`, this informs the model of the maximum number of sequences it will need to 'unpack' in the model output. It also allows us to clearly define the `num_labels` and `problem_type` for this model.\n", + "\n", + "The problem type is essential to define here, as switching between methods used by different types of classification requires it within the custom model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoConfig\n", + "\n", + "config = AutoConfig.from_pretrained(model_checkpoint)\n", + "config.max_sequences_per_pack = max_seq_per_pack\n", + "config.num_labels = num_labels\n", + "config.problem_type = problem_type" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can instantiate the model class with the config, loading the weights from the model checkpoint." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "import torch\n", + "import numpy as np\n", + "torch.manual_seed(43)\n", + "np.random.seed(43)\n", + "\n", + "from models.modeling_bert_packed import PipelinedPackedBertForSequenceClassification\n", + "\n", + "model = PipelinedPackedBertForSequenceClassification.from_pretrained(\n", + " model_checkpoint, config=config).train()\n", + "\n", + "print(config)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "CczA5lJlIrJX" + }, + "source": [ + "The warning is telling us we are throwing away some weights and randomly initializing others. This is absolutely normal in this case, because we are removing the head used to pretrain the model on a masked language modeling objective and replacing it with a new head for which we don't have pretrained weights, so the library warns us we should fine-tune this model before using it for inference, which is exactly what we are going to do." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can first test the model on CPU and observe that the output logits have now the size [batch_size x max_seq_per_pack, 2] = [12, 2] with this notebook default values." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "from transformers.data.data_collator import default_data_collator\n", + "import torch\n", + "\n", + "model.float()\n", + "loader = torch.utils.data.DataLoader(packed_train_dataset,\n", + " batch_size=micro_batch_size,\n", + " shuffle=True,\n", + " drop_last=True,\n", + " collate_fn=default_data_collator)\n", + "data = next(iter(loader))\n", + "outputs = model(**data)\n", + "print(outputs)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's prepare the model for IPU" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, we set the model in half precision:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model.half()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For validation, we need to define a function to compute the metrics from the predictions, which will just use the `metric` we loaded earlier, the only preprocessing we have to do is to take the argmax of our predicted logits (our just squeeze the last axis in the case of STS-B). To ignore the `-100` labels from uncomplete packs, we use a boolean mask." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name = \"pearson\" if task == \"stsb\" else \"matthews_correlation\" if task == \"cola\" else \"accuracy\"\n", + "model_name = model_checkpoint.split(\"/\")[-1]\n", + "\n", + "def compute_metrics(eval_pred):\n", + " predictions, labels = eval_pred\n", + " \n", + "# Remove the padding labels\n", + " mask = (labels != -100)\n", + " labels = labels[mask]\n", + " \n", + " if task != \"stsb\":\n", + " predictions = np.argmax(predictions, axis=-1)\n", + " else:\n", + " predictions = predictions[:, 0]\n", + " \n", + " predictions = predictions[mask]\n", + " \n", + " return metric.compute(predictions=predictions, references=labels)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_N8urzhyIrJY" + }, + "source": [ + "Next, we need to define the `IPUConfig`, which is a class that specifies attributes and configuration parameters to compile and put the model on the device. We initialize it with one config name or path, which we set earlier. Then we use it to set the mode attribute `model.ipu_config` " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from optimum.graphcore import IPUConfig, IPUTrainer, IPUTrainingArguments\n", + "\n", + "ipu_config = IPUConfig.from_pretrained(\n", + " ipu_config_name,\n", + " executable_cache_dir = executable_cache_dir,\n", + " gradient_accumulation_steps=gradient_accumulation_steps,\n", + " replication_factor=1,\n", + " device_iterations = device_iterations,\n", + " inference_device_iterations= 16,\n", + " inference_replication_factor=1\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The IPUTrainingArguments define any custom parameter modification we want to do, such as the initial learning rate for the model. It also allows other options, such as dataloader paramters, micro batch sizes and an automatic push to the Huggingface Hub (if credentials were set up earlier) to happen at given intervals.\n", + "\n", + "These arguments are passed to the `IPUTrainer` which wraps the model training and evaluation process into a simple single-line process, doing all of the heavy lifting for us regarding training and evaluation loops, device assignment, optimiser definition, dataloading etc.\n", + "\n", + "Note that only some arbitrary hyperparameter tuning was performed for this task. Other tasks and datasets may require further tuning to get the most optimal results." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "from transformers import default_data_collator\n", + "\n", + "args = IPUTrainingArguments(\n", + " \"./\"+f\"{model_name}-{task}\",\n", + " num_train_epochs=2,\n", + " per_device_train_batch_size=micro_batch_size,\n", + " per_device_eval_batch_size=2,\n", + " learning_rate=9e-5,\n", + " warmup_ratio=0.1,\n", + " weight_decay=0,\n", + " lr_scheduler_type = \"cosine\",\n", + " metric_for_best_model=metric_name,\n", + " dataloader_drop_last=True,\n", + " dataloader_mode=\"async_rebatched\",\n", + " logging_steps=1,\n", + " pod_type=pod_type,\n", + " gradient_accumulation_steps=gradient_accumulation_steps,\n", + " push_to_hub=True\n", + ")\n", + "\n", + "\n", + "trainer = IPUTrainer(\n", + " model,\n", + " ipu_config,\n", + " args,\n", + " train_dataset=packed_train_dataset,\n", + " eval_dataset=packed_val_dataset,\n", + " data_collator=default_data_collator,\n", + " compute_metrics=compute_metrics\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then, to train the model we can simply call the `train()` method:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "trainer.train()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***About the performance:*** `IPUTrainer` doesn't take into account that we have packed data samples when computing the speed metrics. It treats a 'sample' as a single input to the model, i.e. one **pack**.\n", + "\n", + "So the actual throughput estimation can be obtained by multiplying the `samples_per_second` by the average packing factor (the average number of samples per pack) of the dataset. These were obtained in the `packing_algorithm` section: `5.15` for `sst2` training set and `5.77` for validation set." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we can evaluate the model by simply calling the `evaluate()` method:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "trainer.evaluate()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To see how your model fared you can compare it to the [GLUE Benchmark leaderboard](https://gluebenchmark.com/leaderboard).\n", + "\n", + "You can now upload the result of the training to the Hub if you successfully logged in at the beginning of this notebook, just execute this instruction:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "trainer.push_to_hub()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also save the model locally:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "trainer.save_model(\"./\"+f\"{model_name}-{task}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You have now successfully fine-tuned and evaluated your speed-optimised model for text classification using packing!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Faster inference:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This section demonstrates how to perform faster, batched inference with a large number of samples using packing." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When training, the packing factor does affect the convergence the same way as a large increase in batch size would do. However, for inference, we are free to use a bigger packing factor to speed it up.\n", + "Let's try it on `sst2` with `max_seq_per_pack = 12`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "max_seq_per_pack = 12" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To have enough examples, we will reuse the training set." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "dataset = load_dataset(\"glue\", \"sst2\")\n", + "encoded_dataset = dataset.map(preprocess_function, batched=True)\n", + "inference_dataset = encoded_dataset['train'] # Use the train set again, to have enough examples\n", + "\n", + "inference_packed_dataset = PackedDatasetCreator(\n", + " tokenized_dataset = encoded_dataset['train'],\n", + " max_sequence_length = max_seq_length,\n", + " max_sequences_per_pack = max_seq_per_pack,\n", + " inference = True,\n", + " problem_type = problem_type,\n", + " custom_label_key='label').create()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see that the average packing factor `6.7` is not close to the maximum now (12), this is still an improvement compared to the previous `5.7`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's also modify the configuration of the model for inference. For speed up, we can us a single IPU and 4 replicas by changing `layers_per_ipu` , `inference_replication_factor` and `ipus_per_replica` and also use a larger `batch-size`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ipu_config.layers_per_ipu = [12]\n", + "ipu_config.inference_device_iterations = 32\n", + "ipu_config.inference_replication_factor = 4\n", + "ipu_config.ipus_per_replica = 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's load the checkpoint we saved earlier to run the inference on:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "model_checkpoint = f\"./{model_name}-{task}\"\n", + "\n", + "# Load from Huggingface Hub instead:\n", + "# model_checkpoint = '/{model_checkpoint}-{model_task}'\n", + "\n", + "model = PipelinedPackedBertForSequenceClassification.from_pretrained(model_checkpoint, config=config)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "args = IPUTrainingArguments(\n", + " \"/tmp/\"+f\"{model_name}-{task}-fast-inf\",\n", + " per_device_eval_batch_size=8,\n", + " dataloader_mode=\"async_rebatched\",\n", + " dataloader_drop_last=True,\n", + " logging_steps=10,\n", + " pod_type=pod_type\n", + ")\n", + "\n", + "trainer = IPUTrainer(\n", + " model,\n", + " ipu_config,\n", + " args,\n", + " eval_dataset=inference_packed_dataset,\n", + " compute_metrics=compute_metrics\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "trainer.evaluate()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As before, to get a correct throughput estimation we need to multiply `eval_samples_per_second` by the average packing factor (6.72). For example, if the inference throughput over a run is `4200 samples/s`, the actual throughput is `4200 * 6.72 = 28224 samples/s` " + ] + } + ], + "metadata": { + "colab": { + "name": "Text Classification on GLUE", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + }, + "vscode": { + "interpreter": { + "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + } + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/packed-bert/utils/__init__.py b/packed-bert/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packed-bert/utils/packing/__init__.py b/packed-bert/utils/packing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packed-bert/utils/packing/algorithms.py b/packed-bert/utils/packing/algorithms.py new file mode 100644 index 0000000..a9afe33 --- /dev/null +++ b/packed-bert/utils/packing/algorithms.py @@ -0,0 +1,225 @@ +# Copyright (c) 2023 Graphcore Ltd. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +from collections import defaultdict + +import numpy as np + +from scipy import optimize, stats + + +def add_pack(pack, count, tmp, final, limit, offset): + if len(pack) == limit or offset == 0: + final[offset].append((count, pack)) + else: + tmp[offset].append((count, pack)) + + +def LPFHP(histogram, max_sequence_length, max_sequences_per_pack, distribute=True): + """Longest-pack-first histogram-packing.""" + start = time.time() + reversed_histogram = np.flip(histogram) + # Initialize main strategy data dictionary. + # The key indicates how many tokens are left for full length. + # The value is a list of tuples, consisting of counts and respective packs. + # A pack is a (sorted) list of sequence length values that get concatenated. + tmp_strategies_per_length = defaultdict(list) + strategies_per_length = defaultdict(list) + if max_sequences_per_pack == "max": + max_sequences_per_pack = max_sequence_length + # Index i indicates here, how much space is left, due to reversed histogram + for i in range(max_sequence_length): + n_sequences_to_bin = reversed_histogram[i] + length_to_bin = max_sequence_length - i + offset = 0 # smallest possible offset for perfect fit + while n_sequences_to_bin > 0: + if (length_to_bin + offset) in tmp_strategies_per_length: + # extract worst pack that will get modified + n_sequences_to_pack, pack = tmp_strategies_per_length[length_to_bin + offset].pop() + # calculate how often the current sequence maximally fits in + repeat = min(1 + offset // length_to_bin, max_sequences_per_pack - len(pack)) + # correct dependent on count + while n_sequences_to_bin // repeat == 0: + repeat -= 1 + if not distribute: + repeat = 1 + new_pack = pack + [length_to_bin] * repeat + count = min(n_sequences_to_pack, n_sequences_to_bin // repeat) + if n_sequences_to_pack > count: + # old pack gets reduced + n_sequences_to_pack -= count + tmp_strategies_per_length[length_to_bin + offset].append((n_sequences_to_pack, pack)) + n_sequences_to_bin -= count * repeat + else: + n_sequences_to_bin -= n_sequences_to_pack * repeat + add_pack( + new_pack, + count, + tmp_strategies_per_length, + strategies_per_length, + max_sequences_per_pack, + offset - (repeat - 1) * length_to_bin, + max_sequence_length, + ) + # clean up to speed up main key search + if not tmp_strategies_per_length[length_to_bin + offset]: + tmp_strategies_per_length.pop(length_to_bin + offset) + # reset offset in case best fit changed + offset = 0 + else: + offset += 1 + # Does not fit anywhere. Create new pack. + if offset >= max_sequence_length - length_to_bin + 1: + # similar repetition but no dependence on pack. + repeat = min(max_sequence_length // length_to_bin, max_sequences_per_pack) + while n_sequences_to_bin // repeat == 0: + repeat -= 1 + if not distribute: + repeat = 1 + add_pack( + [length_to_bin] * repeat, + n_sequences_to_bin // repeat, + tmp_strategies_per_length, + strategies_per_length, + max_sequences_per_pack, + max_sequence_length - length_to_bin * repeat, + max_sequence_length, + ) + n_sequences_to_bin -= n_sequences_to_bin // repeat * repeat + # merge all strategies + for key in tmp_strategies_per_length: + strategies_per_length[key].extend(tmp_strategies_per_length[key]) + # flatten strategies dictionary + strategy_set = [] + strategy_repeat_count = [] + for key in strategies_per_length: + for count, pack in strategies_per_length[key]: + pack.reverse() + strategy_set.append(pack) + strategy_repeat_count.append(count) + + # Summarize efficiency of solution + duration = time.time() - start + sequence_lengths = np.arange(1, max_sequence_length + 1) + strategy_repeat_count = np.array(strategy_repeat_count) + n_strategies = len(strategy_set) + old_number_of_samples = histogram.sum() + new_number_of_samples = strategy_repeat_count.sum() + sequences = sum([count * len(pack) for count, pack in zip(strategy_repeat_count, strategy_set)]) + total_tokens = max_sequence_length * new_number_of_samples + empty_tokens = sum( + [count * (max_sequence_length - sum(pack)) for count, pack in zip(strategy_repeat_count, strategy_set)] + ) + efficiency = 100 - empty_tokens / total_tokens * 100 + speedup_upper_bound = 1.0 / ( + 1 - (histogram * (1 - sequence_lengths / max_sequence_length)).sum() / old_number_of_samples + ) + + print( + f"Packing efficiency (fraction of real tokens): {efficiency:3.4f}\n", + f"Speed-up theoretical limit: {speedup_upper_bound:3.4f}\n", + f"Achieved speed-up over un-packed dataset: {old_number_of_samples/new_number_of_samples:3.5f}", + f"Runtime: Packed {old_number_of_samples} sequences in {duration:3.3f} seconds.", + ) + + return strategy_set, strategy_repeat_count # =mixtures + + +def SPFHP(histogram: np.ndarray, max_sequence_length: int, max_sequences_per_pack: int): + """Shortest-pack-first histogram-packing.""" + start = time.time() + reversed_histogram = np.flip(histogram) + # Initialize main strategy data dictionary. + # The key indicates how many tokens are left for full length. + # The value is a list of tuples, consisting of counts and respective packs. + # A pack is a (sorted) list of sequence length values that get concatenated. + tmp_strategies_per_length = defaultdict(list) + strategies_per_length = defaultdict(list) + # Index i indicates here, how much space is left, due to reversed histogram + for i in range(max_sequence_length): + n_sequences_to_bin = reversed_histogram[i] + length_to_bin = max_sequence_length - i + offset = i + 1 # largest possible offset + while n_sequences_to_bin > 0: + if (length_to_bin + offset) in tmp_strategies_per_length: + # extract shortest pack that will get modified + n_sequences_to_pack, pack = tmp_strategies_per_length[length_to_bin + offset].pop() + new_pack = pack + [length_to_bin] + count = min(n_sequences_to_pack, n_sequences_to_bin) + if n_sequences_to_pack > n_sequences_to_bin: + # old pack gets reduced + n_sequences_to_pack -= n_sequences_to_bin + tmp_strategies_per_length[length_to_bin + offset].append((n_sequences_to_pack, pack)) + n_sequences_to_bin = 0 + else: + n_sequences_to_bin -= n_sequences_to_pack + add_pack( + new_pack, count, tmp_strategies_per_length, strategies_per_length, max_sequences_per_pack, offset + ) + # clean up to speed up main key search + if not tmp_strategies_per_length[length_to_bin + offset]: + tmp_strategies_per_length.pop(length_to_bin + offset) + else: + offset -= 1 + # Does not fit anywhere. Create new pack. + if offset < 0: + add_pack( + [length_to_bin], + n_sequences_to_bin, + tmp_strategies_per_length, + strategies_per_length, + max_sequences_per_pack, + i, + ) + n_sequences_to_bin = 0 + # merge all strategies + for key in tmp_strategies_per_length: + strategies_per_length[key].extend(tmp_strategies_per_length[key]) + # flatten strategies dictionary + strategy_set = [] + strategy_repeat_count = [] + for key in strategies_per_length: + for count, pack in strategies_per_length[key]: + pack.reverse() + strategy_set.append(pack) + strategy_repeat_count.append(count) + + # Summarize efficiency of solution + duration = time.time() - start + sequence_lengths = np.arange(1, max_sequence_length + 1) + strategy_repeat_count = np.array(strategy_repeat_count) + n_strategies = len(strategy_set) + old_number_of_samples = histogram.sum() + new_number_of_samples = strategy_repeat_count.sum() + sequences = sum([count * len(pack) for count, pack in zip(strategy_repeat_count, strategy_set)]) + total_tokens = max_sequence_length * new_number_of_samples + empty_tokens = sum( + [count * (max_sequence_length - sum(pack)) for count, pack in zip(strategy_repeat_count, strategy_set)] + ) + efficiency = 100 - empty_tokens / total_tokens * 100 + speedup_upper_bound = 1.0 / ( + 1 - (histogram * (1 - sequence_lengths / max_sequence_length)).sum() / old_number_of_samples + ) + packing_factor = sequences / sum(strategy_repeat_count) + + print( + f"Packing efficiency (fraction of real tokens): {efficiency:3.4f}\n", + f"Speed-up theoretical limit: {speedup_upper_bound:3.4f}\n", + f"Achieved speed-up over un-packed dataset: {old_number_of_samples/new_number_of_samples:3.5f}\n", + f"Runtime: Packed {old_number_of_samples} sequences in {duration:3.3f} seconds\n", + f"Average packing factor: {packing_factor}", + ) + + return strategy_set, np.array(strategy_repeat_count) diff --git a/packed-bert/utils/packing/dataset_creator.py b/packed-bert/utils/packing/dataset_creator.py new file mode 100644 index 0000000..b0786c8 --- /dev/null +++ b/packed-bert/utils/packing/dataset_creator.py @@ -0,0 +1,299 @@ +# Copyright (c) 2023 Graphcore Ltd. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import logging +import time + +import numpy as np +from tqdm import tqdm + +from .algorithms import LPFHP, SPFHP +from .dataset_templates import PackedClassificationDataset, PackedQuestionAnsweringDataset + + +""" +Currently enabled supported tasks: +* Single label classification with BERT +* Multi label classification with BERT +* Question-answering with BERT (SQuAD) +""" + +logger = logging.getLogger("packing") + + +class PackedDatasetCreator: + def __init__( + self, + tokenized_dataset, + problem_type, + num_labels: int = None, + max_sequence_length: int = 384, + max_sequences_per_pack: int = 6, + training: bool = False, + validation: bool = False, + inference: bool = False, + algorithm: str = "SPFHP", + pad_to_global_batch_size: bool = False, + global_batch_size: int = None, + custom_label_key: str = "labels", + ) -> None: + # This list should contain all currently supported tasks (for BERT, currently) + supported_problem_types = ["single_label_classification", "multi_label_classification", "question_answering"] + + self.max_seq_len = max_sequence_length + self.max_seq_per_pack = max_sequences_per_pack + self.num_labels = num_labels + self.training = training + self.validation = validation + self.inference = inference + self.algorithm = algorithm + + # Verify the problem type + if problem_type in supported_problem_types: + self.problem_type = problem_type + else: + logger.error( + f"Unsupported problem type given - attempting to detect from number of labels (default 1, unless specifically passed). \ + Pass one of the supported types: {supported_problem_types}." + ) + raise Exception + + # Verify the task + if not training and not validation and not inference: + logger.error( + "One of 'training', 'validation' or 'inference' must be set to True when calling PackedDatasetCreator." + ) + raise Exception + + # Verify num_labels if not inference + if inference: + logger.info("Inference mode has been set. This will override training/validation mode and ignore labels.") + else: + if num_labels == None: + logger.error( + f'For validation (to evaluate) and training, num_labels must be passed to PackedDatasetCreator - num_labels got "None"!' + ) + raise Exception + + # Get the unpacked default data columns + self.unpacked_input_ids = tokenized_dataset["input_ids"] + self.unpacked_attention_mask = tokenized_dataset["attention_mask"] + self.unpacked_token_type_ids = tokenized_dataset["token_type_ids"] + + # Get the strategy to pack the dataset using the algorithm + self.strategy = self.get_strategy() + total_num_packs = np.sum(self.strategy[1]) + + # Provide an option to pad the dataset to the given global batch size to avoid skipping samples + if pad_to_global_batch_size and global_batch_size: + if total_num_packs % global_batch_size != 0: + difference_to_batch_size = global_batch_size - (total_num_packs % global_batch_size) + total_num_packs += difference_to_batch_size + + self.total_num_packs = total_num_packs + + # Prepare the manually padded constant sized data + self.shift_cls_tokens = True + self.adjust_offset_positions = False + + self.packed_input_ids = np.zeros((self.total_num_packs, self.max_seq_len), dtype=int) + self.packed_attention_mask = np.zeros((self.total_num_packs, self.max_seq_len), dtype=int) + self.packed_token_type_ids = np.zeros((self.total_num_packs, self.max_seq_len), dtype=int) + self.packed_position_ids = np.zeros((self.total_num_packs, self.max_seq_len), dtype=int) + + # Task-specific dataset categories and dataset class definitions + if problem_type == "single_label_classification": + self.dataset_class = PackedClassificationDataset + + if not self.inference: + self.unpacked_labels = tokenized_dataset[custom_label_key] + self.packed_labels = -100 * np.ones((self.total_num_packs, self.max_seq_per_pack), dtype=int) + else: + self.packed_labels = None + + elif problem_type == "multi_label_classification": + self.dataset_class = PackedClassificationDataset + + if not self.inference: + self.unpacked_labels = tokenized_dataset[custom_label_key] + self.packed_labels = -100 * np.ones( + (self.total_num_packs, self.max_seq_per_pack, self.num_labels), dtype=int + ) + else: + self.packed_labels = None + + elif problem_type == "question_answering": + if self.training: + self.unpacked_start_positions = tokenized_dataset["start_positions"] + self.unpacked_end_positions = tokenized_dataset["end_positions"] + self.packed_start_positions = -100 * np.ones((self.total_num_packs, self.max_seq_per_pack), dtype=int) + self.packed_end_positions = -100 * np.ones((self.total_num_packs, self.max_seq_per_pack), dtype=int) + else: + self.packed_start_positions = None + self.packed_end_positions = None + + if self.validation or self.inference: + self.unpacked_example_ids = tokenized_dataset["example_id"] + self.unpacked_offset_mapping = tokenized_dataset["offset_mapping"] + self.packed_example_ids = np.zeros((self.total_num_packs, self.max_seq_per_pack), dtype="= end_char): + start_positions.append(cls_index) + end_positions.append(cls_index) + else: + # Otherwise move the token_start_index and token_end_index to the two ends of the answer. + # Note: we could go after the last offset if the answer is the last word (edge case). + while token_start_index < len(offsets) and offsets[token_start_index][0] <= start_char: + token_start_index += 1 + start_positions.append(token_start_index - 1) + while offsets[token_end_index][1] >= end_char: + token_end_index -= 1 + end_positions.append(token_end_index + 1) + + tokenized_dataset["start_positions"] = start_positions + tokenized_dataset["end_positions"] = end_positions + + return Dataset.from_dict(tokenized_dataset) + + else: + # We keep the example_id that gave us this feature and we will store the offset mappings. + tokenized_dataset["example_id"] = [] + dataset_ids = dataset["id"] + + for i in range(len(tokenized_dataset["input_ids"])): + # Grab the sequence corresponding to that example (to know what is the context and what is the question). + sequence_ids = tokenized_dataset.sequence_ids(i) + context_index = 1 if pad_on_right else 0 + + # One example can give several spans, this is the index of the example containing this span of text. + sample_index = sample_mapping[i] + tokenized_dataset["example_id"].append(dataset_ids[sample_index]) + + # Set to 0 the offset_mapping that are not part of the context so it's easy to determine if a token + # position is part of the context or not. + tokenized_dataset["offset_mapping"][i] = [ + (o if sequence_ids[k] == context_index else tuple((0, 0))) + for k, o in enumerate(tokenized_dataset["offset_mapping"][i]) + ] + + return Dataset.from_dict(tokenized_dataset) + + +def postprocess_packed_qa_predictions( + raw_val_dataset, + tokenized_val_dataset, + raw_predictions, + n_best_size=20, + max_answer_length=30, + squad_v2=False, + cls_token_id=101, +): + all_start_logits, all_end_logits = raw_predictions + + # The dataloader drop_last affects the dataset size due to the global batch size, so the number of predictions may be slightly less than the total amount of validation samples available: + dataloader_cap = all_start_logits.shape[0] + + # Build a map example to its corresponding features. + example_id_to_index = {k: i for i, k in enumerate(raw_val_dataset["id"])} + + features_per_example = collections.defaultdict(list) + + for i, feature in enumerate(tokenized_val_dataset): + for j, example_id in enumerate(feature["example_ids"]): + if example_id != "": + features_per_example[example_id_to_index[example_id]].append([i, j]) + + # The dictionaries we have to fill. + predictions = collections.OrderedDict() + + # Logging. + print( + f"Post-processing {len(raw_val_dataset)} example predictions split into {len(tokenized_val_dataset)} features." + ) + + # Let's loop over all the examples! + for example_index, example in enumerate(tqdm(raw_val_dataset)): + # Those are the indices of the features associated to the current example. + feature_indices = features_per_example[example_index] + + min_null_score = None # Only used if squad_v2 is True. + valid_answers = [] + + context = example["context"] + # Looping through all the features associated to the current example. + for feature_index in feature_indices: + # Separate the feature index and the pack index (i.e. the index of the feature in the pack) + pack_index, sequence_in_pack_index = feature_index + + # We want to ignore any indices of packs which were ignored by the validation loop due to the dataloader dropping uneven batches. + if pack_index >= dataloader_cap: + continue + + # We grab the predictions of the model for this feature to map character-level spans from the offset. + start_logits = all_start_logits[pack_index, sequence_in_pack_index] + end_logits = all_end_logits[pack_index, sequence_in_pack_index] + + # Update minimum null prediction. + offset_mapping = tokenized_val_dataset[pack_index]["offset_mapping"] + + # If squad_v2 dataset is used, we need to account for null predictions; we determine the minimum null score using input_ids to find the cls_index of the current sequence in the pack. + if squad_v2: + input_ids = tokenized_val_dataset[pack_index]["input_ids"] + + cls_indices = [k for k, v in enumerate(input_ids) if v == int(cls_token_id)] + cls_index = cls_indices[sequence_in_pack_index] + + # Since we know the relevant CLS index for this sequence in the pack, the null score can be evaluated + feature_null_score = start_logits[cls_index] + end_logits[cls_index] + + if min_null_score is None or min_null_score < feature_null_score: + min_null_score = feature_null_score + + # Go through all possibilities for the `n_best_size` greater start and end logits. + start_indexes = np.argsort(start_logits)[-1 : -n_best_size - 1 : -1].tolist() + end_indexes = np.argsort(end_logits)[-1 : -n_best_size - 1 : -1].tolist() + + for start_index in start_indexes: + for end_index in end_indexes: + # Don't consider out-of-scope answers, either because the indices are out of bounds or correspond to part of the input_ids that are not in the context. + if ( + start_index >= len(offset_mapping) + or end_index >= len(offset_mapping) + or offset_mapping[start_index] is None + or offset_mapping[end_index] is None + or offset_mapping[start_index] == [] + or offset_mapping[end_index] == [] + ): + continue + # Don't consider answers with a length that is either < 0 or > max_answer_length. + if end_index < start_index or end_index - start_index + 1 > max_answer_length: + continue + + start_char = offset_mapping[start_index][0] + end_char = offset_mapping[end_index][1] + valid_answers.append( + { + "score": start_logits[start_index] + end_logits[end_index], + "text": context[start_char:end_char], + } + ) + + if len(valid_answers) > 0: + best_answer = sorted(valid_answers, key=lambda x: x["score"], reverse=True)[0] + else: + # In the very rare edge case we have not a single non-null prediction, we create a fake prediction to avoid + # failure. + best_answer = {"text": "", "score": 0.0} + + # Let's pick our final answer: the best one or the null answer (only for squad_v2) + if not squad_v2: + predictions[example["id"]] = best_answer["text"] + else: + answer = best_answer["text"] if best_answer["score"] > min_null_score else "" + predictions[example["id"]] = answer + + return predictions From 32603da41d6c97dff2e605cd612c5622e8287027 Mon Sep 17 00:00:00 2001 From: Arsalan Uddin Date: Thu, 9 Mar 2023 14:38:27 +0000 Subject: [PATCH 02/11] link fixes and images --- packed-bert/images/go_emotions.png | Bin 0 -> 110514 bytes ...dBERT_multi_label_text_classification.ipynb | 5 +++-- .../packedBERT_question_answering.ipynb | 5 +++-- ...BERT_single_label_text_classification.ipynb | 6 ++++-- 4 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 packed-bert/images/go_emotions.png diff --git a/packed-bert/images/go_emotions.png b/packed-bert/images/go_emotions.png new file mode 100644 index 0000000000000000000000000000000000000000..2d9da10d24ea44b6eacb1a5d7feae1d5aeea9f35 GIT binary patch literal 110514 zcmcG#Wl$bb(=9qkg1b9`;K41p1PShLA-D$!?!n#N-QC@SI|O%k3GST9`@LV?dw$(t zcc^-bnc1^zPw$r1t9ydwWyO);@!q6h%c?EnCg1q%sY=~DWI37(Lfefc79 zW?%pS)X^T%d{R(Cr~xFt;W?zH`X8LX8ElJ7hv8I7{`r*J_eX3* zyLV?M=cRv%MOOzj!bPO_oe77A>K@AJtOKv}II0j(YrDz2nny8ub;% zdIDlAX8tFa-QC6NHqlKC)0f<8g&KS(>omyX-X1B8IkbXpwyu3g(Z|>VPTgyS4K&QHfne0LlP^ z0YeHyDpv#w3JL)VmWhpti32)DOt67VnR*I%mz0#650TxhQIkv)H1Q6QYr!%lpkD|B z08&6gL`c~=?PS$K9ZQ1%xVRf#`6U$?6jDT;MOw75`km8pzQp1L|2V~>yrs0FrJ*7u z#HU*`=p7{`JS}xr5M;=-EzjPQK}GKkq?+{4&bzyn0h6uKQM|OzCbDP&`oIT2RAE@~ zhxWb75}M$@|G?90nO=AD|6B%t44Bq~{%4HvATBZB=jRva$oAh0upajA9v%=Q1Ze*q zjW@7eQ1Zu?w$GW=Z}N_M(;6LEXsP>j8bs=c`dDDZ+YSp5rF@Qa6hVCiOe)&<3aZ=C z{=0-I<|G@D|1Q7~aqs_`()U5^zl#*^|JL#UIlR}+3)Qcq) z3a7H=o}={iRCj6Wp)=s2|IG=(gr2L3XYvQlhdGY9UAj1n)8&#vJtvp}7ClzU*2YsE zZYM4tzTHF;s^{k~LyhfT=iOP>y>XHh0Q4I{&DA7_@E}ewg)pp#zSpNT*Xu4vYscBW zUK!1MY~%e3+Ez-OnUi$|Tp3zvw=!f&_* zDNc?15|s-Y7}3-;9KWY~PL2sZ?p|YaDWCgg8FiS<_WtNotW4S*gk@pq169s_b5U90 zJ@t;4^G3S)0y~W39IY06WzN^IvpQ)7 z7zJK!d7nnUPJC)Rebs?klqLtXJMFe5R%RISRlq&9LnIz6#g;Q*Rp|d^R<9?JK_Xr{ z#Ty=K?8O+1fdKQl#)i(Rqp>oO&+YP{v>zq=Ep1Y&5d&SRsZg{-#PwouvC{Epx$2vc z$~4vNRi2t<0UGefd~DCF!#o5Nugjg>6{T2&+v~B%g9pXEo;*aP=a(5q_G^g+PH$`7 z!O-ii<+7HuR`x;q1zGd~f++e2zmA$69JtoT3ffb3caG-fMW~iPMqUU7k)H?!sacp= zLQx;(Um*t8TX(d`(L0ocR(Dnni64hS9fw!8D^@P_w7T8j(e}xKXNaV*Pi+4x{m4Ca zZ+44GoCyQ4$;hO$KlnBfvFrPlrz7#4ayCWA+WSwC2!QrO@v5bUt-*Ge$>#h=*+R?B zu_bVQ@KQEjijyDKWB&4zLOe1M|5KaivFA$cem;68x*fFvkH@9FXbcd=lL#vDe7 zWzHN(46oO0txSd$OH1jMylACN0XGN$zux#<;K`Wrb(c;gSErDpJ27&#^R8=kraeV# z7G-H}s9E@8c6&3?W?^ZO%~bIXp%3+gpQOJqgI2xaW64;c6c3r+7nj0epF(mQ|G=e% zg(o&kg{3K>gC|GogEnPKlfu$g6;I5ku=HD$&(%M>BlB@8@16{X{$#9bQgt{lmT0us zAUEU_rN3E#Hri)by7~B&(&yW9%zwJl?ojz7DUqf!C-*xGE6ansXP>03u7f!|bbm{m z5^a36Q;sfW=d^kKtgQE1G%!{Bv42&!|G1pC50 zRkQqds<$;*j>LiuKd@$(POV(G!17_j4c4zi=ut(n(MEl;U_`xH8xC+oPSfPI`Q?el zN^WO&#i-Z%^e7WxpwU_oZhm}MoI7ROYa_R^t$NHsq&TOJe`xs5(+ zF2kM}f^0R>ljp3dy!xf)&8~qe7kp%M+}*Fs3b5~;e$y_4i*ZS zeH?nS9ygj^v*D$j^PjYw&Sm3Xg{zaB&a>8HO=K}sgOMC566rX~*V^3Nyp_pzvC)N( z*e{doe-s3&AzABBb{l}MO~249gU{h0NeGx6F7!|}PEb{vjB;HLAZl$ma-D=jT-(=d z#|J+0$(=3LP_>*k=sVxq8v)BD?g`mdm^&VeQW6- zBd{%<`}FoC--VW}-0#oHjOK8*cvz@MwH#l~db|GxHhUsZ?QKeXy@U=&u5`XAUQm7d zcDb?V%lsSHzU|Mv*Paq<_Q3G;xonG0GEOe6I;VmDU5N>V7VO$uMjop*b9;mlyzQ3- z3MG@ugGC!m9MT)3>4129#teb%sF7D~ zw%lfW)BL{u5Q37^wLUsOiKsZNr#u&1Jb)_mq3i8H$7+Ojcl?+3ZIR7EIsB>J*Hwa( z>nsJOe%XuxSyn`Q=k)}g)Yq?WS0M<}3d+3&r~T3y$D1mRg97h_-yqt+ieD(&*|xi9rpi<7o3-sFXULazyy?nOJ zjNB{b^I+Eg4zmCQ_m6u-9*>7nf%ji*wL2Z&=HDGN=(Uc}qXk2=79wXp%=`!(wpyUz zYi(F+mr1OutC-X-p98$L*!OwVZt+CQO4R1(ZCD_>hKG3%ZG*a(K+nBTzxLy0&f~>@ z&v6S60B|M^0FK`0Emyz^Bj8=Pr)i&%eB6B7Yq@ygtc} z>eVpQJftf(wp!%7)b=AETqTc)@?$pGi1fF)CJ*1Z7#Ugh$uc2c-DMJpUlSQbnLz^* z9ftD>tL;m7p9k(y`QnC*bJllkDF4G)44WV;%-18= z7-?~r5k2;q8+oPOs?rmTLWV34n;znB4OgKNAvE23Anf3eUyu4LJKl0WMk6utEVUvu zK0f9pO}scuIs*&HeJ2@B<#6w_QNj(F(i>+Qt#J`xcs2_AH`VlvqKOo8qJVD>TZL*ADV;q!RaJ4$em=cw)5^?V z=}A?aprYVs{=9N!V#`^n^9_nR?y{oYLB-gRWvEO7>Q(zb*0Ug*Px5}3)c~fB3rkr< z&jR*^^B#t9|Ll5gbr}#mC$>VEJEPxvLIA#gWC>QLj2^bT`uL@PRHwt0D$xwu+dcR? z(ZxdTx!&U8DE-U8=1nZ;teVj<2%xb`!RLgUfj$|6LYz6}!D&EPX;w&i29t zqDJB^**+>J@2!4Zsqo$|=MJBmV5`~0p!!KNa&9;hOw`*8x znKk=4DA2~+7mUa5NQErS?@yL`q0BV0W9t^Q!FT># zB^BIQ`_(V(D~@C+@KfTFg8Zfb2e}I@G$cwTe*Lj)YSKZ+-%bgStL-2fm*xjO>Lu6@ zzrpzgCUl0z5~_UDzCfgMmg{!$RgA>Z>lJ#|!ef>GfQUF*ZtwT8sj^}|Z!vt+nyW~YI7&=%MY=U(5H#5s zpr5q*Bx_d-_2dBQt-Gp;0PXH5F*!6zm>ZltGJi%4&#kso@=l6#Os6h;P=%}cp_6`9 zd#6NT?w&5bgiMWykC@wjO;3!hx7qfodqE;JbXjg=Du?{pR`sRb1l&lmm=0UH45yPD z15hgp33v5JW7pNrHjLYOOCK7nPEK|P=`}iB_rsS__;*n=VXwlA1Q(h^wdSo%yrlLS z&X;QTA7*fx>OFm*0Kw<&``OOQO>9*M2aU)1oZNlLJLpq<4oBE?tG?^`%C*tVjwl4j zhaN2J$H~K&r-<`47t`hVBBpg~B57$6sDOu4o{qvFv#K5=bMzXG9yf!3dz&eUTI#eK zJ=mX0<)_p;i;ee;SDzerjiI0-4k|qeiJq8PQqnoFQg4sezc~GV@B-tTaiLa&1+nY? z=#;$=jO;sYj~~@{fR2ADh;ul~mJD|$!PG(H>!n>v-G$YZ>K(Nb?G6~aSUV)9uL>;^ z--cyFrKJ;}>z{UZsZ#GU_+0s2eZ80xjh32RJyngldD1fyTFh=Btx?W0>eXzx>>f3@ zrmm(0#$J^yjTLr$K7A^0gib33!A!INtc}+*hC7A+ON9~-BqLEL=71mJC+b~Up zl}h;~tr6Nth`}r9KN#-Kd#)c!7adda!(AVqZCadA{rF`Uc6laZfA*c6Rm3l}F{u%AgnpwtjNBUsV9{c} zST}g#+bup%#moD!Kb=V(aXu#GkzdNDa`II{{i{>rK|j$n@g31f>cVz{W}OmE1Olc8LlE(5#CDQITHv^`rLR zyK~9rUv$oFISx+=@1;tV%U7~3p3pOvHWZ-l&bbF=0|WewjEpsw)oW`X{lod595`7y zp}fmR%ebd$5r7C1erAKU3AXc>RZ%cy7LOgt z2QIGC;yEiaOpbr)A|Z63yOwAik3$MgL8TBaHi{Ncs{BrAv<&rxVZFhxfAknz`}>6( z@K*dF;6xiW7WNM2-H&=Xiw8E{bCS_h_pS>P4}3O~*-E3$lp;WWc6R2?)OqPRj&>Fv zzf?;%(FHS6as6K=r&h!4i}XkpG~6J36W=8&i%8IaeGjeQ$|jz-H|!Z zg+NDcBXK-)=dw$sWOl<Oc`~J~ks{rQu)cNF(D8pz|bbI(oo-NdQ)=TN)NA2zRYlnD?Al}}Z ztI*}}*izRS*>`o|)i!q=8y|x5PFr~f_n$V-iMp*fJrxI;py;VWd0@HSD-)T>O|}i) z69vExSgd~1+s4aFbt1OA5k5UFs|-Bb8{U1qwVdye$qgtE=)TyrSg+bWy|wAN8TO5 znu^|03)l@mcuA?ZZu$Pg;+Qg`KFwDDe4d}<1oO>mp$J6j`MBeEW{{j8AO6+Qj{TGR8&&LHs@9Qa`8qs`0hP5j--H$a?mX6eF2(sH-E=o( z28Mt=?1WpR;dCzBAIs}fbDy4!_6x8If8FjGnzS?>*~Khs;AyrS^OA%YxbQti?iojt zFcg(M+I$njV(k2|y%_x4Z(#vf$HE<@vIT@P@4eu)o@!Q98!P-hToKV89e1Si9(y^E zuj!~K&X=mW=FbnnP!P6XV=O~Zye?2;^F+DwlS}-mn%vd4c1{)h#si+eV`mR*3cgsY z5L>$-+Q^)2soLy=TG+$+(A`R~2_t45R|{~nU^9C0Yi7S@izM4DS99fhqE_{W@Zu^1s&JuZ%jUxjtom9|*wsY|v*~Z| zH7JKnANyK+`x^TNGaC|{0XriEo9SvTmO3_ahvUS56t zqVU;EosQ0yZHm^_aOb`c9tc^&eE-CdhTq6%mBMKKG&yb*LcYF@E$+0G{joDP4+MovgfjZJPrLQ4p8BNQDLE~ZcSO6NniOR5=bDDF#<9IX{LY51O}kvxN~_WET)^U*XGGsN z*NF)+SG5$H0(W$~-Svi~-F^oCR(+l!Ag?fgX))0w{_@$eiNV9tT7R~<^^9lZ%I}w) zo0E-Bv)i%cCXwA*9;JutHR_`yAv`yu%W`98ajx}*95$&anObxN$p z%twCW$J*P5xs{(pHcq~OSx-*k=dc$ktK=pe{c&29E94s;!2bEy_hd1dr6VyK>M1Z8YxDh$^`2h?^mEP! zo|~QPZMMfnk&z!<>dz)iv|V(LX!&m?3`H<;uY*`!Q8~frt$WF55A~1YOr%7GEgYje zkaoChnB{WC;^)FU6m>FC|JW>@SY}8U2kAZd=T8=$dqS|pRBp}-hru5(XTTf>akUAJ z0oykNwSt>v*K+*u$3Ii;z>kwj1_uA7@(HG?zXk|m1CyrfL(Z+DdR zv0APBUBz3E9-CJ#?|Z)<(AD6V*hV0Wq8JqDl_Vr&s2Jj^TI&jl^NU8zQDftz?z81w z=y?+*`0lDV9GE8BfBK5P-zM|eaV{vz`Lxn$|8R1c>wMneDe3;#Zmi#f_{P;{0be~P z-HQt}4tV9epEk1<{?P!3D@hc~3~MN2Y_x9Fuc`B+{-u6bFtxv5pel@bFTT?9w6XSu zp*21uf!X@PZ-Z_ibR^Uhn@#41WUuk@uKwgCyIuSfJw3Vg(W7;R`cScGs_|p>Zh==4 zOMidG!T<%}JkQL5O%L*-y5Xr#rc({a`BzkcU~yEf)fJ+;22LIrKg}IZ_=|qvMuev` zlpC9z4<;yr?nyHLp`ajuT}U^dm$B_(taN`OW7@-lleLkM<*^W7*8ya7k4Ew?q(0zG zWNxHrc#3)Y4rID$q21kY>iCMw@wq5-<}ivXc);K&E7bmZQmQ-s(`sBEAm*sGT6=2} zNh_RB@qyXjidh|;7^1`UeqXkMN^tU#a%^TaMPbJI%AvDm1+ zw$GS|)y1+_OuB#0|N9y9tV*0g4f_mLrSn8tAhrBlMpFoTzO6`?7=eVYI0;SCnb&pb zIXKqyp!4k_xk{?BV!dTTRH9f}CZ(fIa`#d@P)v_xk6;R>s-}U3m2+fb1jgIViHvoc z)e6C=NO%HMS;EO>88%93to-PY9u<_gVsV=*v-@t&tY!G*N?IE^%fFsoUcD!*6{@K3dFKZ=7j7{q1!FqOPA0k0@Bz@(QX-QyPwb z-?&2oyXo-gacpA*OoTi#qHm0j=RDoJ>aZWQ)#x5^3$mid1U8(bANnWG7KTEhp73we z(&C;|%Jbj>5Tc42cYUsKX`p74Ajs311_PoMl*of|k3 zu=l<<#P8i`-L2;tVKcb$p=mP>$~!o%3W`JWXKE!~9)){q7+Gh0+^eoG%=v=1T#WHv zHk~O-KMC|=iDe6pEFx;%o@0;Rildz0`Bbj;xqnWnNT`+mxhi-mQ0Tm>iHrzYe3BC? z`E?Af!k|XDK(tV7);L*CP`o$AdW6X!-&2CKz^ zj6C%5Uwx5}`Y&0S*$-#m(kq^q94&kEt2rtc3k0w!EcT7PIwPPM(>tHDZ3SVo<@VJ( zwt`5E@lj@KbT}Q!PH#rQ>=J2iZ_g0vK=JNeVE+eE6r`YA`=|A#-A=K!=Ke8rwj-6$ zLM*Nx{l`?bl{zgXn->+i?Hr|;46G$cz`HWBNYeAQZjig89|Up}G9>WSxRepcXn~>V ziQ$Mya4lYU0o|U&VykDfbK5_V=Z#8)}v`RRIE;PjZi;B;`E5)L71ZEBHw zjPgm3j4yY-IYKNqz0=UZc3?Vm2mPj1Vm)Y3{J4>L!?-!=I5Cl!&r!%LcDK|Kb5RGi zm#J`E+iczAIgG0X?GxOG8xP*QbIoyoiLcTOEHQd8paBKqeQpy_V{rQL@u&Pb|?ApHOAcGOm?KObh9Z!=g#Mr(0%acrz_R?+zzq~>eq zWEKzO^5TS61N<}EspG#wkRG~RPB)LJJvXp`wU=sNO2l*4rNITT`gQG(Nc<8TTdSXr z?BVe~e7nP|T#w7?W*QHyrr-rA(C&cX8+WR3ai-~jZ&}sJX4i2$#M=9$pqyK^WUF6R z>e7Ay$(+;Z=iQhE_Up|PqQJVQk6S2l#oS!Q#*n*05ZTLeSBX$(Eli)jB5gWw+kYS#0Lz z(dhv-f~c3tBXBf{%SmSEWg%~8o=R5FdQIPO@ZkqT*G2bg*m?Kbxa)y8!^(+VXIa_I z&o0KRxp9rA4vnX1U-7ZcnymK=n{*oFo=IN!>sJy(l_2@fX*WI{*6tjL z%N0)Owr^GH_(#ShG8QW|I?Z<|jR&r3-`r(@bd!nU9^fsU{cLjAsa+LmD)We^|IhY$ zEnLFh%m_IbUfOvSwga)Jo)W^#Z_X`#`ewGw{$ORrF)=ZM^8ToIIg+w~D zC%`foBv3A9-{tiJ@pj|h(UoOb852stIsQTBDxqSLFUJOk1=?UDyHPDVBRtI=WUPC{1}#(}Eps18bc?jK0x%}_}AT=|Ax04l8L z>WkRDjYe^2)q1Occ$p75l^z@)79JWh;KH>C(jr>T&MAVVY}DT@XDeFMU!FR@kzMOt zpnUF@aWY^3U2`9C{B(C%WjPzssMT&lIxr!cJ5{0G=(gLk>aT`GG?}k*<$Au_t4CL$ z2iJ*nCo+G)ed%er^P+A|FQX)CO!Rd3m+!Sf)9mONu`~O;xCsJqu$WKcvA@pQ)mmIU zZalj?cvIT^cI!CzW~tpZc6&PIxVO@|@VQ9&DA|5~G_AFNP+@fJ_ZoYtZEp4_t$KsS z5+X1Qad|+0K_Bma;pyb;(L+w)%t5)`zW)1V0F%cL2l$-8*y^xO2_vo}jzCG7gVZ{w zp`m390hW!e9V=eN4>S0sj5$b;GNll6i5uisvTh}L*Lu9jqRM)PV==q*-giovWMD!Sv)-vJg9- zV)?4L^LO5PXT0ft)~-z`e6@2zY0N?hTynD3YMLo8sbJ88;y@-cD616k04c!bDhn%$ zM9CHO>g0v#>71Rvcc*jR;~iP%Gf`80{IKQ^&Tnp;KjznTOfxpU4Q>eLZv<+5iJ-xn zvgF8wy=q7Z9sv|!n!@Qe6wp`tLZ{5;t_?CMhVv?$byQiE^*IMI6 z;UlQN(*Wz?g#4z(WSN`1P#0QRz4wcFR;P`()VB8VUA*4cb=}iQ=rT6kxt{U4qsB#n zH2#)u=g7~Ggu@E~8B31e=0hCI@wpS~G2q|D(sEaU$%%le zI9+SbfAype!RO8zsrI!9A!zr_De6>}7HF|i*^JP72l*P*y?Un0Ctn#6*j>_hMLC+x zESWXW36{;v)UFf<9dQ!Lh{F=FjT0wMlK5RrH+|?%Ez4EuQyXjsY9bHXZ&}F@d%@-s zKro)WkAD%3Ru-g!0~MGxk~Q4<*M}C1t$VqR_xe=*0PnEt(LtVQ##p`LS#vF(WqjS) z&`-y7!HGkRGczosqocH{c3XctEyIt~PSS!>onrO!P=yIc5*oH5iSkT$jwqcP3T)-JWVU8Yfp!f15c&t@uSp$4Ae; zwYM8qr(B)f9O%>zvRb02xv*S|$zW^->)s&!CEd4ozP^D`q&)nDHlNAoQKn3LIvnR< z=U9C9;L;BxKG+mihy#)JC3e~}^HxTols!AqrcXd(>&iuiHr{5bsNw5%H%k82O1P_b zV{*l}Z?9$5onx=vq=I0}z`V7Hkt(h? zFYadgW``QtiPoLcGPO)NPk%~uM@CND1EYp8H#u{+GI$e*cR<%>J)1@_cST!HxSecK z3DGq_kMo}vpG?TB@B;y{a#gU2W$WU#+=~19gw-_dm9|`rwN4GgF$+0m@134ZRm00K z-!rmn65oGNd?|czY#9ZrE8|oOQB@}S@4&j&&($=rl3CMa?KA0ixKB{6NBFOO2Lf2> zbTinCd|{bc2fBI(wdOc(t+;Q$tI=Rx#)lSGgVi0=g)$;zd^9lUqT=jy35-U`arjcB z@{{&y`Glk?nycXn#%ck3PNU<_AZ3-C1@w#ki!f)rF=tpUJ68A5|M^sNQ)}3-da)>PZVlO@L zQRnHI+apg1;mh^ZeBv!uoev37JJ<}+CRf{AN zhQ+!!*|wqQiHPvLj!XA=pryY!;PPc!@Mt1p4|6x9cs=w_yr_Z|y%g zS@v#HrL*Pb*QPouA_uxUh)yj9%$(JgasQ{m%CJcG#n|D}G!b>zO2I5=o%5YF|v6EKz3e*8ph;sLVJN@+Q6q|GEwtWBRO&>|y?qGxtX7Y@7FI{*E zq!1hv^pohEa^Jb@eyKdlCt$v&Fx}N4l`)Q61O#D$^^hbk3?EUm0fg*IVVXcSInt+m zYgOv^FgR5VS|zr^2c_RJHKaY!bH^b7NI__h-(_SMV^Tc2upX_rLv-aI@o>2S;7d?f zKurmJTvpw-5m-Gtu>V$fG`ien>e7JZjf?&L%B4;^d&gKW-^6Htx3T#A#UVGSgH()u zrVV+%19>J-&m@kfi`1FoZk%qie0^ntg4+iZ~xq3c%@LQ8tMBa)? z5Vk42HvpL8O_~R5;Ptmt{53~$X!WHj$e8j%%}p>l!k)~DyDQa0-U4rh2tb8t zO2MVmv=-DESS)q$cf3P|1jG^wB-(RaJQQMGppvwE$iWU&Z-_}qQ);pb*+tIXcq^I3 zIcQ-?o(=#E>6!jJXUaFV^n%`cF2Br9X9ZzR4KHEJ6HckPie zQA=~p5(Tl$U|PKwGIhLm^aE|OJi)Kj82GSH3s3;5pm(-lYL3KurRjARVWl;{e449RtP^{)XA{;s7(by3{q&^_qe3y^tt2yfkM*1KP<4|KQfS6 ziTN~JnTZo+PxvbAmg=P`idPyR5R&BuMWJF4x2683Ktx)?l<9|XkOcHk?%BOIRZgHW zDTSuqXv9)V!j{Wr7bKFINVBM!A&ypC170?AT=HD-$~{}6(cP{rWD7_B&}nmA#c#@l z@?+(WD)TRz>g3}Ch5|Z_Kk>#)XI=IQd{sY`aSYXsn^y-@a|&R9sX_A{_4pqV<*90E zvjKdM_e(1xq6?$}c^J$x_q@Wj^aXIJ)b~vNRRZr#Ng;`I$Cde$gw86N4}U=W&E?~{ z`SB}wVwata@HRtJLRPWpi?G4EbmiiuKx&>p&YKG5L)0e>&XOv1cV2xsa5CPr-I{7s zHi(=w>Dr-?oIoM{Nr^TSx2ffgaD6L|L6(K9l5qKoxcf3oNfd#eEkh$loUPbpI!mJ; zVj+Oano?5$Cj3ZbAyV3aeS#ycXr$g=TeBJBi@y29D-__OC(9qD` z{TZg+l#Nk1uW$(?)^a#R+%N8ozKicR^eLEnI0ojIvY9-TJHqqZeUrt>wtH$ZWy02D zWe9OyDPH9F2^|6+0eP-oP@`@X8&@Ug(4FRk(tJ0xxmo z!1g;B7_ZIy%itCJv7Ja& zrZCmiw@-{^?Wi^WykZH-?*Yn&+u^2qO$YuDnAy~!c32-d#pA21OH8n$F|=od?aGZ- zQ~YuuNSP^aEbr5m)U5ej6SSB1uVtP%Dxp=LA!biAgV|+%V_tY)&}=|t!-3NF!F+_s zL?wnGM#c9Q4@#n((~CBgwQPw`yIky z*klyCB{K=70Fnvg-yLD_4xSX6dNLC>nVl<^8Vb$-R{N3h1~k` z%hR1!gQb~`jb5vxm73c8-k6joVnxVuWfK>O@qfpCuMO>j>LKqU?}8x%F3&o<3jNlWm;glk$>6lAb`51OeXd7S4b> zDH{aw2W(zc;}KbhZiBA4d(?EeKb5$S4BE>Ml!pfX4N}2NK$IvMWTkx`szw~Rl5O#Y_4|#6w)kk1M zxA>QDdia8f@>1cA%+vcy=$lfDVVa2)R3W&kWe@Q=WCpfsHiBr z^@=)eyq(mJor?s=pS%D%@;#ikcg}wD6mAHp?Ty}W2Z z;j*1LS8F2Iz+hCV+upIFkiej$S!+C2Ad`x%J6oZ{V{dN{n67ucG}=$GUaIk?Y|ebM zhW~g>CQ~C*cKX|BxP)C0sH4RKg38M3ODyZPkpVx{js@wzhM~oRy@fuec6b3`W;$k`V>m-j*M!kWF&5ClH~7!c%JK5!%=yr_SLe#2}o z!=aJ*wZQfJO(b1(DG!3<)B$72e!sc1jP4zygj<^IbBXh{b_nl@eW%0eT?{97qhZWq zWyzWA%iWRft*x__W>$~K>pSbdLDsnW0(AS3q_#1(>5p>b+HD^sJadiLVg2r8E#5UO zeB2A(8YYtHh?#1C=PZ~!{aZP|iHo4_9{b+r5%g0^?iQe*;y8}x{)=iPi?O*ZOHS2SgKwgxB7mBbn*QbP2)zhFG*ju3R#3HiQ%xWE6JKh5oM6ekJ2va+)E zp97!WT$vV)8r|wJ8v>(VtUtF~8Mc|_UGA(RZ%5aPHr-gPQC>2a>PBix^tDM?RvC+c z+Bf454RpDqBNWg8E~zlF%)FgF&FR&BhP#E@FIq=VsKvOXiFh`2{5s6CS38X6mG|9OZGgyFI7CF*(V<2>Hqw>f>8-PX|@ zEcnd}+9evj4|$;-GT{V+!2|*8O2c;wh{>$uzN`)UF5ru9n<5c!N;`vq!4JmLGpo_{ zj(4V3#%Wtq4A}!oa$H2$NRJ2qyEo5E6C!z(WW?Ekvmmwbbw?i5J3%OYo;b(`3>rsV zDo7DEwNMCBKZpR}i^5*r?W~#A8p3eT16?NU9zm!|)7gQVYJ=rsjZtU$R1OU*&dgr7 zOCom)+>Sf(q)6%L1eT_HvikDw&aTf#y;21wHaSy$nvwAsBkFg?*nWFc!qH;ng){*! zTJI-J^N#)?Xb!P1Rp3n&I38yFX*^$UM#mLxpml_)&VxgNedhuI|QaQ~@r|NCJacl$w;SY$d_-IOFF%keadj z$>+Urv{T7)y9WXQmCmhs-@uN7IQ+UVxR)w@lhnb{1zgaO6{y(0_TcCKJs$EJ71uS$ zJXw}tuTFY46iX3YoD{^!g3E?=SuSJdQt88ykJw!!NB+=S*vts$d9rK3l4U9zPELCZOPBiFDx_($7c_kq7nsRhO@J?xp_%? zdU|7HW2lsbRO$Ec-^s|y@dEx{TokF4CcM%nOFNz})#EVg!ut1s)~8&l`$8_evsi25 zbiSriI(u_GU#Z>9(x{y~wbJ51$IQ$uAh5o+)?vNcf<>!*AM`?yn3(7U!WwV}RN=Jr zbOH`bQ)A=XwemR0-PuyLOg=Z`@87p}cXtmTAWnztrV}3$dCcs-J7c--QjoAi#e2gH zKl5h$WA)*nhJ3HntkyFh zba5}%)}P}4)zBY+L?TzNy58Zz!@wY$E&uuRXExKxjS)VESw>-RRn%yjf+O{-G!Cl6G^ zN)fm!ki<&mv@td`Y-(->C0_Y4b*0sbk(*l+6b2WUi>qq_om$1Z~$x2aIo0vLRo*C;cD~=0{orN=@f-)Y^G8)l;WM5JW_T zWNRL0RooHWu_rS4LW6<^2M19@`_9hJ9FG2EQyY{|gNl`tlVfLZPft&8F<)`Ix2GDD z`|;yP5fPD{ot-#Fy$*?3N^ERw|6cbMcH=bF(suiOS$Fp~#p0iKb|re9p3#9ETe(Uy zPG>7F*9Xz6r4F{XW|LVy)rozrJ8F~|A5i(Huvn)7HVAyxGO1S2FSJUg1K~BNGb6*_ zN`u^{9}tvO%=?QWZDu5?h{qz}q;cs{;k)8gsr0@X9{&D)<}3TA#EfXaKc3FxbjGB& z7ELa0{-;PE6#_(py@4n=c>Jua_6t?|@86iMmK(w&BfTy*Ab8xie#3G6u(T{w|AZad z*J!=!`gmCXg--Gy*U;ZmWuQKN^$Ud3YR@4 z&gkNz#;1RhUf!69$;jY$y*7& zbiFz1`~5raqLcBh|Hq;JBynKbOH?%q%WZ6$Z&Wi1+k7c=JHA>!xh7-#@Ax(S29ifl=?xiQ+7)7hN?1y~dYDaizqo_`3^`ZGgD;?{-#QJpL z%FD!V?4wU z#TZo7j1XV`kn5wD6E&3C{lYkFP)UD9TX)N3;xz((d-HK_`YnC=P_fD}Cikd1@zN)e zrKVs@RmP>M@FXU1C&@y0FiY;4EZ^)wvg;$K%%9VJrGJ|?m*9Oh$=BDl6Eztmd zODn6s?|*MkmysG=?=SR^=gN`3?{07Rwv&HeZnVJ(?Q3XgKCu_#0>Vx*Ad3P^!gu7SdKV9I5lEd?lZKC>X9X9+n!5SJUdaYMmB#sZKr; zR+?SxFNsA`7I#;M&@DNgowd+m3mq{lTs931)kXgKQuXHhmiBh*fr*OxEehM|#}81i zpn943z$U5CzTN<&ZAg2kaIxHE0Wbb}!IA0d7^Etr5$xOOjLb}%A6W(i5zQ?v_-$w{ z4#(I;MCBx59)&+7T%y1JrV##aU=W@e`V4wNTeU@sRV~tQ!B_FC3?&0=M^tNfzN>Si z)R`|`BO6#T$Ay%^4!0R_2nZIFS>1i1I7p7;L<*M)!r2>+NuV}VK?9&yJ(??T^>}h@_(BT< z19ST+Pw}unoM7>1lBaXCHbeARLzcxq#Y$AtNf(bd7CbyWw~TN=qs_YYT$zTVVodcL z*BN8kV1bzqr*{HYdU>fbR0kYDZN~!PZ9+zgnXdr0_i_QSpUMEijPyR_Q)<-Iq4=)- znUuPbl(Z*zC8cj*RMyOA$5K8~i^+Wxpu3bD934HG%T(~cvvXRu4NXi zo`j0FzHI_eo4sR?!;gNz@+R@rQ=*)ieWvGNCUkrXiffG5?<*=!&c-i+$Ot$LV8nw4 zEG&+}+(ii!i--u^qGRw*b))a=Ha-Aq^L&%-HnuHlT9rkoKVJ8t**W5JRerA^N z#>r4&X;sSxv6((nP=w<$f8QHTe)Or=JiR`gal1Vk;vW?yi>|G$m8XXGE-ESlr8<$$ zqxJ^5J-q-jaiE34AtUeob;hDoHJm{NqxkFdgEoQ~bvJu~YU!dzh4eJgs`@sU#OHRK z#B2aE9PfbL;dn3(rpQyrN#<)cTHpNrt3Mct z#>LePLQs&Qa)0v*(}ZV3C7z;iUf`x-1KuSs9#su;vb-aY#%?)>{a9GUAXd=sOb+FZ zLnG-(CtQ8V06>yvu@z_PGi9J$#$SemlY-O_gtu5YzJFk#iJ{@|-@o$;3o`}4Plsnq z04DH#!2n^u887*d4p z2M52}*qnEDbtNSwf#3`gj|Cb)Cgyd1d2|54o%kI~B?tOtgYVyKYHJ%U7rz@DQzni) z-JNGO^bCJdGs^Y`m#P!YiGWdak0)*qXY!1dWG4>&sy&lBkL78e)6tf8u| zZggbi>guZ1cKbabkIC+GX$S-b1$|4zU1)P*1uh<*K}L*#r$iO0#&83z=eAlFix--R z2!U2xrGlDmp(3ErJJqqIi^>lb{wJR#1_0?NCzyIS<%M)T3j+AG8=;9zI1EZZe}4!; z5Xgbjjskr*xQ7`T9nOEhhvWnx6O|~_BHg}$TxbTL+uq(DH#hhH;qEKEqI{!nhi+*} zX^@Z*kuLes(k(4g(%mgccPX7x(kYD!NJ)1n0wN%wbaRj2z284^XR&0>tO4G6-sd@I zpS|}v(g?XQad2?Z(?4NI41N82rd&Jv`I`@}n|A}+pzE$ovt)u#42ToB`QmY!*jI8(VFAv)nI%l0inEg5Z>Beu#!$9)uN4i&HwF-gFLA@V;FL!F!5pR?GxB} zCV&7DQ>E%_aQ4p6&+Vwvf`Wp0Ee7A4blpIZlW{l$vS1`rU~zG=+Ms#w=K7M5kdQ;? zJ&#M?D-8Uv8IR$jUgJ@%K!qkImMxah)zwv5X>V+7+!b)0lkMiGQ;Uo5z zlD9MZC(#^{z7vjpH~7AmE$D3aF%*kEb#Q$AAswA0L!xNljUjvL>(I~|G>gES^M5e= z!=I<=K(8smPKUIrz?Ag)vvir3&+oa~?dgg~xb<#Z6Hu;@i4zz!+V@703JD4Qrbk9b zo~sUqC=nGOZ}rOm>akAD+??LY-BR7^6+r2l8XCY0O-xRvKX@jkJ_X0bax}-@{19L9 z@fP%t!#kR5Yq7Ag2S-MfRaCx?j2P9JB4w^-@x9b=K3@s>kHZBs7dhkC3a(vRJVD+Q zRwhMK6hxdwbZO(>9*b6~+qr^(E^)_5*p-Rmr#SnHx5?X?hyJxE#C91iW0#}>4fk)}8S%QR!IjKJ% z0nQ?sE;PJcBH}NWnRJtvmtXz!4@krp+@}AYRn^VV1+#v8yXf^FXlS)JaD+-bgh8G%b_u16@(l3W`MpUX~^a^Y(I>XmkJ{^RvoQ4#?$QcqiU zTCXx1-?Chv{&1a5n*#AJ{X!?aVj4Wo`9@W4ZC#0D9kCM)u|FCsm1QkkY+XO;JJiHH z1E*DQL{j7aL4Ok3Bc|K2%~3dkbAJkDV<)N)J_`q237#f=Yj{cA+zu!JL3cMAkA3YS zZgOt!=*&kK7nccZzTY1qwbY8xhmutF>Qh%54jv)zvTE)s8I0_!Y#&c+iHKUP) zJLdj4*d*V?xN!%-&;H4P4ZfTUwY8@WmZt-?XCAOt zWSi3;5-S!~rSh=cD%=iSvOap0d=I~SaZyA-VAj+gC>LFdHYjnIT=uG*ot)rCvbHaF z-QHy4buVjv`0(NPeZb=QP1G7DZf<-AR}O!!HYbFb-#hg4#ISI)-I{7|g>4_$(nJ~T zx1+Hf)vc`{htFv7dZAJ48$=q2RJCRAasC;5>fQYA$BgIgUDglWZrPg~8?{2<^I~VZ zf1m38$l#!vh|kf%fmv=vVq&7*;X;%B;$4-IpYSext6<6Q>6Izf7gS6Z6dhVsPYRa5 zJnj?Qtv2a0URS8)#00R}{}mv9ES|}kP!}Z3M%&qK-Nl^eQIBeE@?V1YEnGYJRW;`Z>Q2*XXotQpnrxwo`bFbPK51hsOHCw z?H8?c9#yur7Qx8}Wz83n&D)9T2LZ}5(#q$`s^_HmTNI~RDDE+|*Y&K~KIIV67Mtuj z3~|1Wk0)|gzZjZ93VQwq2pAN}wZ7QE>*LvvZTfZQk>nO}57j*)FHo2K$>dR$QCpPV+Ro;tfbGotk3)c1JKEbN_o*eM6cw=zUW?l= z@bPDx=c(q3Lsmx%LczGx?)~Cy>jLV1o?n9D1iaQ`bch7SVk|5y=17z-DOKnCJW)0@ z+`_b{SUgfspCS@0ick=}YvT-rh2yNu-iU6p%*cDxVQDi6{V|(ROG}pzvh3>EBcJo8 zSuI?GH60(=T?zgst^xQLeV3EHeP(((6)zznq5E$Ao6O95nRz(dAu~9f)U2Q`i#>F#jm%H1o3%<>t z#AL56Fpy<)pAO`%UnrhkJ=a8!i;IIr;#e92r>oJt@)0u&3*SX~~b%dtfP{ zexI4ynG;QrQ8tG49Mem>;<9x3m*SnsJr6pj)nYy-QP*wSS`JTf-{dMU&qv6Jmv09p zJegDsX~#s`sYLs&h{-Etx>>+(K6!U(&?CgrXk{-MD^8~B869=omHhb)(d%MCA+ z^(*-W1n@C2S+g=8KYFy-71$+7r^cRY=g#CWE-ua}7ssdfYiDQ3(vTJsJH_Zz+EZD1 zW#z|u$8e#lIimAtYO%B#HWLEQKcrvv#Ui>p_w!IohFM*MoyIr!C=o$l?{VVqm92M; z%=h>INkc+xMtvOF$&RBf=k*EYR3=mLC!~|E;CXL)R_x#5$H*ZhH2>{eAYkZ6mcx3j zZZFl;)O2)oAhd#}_zUSfRI{VG5*R^|#P{#cKcS89rxI{51f~U$+iKWStd==@@UHy> zD3F*+#u{E~uDxObIn5quhm+s)T~`M~<=Pce|NWP$SllBN5DFj8n(y|-e*32i)>iZY z%hR-_G-FH!%+Rz1DJ-NdMKU!cBpEFR4KxxNnLnwcuZ`4Ejgbh|qp(9XNQ&k0h-`@T zS+PRZCUz>@L=LYX3Jh)hWGPBS>|f4U!M1c;>GpnFh%5A%gf1k+FGw)Gh&Cp zu14=)ql~ru-W|L#B=FK~^vYOI&ubAoLg*4(;3D%*j2gcWfnXz>M6Rhq@g!1^DgQj} z`qnG2959JsN>&Qcbq6ui=@n1^u@MN9E!1K|l5B{?-WO>LAH7y|aXA3W8p-tHXp59t zfeJvIk;u)Ys>C=Wcjp2`f>+!xFltvBzbMsMdhbI^L_}$=E;6Cq_ShJy{E3&JQlWojrH-T_tsHl&CztcZg z`(nc{icfTR;C^n(q@N%mC8c7CfjOy^C2S`y?)-yx_CfQJU$fP;$GUMeVm<>&%+o$< zBMmGR>D7k!uIJU8(rwp&%^aqh>|*W)m3E%?mUeAgSXqs}dAJ7n#zWt|Qj5K;iQXzN z(SLNKUTQ$}Pd5r_HZ~R(TcDbX*Dh~WMo!1Bi<2TV6?6au_zkA8LD+K<-rLxa=|}H> zATqu2F$@PmKu!Jo0F4c@8TzaIq9W-#4@>wQmgX#g6`IIn-*-^VC|tuyVP_p!>9v(& zp41*U(`kK~qMvQbtEfXOouzm1if zl|`LAfQm@9`=Bm9u@00O_`%rr@jVLsZZgaFw^0$&`E{cy#E2Ly7LUn90=gpQN7=b0 z?4MPAucZFA0|7umqF%X-^!?i!+CKNwc&te>ir$J3GjD$SNb`K>ZA%=x9*gW9q*&U1 zpenY;wF;0`?WOO`mK&#jgS?6gS0Fnp>lZd#ALcRRgtRo-7rh+$+t2b0Kfd%s(k$Ek zF56SX#~qMcHZ&|=iGlBugOl`fd13c>#sOiG@hH(Ze%X>q!++F#NkFaftq+F7(|`bR zJl=9YvM~U?fEUF%I4X*YK07yw@=YDf9%|+2swY+htz#!?MIageK2mxDDWp9J#ruFq5Pg;Ll3db;z)%9!=?5hbvi17XLV`Y zlQ_<&p6c-6pw(knQN{qwHBgCNUth<{u0e?giDn~(TKj9HRw853foAG{q*m{}WcK`m z&}7KmfYcwoRl>~2p!MV7;`(aJv%a}W3sh})Fv^DOv=$kqkN!ZkCKM)$ii%lggtoUc zi0+*XMVU)&Y^T{LwOhOdHEAiV)9kS&_iNvJ=U^}_B-;@gz9S{1Ba(c3%u1Ck8HCVS z3HYQS>%C65K1hXP6t^=+WE^n*PbHcRvf3VammCW^02D7>yRL}~#DDb((DJIxBZC$R zM&;5XKd}A$M5{k~jT-8-rytfV`+UsF##eywsnX{M5$Bd=Q@A#c&40!-s<>S*60UlM z{S7%JsSyYkqURoKVv>!}TS~g23Khupryt&T>9Zfv+_3x>{hG3c@cSbH`z6MIG2;^u z(9Oa0DBwrR^^5?m>a7w>hjEw1P#SPoLOFxO!_A;OZ4=Uyl4jfr4tZndOtq!+#y$;aDnR! z*CkV7m}|vERa?yKV*RA{)-U$c53^f|y}i9vz8VP|^8v~g6YJ=UuVN^=w4SR+mBtN0 zEkI5B=y0hZT`4JWu!}>IiT+Y_X;(2Yq03;CeAFJxBU!Ix)!I9ODM5a|vWAX5<>qTe z68Wj{&CLyvIjId+rEvqgSb(G8+26l^6MPv_i>CAb=xhrZlwNVOS*cp>Rr53xU_U zKV6ZYT#keg&i=Y3m!yji{F;v2r-QN;s(TK1A)eEDLl9=trU-h21%7R+fKfF%gwfkzOnvm?TqPPjPxqvNlL36zpHmxjs8q@E9SF#Se6FETq4Po?7 zBNlK}vINyH4VLeQ1yaw|__v~<^dwlv7M3X12HeCBAe?U6Emq{7X}&X|5MX<8(7@r? z;d=c=ojCDDJ+rS@lh`^bakn)4!}1IX9sJ`i?)>xQLGC1%$m;5Sd*(_%9WKMoKccF) zP75*VX0>|k&Mjvb7vg_E6GOfMGG!a^G#)xipBLyD8=fxNto{z!pp*6(oUscP)@ zvX7~6ZjW>D(V-JY*Q!`X2fy>L1;;MIY`+#dZYspmaaQLu1U*cLXmvLMoH*||I9Wj2O1Jy>=^`eD<5VG z&+k_XQ2I9)4UBU97mjy8r}1_a4aHQ-j!47nAcxbzB)HW-^THcf*$RfWIDO3$Ixub^ z$=O_6TSGDh+LkMD4PWBwk3*(o%3)(#^_vk(DiEZG+!Fb&oz>M7+iOaQpppcB4;Mk} z_ooM_=pnMXCVQ&onU_!>@cs8Y8^PxGI;ZQeat>#s3uZ8K&4@L!Qe_V>aiV@q-FU=9 z`Gb_gUp0e?>cj+6d}SAV%WV?p>&XXbAP2Ch=HMbg*oYVNh{n)`j$prbAi47n(tm!h z+%EKAbbXC!mWV(}lz$=7TS`x}5sejM`jBd?-?CmfTbpe>bMk6SiBX9*{&F`=Q1)rai!aW!;gLmL9{k*>69Lds@6&7!Nj!ak)>Q zKJCyYXqU<2i{8;89A?mysloiBV=1;EZTvY_%nR-N=k)>^+v+Q;)m{NIW5ct_)WGCu zk~gT7USC;4f_hjOGEsF1pP`x)$`qQjP`XE#F3iL_pMs`?NlACQZa~{P7h2&A}NF0 zq9!@S;&L_Hl;ZvOLUuQ1tBt!@f>7xQ8KmY}B}12_ERuueH_Z5+YlW#7%7Xa^xf=)z z6XV>5c{|N_O5P{)R*MZU|GOrEC#!;0H8niq;+^e9jd~AW^u^TbJrEt$t@D1uiG`0} zf@3~YWwhf~4&vVxiyx2%zaiy<-hO^zJDsqE~1Wz@RcrWizkOhn$`rx$@;+QWwA-l@Ixo9y?GK#H8AIVWjj$HUi1y%w~-kWU=1Ehg;8c z;=C)hslQE3R?8))#q-B+QIg!<4tN?M5lfZvaOHk$W5&f|jRFCVVq^BV@`2{=QqOXZ zfeOfAM(TOJOv}hMtv!_L6cJ9cuGo&+y2UR z+kR)g-tV?8?sAq9cz#_ka2NyTzaRwml>fG#;`82JsFIbjb&kTPnflh7oe z|4rD3AujusT+el-f$>Mo1}B>x*Sp1Joh=F8Be?v850H7IO$oan1U>7PU0uL)FMMm_ ztiL&OO@4~x(~;9BYt-)bke~kqPQl36uYKeupjHCF5B!xM{$7Ve`_wkFx%o5%4F^F7 zDlptvfO{#a2}MOk5N;g7d5E|;-RH)@n4u0$u{WVFCaP5k8qkiGs5P@o6--0~<))q5 z5L3)PdF1k@Kjp9Zx z>3`V@BXoyA9&b(>nXrx2skp~yP4JI=vU-zU#1S4=xV#0KkvZk6FV)I^;(7dsFSV*dF^S)*(kFT!a9K#Z2Y z*6NIUpx;mmN5``M{H;}vH!Q%%kN=dFUlU>Z|Ao6yZ#I-LAYv#VZ+D!oJbn6f5$sN4 zB9_DC3j&5jBqR;qhd)<8g?|H6#__Dl?P%Z)@!h+!mCx~QTD{sAi z5a#Hn+=V6EpT0d!6`sy)on_F}c~x z7`f{=$%?u;+>gP}Cym{qRn6~G?S5*sr zoVZl{zlXTHvfyq5(+MR45BL>6h@o_Eh&LMjuNMHgjvBa~&;?X+g735UVG}GoE3jjr zm;rD4Y=8fKaBkkeFSp|L%4gN~mJXqYdR%vY(#?z^h*olUL@z1f^Z3%74D-ZQ`uW{$ z+3tByoBOB;Yx8^B7?^}X@~F~MOmxUi=H@nLn>FfrX))A72SCPVKep8Y-xL^t{QUev zUcCa>?0V!LX!7j`gB&jBaQ-E@x%VL^h&g?igu)mF85tS?+;$fpd*{4XnOGVZ4uk|P z;_;j3tr8Sp#vq&mZy6)|LRJsQ|F2i_Wtuf(W{RFB<7YZh|FJ2D?1I=Dd@>wCNL#448Xe33k5CF4sv;vJ2L=WTSjBJ7*8WkIAKKaB z@$g}yF$1QQjmA3$i~K|bp@rh{q4;)LTw=}9Qv{7mlF)ck{gG6$Y({$9q_aDH69q#MRu3_WRQW;#TK~e7a`uw|}a(5g-c>j`# zs#aL*e|Z8vp|@hbO+zP!k5+v5JD@kI$$A_aad2<|dGtwaADDJFzzJyBQbU>9yL?A^ z6fG%HvW#!&@~Js-gv^64&DY*DT3MO%sJs#K5s1&S5wQgMc3-6#PS&QZKT9&1btd%q zHqCw(!eHjL>8_~mynsz1Cntv@37B6)gXl5k_0`qg2YNMbTN?fImSIx0&}#u6tFaV} zXV0F2eP_zrZsz?w)TvOn>0#7QrVL>er_$v)6I0$1$&my0qnk5CP&_~_rc;$f?Ea(>$ zf%?1OLKmk;XTa}o(z{PeD;z)KW||n_zYE}UEvNNe8%*W6JlP%k`t{`a7@Q>>Bm6o- zaLWF@HaK{2a^e7InAzah3Nz67R{(oJlA@=l@9;hC+!b#}I05Q~g`~oL4efb~SIlwd zR#y99kY;2=p(mlFTmWaY(72Nr<2p6#{P^aKpDA*L4 zh@LzmXvAt#@IQ|D`*#>NIy&wRb<7pmsNw-hLLUmG^O$upbOC6b>*(ppGjBta<+jV@ zI5?_cQUzeZR7J3TM1x|Y#;tH~R=I{W@T|diTb8FZ@(CU#^EhjlsO7zJW7^-vOrkC% zujLu&Rx7+`NF)1%oY_|x=n4}&Q%MMK-#tA&=(O8M3c8IANJ&n1UHyobIWhy1pZER0 zkZ_opnU!l|3*A|mD}^3?#=Hf<$OClU+ywq z*XNOlqUZixrs{s?!VS%bD1TD$U^~sXR%UqvER)&|rIsKngAoWR zXnoy$TgQ<#VnpL;D1FIspQ0VnpjenW|T9>=~U^x_dI<&x5U%ec6)zs5~MiUE^YN z4IBUav7#DX+umo*<*l^&blzSpYguYr%QQW;qClrwb{A@?W6I(ma}LkEa0~A^*&Y{2 z@`X&(Yo$7TeU7iu3pG0yNoHcxH5r~6-%8;aDpPap^D68Ry0Jg;&TD%+kaJ+Y5!gdz5bXA?x{=tp7KQTg`3cLUw9#zc=Wq zzZV)SuAWI@x@V)Jqw8?wPG%#@mPB&{{yU%fHt*;5*FAFGKrNAiG~;(J12H>q;g32A z?2*!LyH1;qfE>e4n?G4+>R&dy9u@xWye}aU_$7fJ6w!;bwRoeu|49B{cOHVcQ}L~Q z8c%-rrPmR+#hF2Z#nigdeSW)mIZ{J^=@7IB>m+Z#7A+oh`;W;kzrvYk?LISkbsFhG z>mGXOJCT*cM*56r_a48Lo!Z@5`*=+>DthD|{s9lpQLk;$iKbJ(gBoEQKQXFUl_^bQ z>b`3{4R0@8MYbD8eL1D_sKpdz)aX2|rhpMnKUuMvy_KH;87%855P(=UF54=f)w6(8 zXst!~uRUX&ZZk!R&EJ;e;rOOQ+nR@|lVw7&QJQQ|dgu`higPSP(c0{fqnTN>O+D?K zOQu&<89SRI1t?CLgM)+l(&r)w83V45b^A%s?nYbm#ZW+xJbDnMgBdGwZD#8Q4_AXM z)1_u1t4m6k3);qs@`(s+(jTPV&Or|`k%=CBHj{m)aR*J7Ts9Wd??~ut$Je_CynB|A zNO&JT!lK{rIL(2A4B*JU+587kP^`3jBY4ceBtYbTU17N^DSz+0y}WmAroN(bQ#X0y zQM>UgJ{y%{X;FNHbW+WT>;wL?DW+8Yl-SnoRd%{{}TEN8D=jBP3X}v2h zhF195pRYp`z)#JuE2Y`=KzLSfU4WGms+n;o<#&HlZt0%2o7eXLcX_L8Zx}4DL8w zY^-8`!vB6odMH9n^|WT>Bipq6ZlCXkwSLp|DS3g-%)9&@uTz(+erX$xon52G{#U*_uG9A%Mk z&2emu2pavIn|ZmDJx8sj>*pu}qR0%*OIFdsC5CSrXYRJYuveaHw1JBQS{2X{acg(E zjSVk%JvSPkt330gXk4YSQ9s27_wF0m(GCt(lg>Gt(1^eTP8Q+FdCMj3e}N6OERZl% z6cpG3w|g*XVfAQjw+PIa{#)wb&2pw)n{(I~PUl?9L8n(CJN@>$s@MPH@Y_fKOg5$J zg@lZrn0fIqmsir>A<7_o$65Qr=}TK@)c+~XOEpug%VMIUxB=h@TP7VR(w+zzyyTJtHq zfC_DGYyI#G0{s5d4ecpK@yZGOUE5FN`@8ma73QlR)2XP}CeH^%LdMi5*#qKoB6S`Oy2bYEV$N#!Uy! z!v;qFD#m5}EzI@g`3rVdOs9Wym_m@GLrBE4PEoLLbNtjy)V3*G>tcjasA79>Y4~U* zSPVayajwi8)FY`fH+~2m26&c|~jk{kQSl88- znSp{xz$C;fc{zCQwm~KK`O*{p#=-N7Ug`^2i;?K~`J3`NKDC_EC%LVeEJk`pcu|xI z75xa3Lm=X$-XKh+q^0c0mGWY|o{e8&6`j1##jcwP*yPz?HagnE(0@TcIv=>NLG`tf zC>j+key`haTFccL`wu2I+;&R#6YERX8D)_+4IOdjb)n@!<= z&ax1>?0C}@GKpO3T?7V|tRn+lRB$Vr^|W_ZGPtnp(_;Lh(M$MvLAZA3$?*J-5454t z-{=K!9YlYWRrJ=PB9Ise2_8Z-$Tt}IIeV7U4BIXPTX$Vhyd?4t{F$eULXb8 zJxL$55^Y30f$q!soP|TKJ+juN<~7v zV7JN$5rlKV$sLQKzP=>JXVf~LN$zQn0W&8(Ny8B)j}sq_ijiGk?w9A~z4mHXLW9YpGsmJ6>#=lGo^7h3 zp81JsysY$X%l_582Mvq)S?`lg5i~)Y2^!2l?Fj{TU8%0fbWM8HP9FmIg7iO+47iz6#TiNN<`WvSRSO^O8hB+YGfk-?`AgPi*7A=z9_t0sBLC^{EKSLzY2` z(Th+t_FA@~hWuoM++6S5HfQ4t8cdVtOV{#3{SN{q;+Bvoywn(q5NgO8$gBj{Yd_`C zju$?-rlWxOQfLV5#4S}l(3JSRrQAGY!0opM~a_O5Z$Sx z64mjZraXH(4}9OrKcf<&Wu9<}z$lW=LlouRYCObIV{u6&3X+pY%0qlcVI|;YmDD!B zDU@g;;XkV2AwLec7$wPSOgz+Kzzg@1v{=B@R(CjOMM}X;!DK`7k`xUhq8Gz2XE>Ze z`aCGCQu#*Q-v?Oh%*VEA2?=WW@zARy1!kKUFJ6S>QR(2vyGA`ELW$*`Bq1c{ z3rN{r_X%TqInpSEhe(ldm918XIY(C$v;2Q#scU5D1g`4>eU)p2cv-E2{ zcJ&cFnIr#nnuW%l?0_W@BeAiu|3_1hEmSY5w=zy)+5!lK7;ABU;kYkCj?8W(VJdP! zyIv{jfXPPiWBhNjWuRxJ4qJ{A$F-OG*NgrCKGYR&-Jbd&dV&xN3VrGuTN!E>N5`QeK&0kQ0Dv?KYfMqBL#(pP$U>H<5XT6qgD~T^e&QoJO5*;x=vPk zM5&D(5noCXnSwbG$f5SOd35j4prefECbnWo;I1iixEmc)pwfFZ&mU+Mo|s$l%7~yN zACp6B3%SxPRH-R_&+yD(1k0zvz6SWc|Vw-|G+2JNcb7F&eU^v zn=Yk(4|ycDp#SxI=_O2O%R(NgkMlr!DiKu|c+N{Pl$tuufiQWvoAYC8h&<*JKPymQ z$zX`WtB7ky5ZioTTro7oslI;cx1id)(mAW&yJ7C4-#LCRYd-5Qdk7$05W$){NdMpK zsZVRrm1@O94#xl6o4Rq#e^MXx1i}N%Umsjm@283G1g9qi%Q+anWtElX5O-<%W3JDJ zAXk<~O<1HUS$Byl`jETiby!Fx6gI(#vUKS-%Sc1d>_mCzLuA^2t`SU2Cm)WnG9AsQ zu(Mp11~1bOeuXWxSkFVMp)o5+Cn+$Sm(IyVM^1n&`Dd4`C%S8B zv897aY?9wIDOU>TUG|#WEB+wY0?9rjY;k8)gkWXPjV?P^9kl}3jM$JUzv_i}E95=< zT6sSXX_UCIwm@W(xwa`QOGMKdeXXj`=R4^-OGsv&T{-Ifr(Kffs;MOUP*X{2hBtJf zza5@OBA!Shj7aI`x>C5wrE#ajKX z7!oB^IpM%}5afkvUqHm{8|*Re;Gj>_UG;5_C*XKwCzxg-rQrw87R-EpZLRI5%M5(a zz3{`)#jo$x4~m}1_ZA!0bk_5fc$aSXhtXeOl6w{V%OFiV zIxGZX5_nElV1SwHpjzc5^Kbj;7EV)`?O|fv{jmE^&a(e?k{9ah4I_d$SrByO2AWBR zl(C4sb#+@+npFE(nFnc#F|w?-k6RiVR4Rr{mrxQWl#+4lX^aE!qTli(y;}aLYD*LInft ze(Iplx51A(Oj*}ylLsey_;)0OC4==FHB_Tc@7Nc_#JOdPaeV(Nij7?VHUfF3|;l0vOmF$jES+Et;$|wtDpA+vwutsZlnG(=!6m2 z9`Yr6l`|kTodUbBSX>7-ODJ$XJw3a2C&2&)Ew_*tUiN-0}QRmk`UNlfN1b4rMp04F(pMNsOq} zer@bsHEM(01Y%3)^W0}Le4327c^kdMfe}I&FR#jtlYQfC8V542mEbRm_@2C1DcOPk zwb#}+f|=~YxlS#m=l!j%EvUgqhK44$+*Vgtq3|v}j46b^S17^< zhlXsSogaoZXmge6kd-PgwoGEC-yp(*mb6a13AdZRBmVySyPCr&+e$zpiqFil^*PM! z!UUh2<}W5@{y;4udeP53*8kc)B*$9}t~-@`38dnnvwZyvp@!$x#ZEPt%$d-&^no5%cVQ#AVrqP_F#Q`$5a$VEOLq_w2&bz_vG*Tw2IDw}bG$*o7Bb6#kxl-+#N8d5cf9t4`QaJHg~%HFrGuH=btK+wP@ z5#etKhG@%8n47yAf=zln0ct+HAe_AiZh=I6oeHB5pZU(;PKakKUb+!kDK~$P&kLL# z*p#>Xoj5o5QOvje)_sQ7^LcdnTU6tAuUgHVBe2cDXB0{EOtAyziH6TQ;YIzFO(Qr4 ztCD$&A_~sNK+o9n@YVas|#Dby!# z1RRaeareQop#~!{CL6WMVpRMf41rz8)khk(_Z@0MwN9sLxu~e_kTN&zH-D?04m-bm zldxY9nfSAy8St~n`mdqE!%BPa$n68h97kJ)$bhw%w=0RS#lGNAguUav6%~ zr3z0ydb}t1*dh5b)fm`ysRis|&Om^r<-4}FHkA_JwO9f6a`2Evy9s9-S9{v+YLom< zfhPO`qefA1ZHQGq*U?Gy`va*IIs!a-abd6$zvIepFMIbeL~$#M+_ZZo>_;?obm25u z5Qxt2OC482thtP)m(HwOyiJNWOjC~K8#vkY1t}QcHTl<-oXV|zeC>8bPPDVNq&cyu za&M1CyJz@+y#SZvEj4E=0dV^_o5FFd%m9P3qj6f+6`p(%V6ki6{#}! z2Lld_TM!c!MS#6FA%Q@S?G~uTFU>@QQfpZ>^w5$=j<8;CZS*q-H)&lhOclLHL&J5+ zdE|4QW6(Gasll5?txOBj>y)~g_{Fb6%lc>3-jPNq(J> zu6cj{_&=Ny-fJ^lvK!oHn%E}lDBk^ezTFsB!#xfPBl6M4O|Gc}Hq!Z67}_;$vul7q z$@Tr05fvObQTczy7MiQ=ifTjgAG0Qb#b{07=Zn{BhU47ygQ~4qpvK_kEtk6mpXF&x zp-17?kKP6UM$fdiN||XT(87w6uu4L)NzWiuu{B+CPa zyX+e<3ysg>{4H?g5Q=I0hbp+pE#B+wxQ>e5CmY*2j!V_8nEq#Qd2wzNOF|kvB2;Yq zNlAop_>$2o=4koW-_^jS!HHj1^qY=ib=~rt8672i+ozE$jSqfrBnG5~${*&)uf5ax z$w>RH3Hewx_ELRh!F&Y$2wV(8$_~T161fBie?H-(1i{EF85;{FC0qo|mr)kiaPLvs zf|g?hE;#&tTUfh;!+glWAB0demgMx(wxh=vxg2;8CO>k&`r|{mRH%1x z!lB_>PrGlO+2D~GYxImnw@}L9QxIop&LuN$VU|^DqJp)v^#RM>=f67ByU5y(R9bh2 zvxJgw(pi2wD}YJY9V1(j=VoT;2!f7d(Z)B{okZ4uOUf=QI;-9BArpFVyA z^RWgRnJu;ncSA=-4~HQ;{vz{*7A2_Yg~bb6TD;(vLN=!cKVapbK3<429GA5ZAyyCW z&0Ws!&H27eas0;mDIc%Wj%@KKb#w;rRI_SI{@D~K??vVZdI-uD5_cMntqpuwejZz0 z`N~mxJg{{Tqp??r;f17gJNeGqJukt(psr*#r41%jIlmOxTj|KW(+9mb5UUn2?|b~Q zdHb&?m8bT|o)WV9tY!Ro2tz-`HcItkq8msx^4G6GBVMgp+5L^+6Vhsh01sHVl za&ad%$h&*-_hcz)?(Mx z)ZF^J9Y;b+Pio8qIXK*9E=~@I?V(J6VNZA@5 zboSi@F}gS=n6pz4FtR|doMTSmc(<=4#|InXm2(TavpLnjaV>l*+GSdU?~EXAm;LNk zcQ@LBxiUw#kx!T58V3qXdYGFiM39U&lB!I`5i@&c{g9xk{&&7_`lJ6%Pom}E;c zPK&IU+;Opqlb1Is=VGsI-+GmDF&0iObwA*2PQ4m1)}Djj_|5MSkMsM4!+Qs^Qk_Mo zIw~k|z*>Fw314DtQRNNKlNpz?!V>*gDq-^cYpRt>A`B1eMj4p(1&JArtvQNy2PW?u z+1xK~CjOU>KSMfxFIUKU3*b>Mqe+gW(q41Ea9`BZqw^sf+e@%TZ*=_g1Qp(h${|Tp zQ6sYU=TC7nL1oHqWsq3 z(u%^mZf`CRJmccyOBj>ii+%Zoi+FF){I$#vklZh}N}Z?57H^Z1h%PvVJl=$cq6({1 z(9)XR{2$)l!=LJ~{~tfLjFLUe-eix=Y?75ND|_z|G78Da-XSX?WEVmzTQ=EyW=l4| z*SWu+`~DMt=h36c;~eL_-`91$)^lti$pk(gc+<%eQEH3X1PNJLj|58~^%A|4bKi@7 z`}@k>1wzg=l^{pw3yvl;)2)90SzoX5Ghv^KE(v0E^p7mR6*g1W>D&Ui{GyRbGw`Aw_~|Se&ssdx#iN)l*HF> zU#1MT`xss2rOpniYs@7PPe)=6DGp^6K5_u+$Y@T7^t-c*hy{r*jyc90PolH>#2Wzp?O z0(Vfo+}z8Se12a-YbHLAK9oZ&x}}zH+u1@nq9B)=>>6j6>N&rHg&f)Lc>77Jp4h*S z%}f~d8}Fn+m>6J z%xZ735zWobH8)13KP7HX#b~$O1re~vT|krR=;#m&933S=9i>6W$1WGQ%@MYu1bEY@ z00`8P92mU8dw6##Jmn3|E6ie*jqN~FVxhdowZvO~G*8A$0y>nZrmPnknqOK-r1auf z8aq0S&)9XG3U^mY4gK|zTTtKT2=deUu(qF9P!QUn253}%e*Uaqq{PGnH=+TUY2QEo z+Zi2Hm=LKbDl6OVl%65>wfHQJ@d=Mp(vbRE-VL%|STw2gL#2gM^*q~-7NP`p62w*# zq)rdNFewS{%8eWx%UFvqkVy^rUzp|8w+o-0H?DHOzk9C^5?5>%3&)PUGHu)42v)=A zMJdiYw~-m^sWxbS0fbCfGG5cnY@WYg91Q`1ai!u6QjHY~n{MDaepq@^sMV!qVhvU6 z-&=|yCS1#LgAijxvOr#R7ZAsrX}abhc7WJ}=fu*8)m@ZH!n&$8jNvt@?H2u`2|NMI z*sNaQyQo~J@mWPaw#N8sqW=8w&2Ft_7oki9AU&Wy?U9m@l+5O?kNe8T$DV8t$`oo? zfZ4*lNsWz!uX6^~+e`DTe)h7oyiSms75X(}PcfqJHuL>uto;&m5&Pj^ExFJ}QDYmP zJzieQS1RsK`2u}{>0f@^B%c=FXupXN5F*oMhvBF-jNnI9V(W;{Lx4JpxH6QuHlwxU}S*NAh#dwxH&j_D);WSTqGbh*WOfq2DKqSsNK5fp25p_suf?H z>l*V6NiOby9M|!jSd0c@0HQwA!xYG>E&Z`y_+OTw$8=r z&XgJ&!YQ+nY`NjLM)~8n|IQ2Prs{=`_ZLy#8RN$O886K2c}0MD^+~kiplB{%8$!{E z4T$cH0%%*~iZFZ4j&TsS!83qGcRXfJ+<-fd8Vor1$xwUR-wue&!7M)l1Zdybf~MA+ zhXPh5tmq|TuG+@qB?ZeI6-@^apzen!HHKm<%nxw+`zEiHQ}g}4T)UfDekED+_-)-q zSneb3>&yGT!xeq)v4;!KzO>Ma?lRHR9-DQX0)8KI9~3jt|6Vx^1bPv4F8`kIwn)KI z?o|DLCIu)vH5c@*sUz=jN1C0qWsrM>U@)2(d;edr<-h0X-5GP=WZR3{O-wEyFMK&1 zn|{K(ov_{dU`a8@riJISJBF%$HK#>z_?|059QAWloomzMb5~1i>+&aG+@To>CBr1l zEckDl{~Wr zrW>sMk`K$Ep#oe16f+3YVdeABvfD+rr@stuOe)@aR&as8GW*A!O#96K&F&E((FctT z%5+mn)<;Xp5L$ zG}ebN47|F#Zw)(RA`qS`ks0%q``FWxsY{oAu%u8CTt;;TQ8wNA+U!ND>8J?6^$MVy zR*Vnt6$mbpb0DYMd1V3L;xJ1VsfA5j190D(|0okT+G_ZTeoz?ZE?fgs)e4jtq)_g1cB< z%czGU_Toi$;v4E`aIdCPDjEnr>>x9#Qec3LEFcxxV7raT$rn$b6o**JQD4nJ`TMz% zn*Ez$jqSog2SzY}ZKG+##@xPG^l6lix#Hi`<+^pL2_EV`nS$K69lJK{gndk6T zAbbmrPe4N75pgvGjXrAsK`72P?t-{8@HJ$Zgq!@1N`}Fi{{2d~-husrKj^1}L^Mh( zK(4be8CkIDAdT{vW@Hf%7_m~GytF?YMQLpgNG2Emm?`&f*$jcLSFjmbo|4VS_BEcb zGwU%&T!Lai*qa+KZro;iN)*PK&-GB_vxnxD@a4Ccl!rMc`zEB)0VB0XAv29nKD(kI z*ykE`CHN$JY>>|hx#GcgGX=m~{1M535Mji#-WFkU$F9tww}Zt6d8^$-t}$x3lB zbYGx|#4yjFt~mtNUklnZhZx&`KPa$zW}qoQAv49JJsC}cBM(mp5naxXYAD@q0(1;8 znh#Sr0dy}^W&{vppatZibL%s!+X|tl<*Fna#D?mr)&9UNE`un9Jvf3Bja0lTpl?I&_W85OeF3{I&nV_mx*phSFTJXN1vEf{2)L*B<#A&A6 z?TNnrZy+TBe((ZzfaKB0W4`pQ)xJ#c*b*m1g=C9RI3P0NGQ);?`4JlmbYmDwK!1>i zfPnVzEjD)clC>LQ{NkbQbB31{&PKtj{+_`0#E)vkOPRTIL96Oz-K? zxzocv=TUp$YJXIyDr+CI;Y@K#J{0MxESlUg-0j1nL#;GuB{w9oIBnXixOwAh@_g}| z>%a`g_Ae(?hUI?TXGd+)jdgp&v`rrJAPC!uYM5KfK4=VBg;S;Jn0SsxrL({J=eJ6( zMIFBXm?tDs5SUpSH?=FQ+Ch^;7f{5VPLuWJEsZ6|r3nQRi3FmvuP-Jlih<2?dVFrq zl)Q5vO2L=~RfsT$&ZMw}7r+CYqcDIFRl*5np^EVNA zShVCJt2?*kdn}VI^5frWYp^pJdk|ikl0M#fQ-^(ZIHZyOk!Wgs*_XwQFF!3i8@d(B z3GPen_>0Vaw|<9IS;nxvr+lk+s#i3lNSA7qJ30;g=M;vn-`*m z=YOT&4BB-ni_cGLo`JKoL63o-gBs_I>qJ(H#|Y05^4D~$uV|! zO4A0hcXWgddq5df%NBnIvmfC5FtM=jSA9BP(G03( zjZwUYVK}(96nF@X_KXM9JDz9BHCsBejS>=eXAdhMj=pTcY!9YD`^4O9fZ(xV%}>fk zAd-j&-Q{{7lqf{7vwc3F5W8306PQEvXK*kfvF8*m7~^Fr`pjT$Zt-bK$JA8gr%wZ8 zJ!%ogqu!wI0mF4@3bcSf0;s^**|FU?R-~4~?xWL-#CVA3^0nP++WQQTTsAu!E%@f} z`z{*iHtwgXbz`W9{8YgIxH*>dn_Ui4Jy=ar+i~g1 z+Uk*d;qA$E?`_B28JF4a-*t8~Qr6;!Lm#pe@5YKhcg!hbQ7m2l$|QKKZ{F@l+NgBL zghl7`8qv=V*LvLR(t!}@6e|N5`XZwp*)6c-`||m76M)(RvaRDeinZ(3Ia~*Um@I>4 zv)JnKd*J4ag=g_+!;xH{jD0ITY1^qg;#0=DZ^TWv!L6+@ZiEsmrKE(tP>aozv+4P{ z&<$@spPeTh`L_`lmzPum=*}?IzjcG?=^u25ROT(VxDgC#RO_AWQLHrIxyoRJMM*s} z^6_X74|0!SUNgK_^M}vawn=6?7cYxp0cPc~(&qp-bilqA7!VMC5j+;{c)7D1Prj>- z9f;)4(uM-!N$O<&B|y7vtgRE$UO+i;24v*nxXjYrc$!xh#t)ITFGOka9Gpat5eh>L$IZZ%=1ui^H$Qh=*YOcSD93(t05odB_-XzelgF9zF{y`dHB%7 z-TmYTtW-<(yQ4bxJhQEDYc%RKhK25~`L4GJ&fx+dMAZC#l)eDa* z&AXdhp41N->7MC@vOa6{4wvEY7!zoU&K|YO_$%tJA>W0j(0w02cS>HlDW71)gulEa zNWYb{Jl*9-Q>p+@@@N&USgw{~6muQbYo&evHf(cpYiLqG@)xBemXB9Tu%t;P0SSyc zzKrB#dHRzZa5_wYuJ)-~(<87LQ2LF@=WUw`TaB>jni-2uu5Hb6wrRpvV6rbUDd|VW zge~I{wW`*@dZBbzNWfSpDnd?~kbZD9cxQ)_3r)$02 zAWf%@qvtvNI~(ZsW736;*+fJ$Ej5f63vqepu#LKjzVz9_zSZ=CjI-x?!}H3kwzxif zn3%q~E9ESV*5L+THkiR6<8NO0xLBLfeHGGbxWG3{fw4(gOdD#Qz|kP9xUyLBjSXvq zk4VqF9VC3b3@?g_UPVcl*0p+XK-IQ!v>k532zT(+kFjc;QY0C=N-_YH=;eNyf(Dqm zIF-rEO8*u>Y(fnd!+HImmJ4$9;bcyZRAk@!&3o{b%^yeZA;VQTKBx8m^QqXMKl~UU znnp%(&T-!JvznaITymC@-1)~(wA1_S-rD?i2?AjeP0@;nNgBMuLd9!i8KU)gr3AlETB!#Ndiqq@AsR)Ai z-gny2PO=m$F48C+dH|Sr1lf>>ukd_6+b6+Ne!^D&?o9H%5E=!ulCttb&LM_w#4_wP}1ayUV7NmR1ARwcqY9fLe2~~DV`URx(UpVm815!MaLHUaQoi#o zYo%32SP5_?w}h!F7am>{F!_(<<8xs+KZ6oYnmGazCu^VN9?+-&Vh&2HE>EXQ=GN90 zs5r#8d#3c^neca!kjy^Z->C9y7}_VNc;eVv0-jlcr%W^JznyP!l}({ey&#~U9fU{^ z@vzt_92lbX=|K8sRG^HNqkne3U5{c0rIDO5QXe9e;-qz??ifg|T|xsfoKh3dDn5SP ziV>duwD|x5YPv2UGy!r>wEU-!rXF{;S<|cGGrx(JMWcANYX9*Y?pmqMuP#wveIpI+ zYO9#p;K#$Na{;Co6+vrMUdKa4yZ^-kP*;-kJ|qU+4l~T&iD#SU9Jfb5*!`9g>zm94 zy8uF7lX_>G%7-~1QdElY{Ux|1_qD6DQzYoBN#_b`D19d-ZtN$&ROemgm* zJc-uzpwiB4@KXW)M;}uYHA7)THp6J*Vv38E6}b8Ba3SOkOIKfRCFyHv9V{oRA5>-o zOBsZYE}*+Kv2*JN0><6Y$psyd`C({5D_66|D?!&qfsl+-U;1eV%yhOv{f4DQ;7JA2 zi47?OfcM>2AOq+~A4PZm0GBH0XjBhn9@>C@tLZi;cTg>wg08Qim;#*JXEL8DyORf5 zkj;bK3;2@&q(rRqUg3P7^N=7~z?^Wow_q|+E%WEVk9nUT7+>Gy@QJn3S^6-j=&yt z6jq9ki2n0w2o56_OJdcWgQd=Jkc!Qiy!@O+E0Y}_GykbZ#y z2Yy!cOK7Qtc)qJc^M);Rp)df(FpD;J03JzTdWb9!OB(>{beB(=4`Kt%h7PNhlarI7 z-Oc@)Owc|lvF8cZq}zz;IK6xWyhSL!!p@!G!>j^c3?&%Ol+iD;v@%JI^U6>9%rGr( zTFBl`lqAY56TbAo6S6iOn#G7hsZJ{!@F0!zDih;Vq51);;*Okd07HO90&|~(ZA(C; z0g?;kD{dju1RY52xOCV_p1A?Ijj&xE(2LZ3mP!f=;Z7Xe=xW*_?LFa!Wz2ulVNUoN z3Mde21ZUl;$)Pu5S@+|76Y1_3N9A4ee}CQ{t??>cOAGWsT~^6l&U1!}H14n_`j#MU z7c3b*k^jy(Rrp{q9l?wBw(BWdiPn+vKrmkof_w=nWCSPTn5z_iE*y#?B57|YMn?rf zd6Ay}1t%ZaJ0K~(DAHxRho@b9V}RFbv|+fZhD4KWtAi856D#O^2jrPl><;E*DJx=w zMVTlgW9Z*PxV&^Bo?+vYAWyT~yA87MuJf&sUr1)}LQNc|2Odu=OSTUg6F`#&aIk@h z@BRJvK*poLx4r-{p0!-!yAlcU@#g1$0Pc*^+SAjs85l-2gN}|KXU>06q{@5_LN8&b z%B&?oyq$dh3G}B>5S=UT=6PyG4CZIg-m9iRJ$B`Y*3;E>*)6}87;$Cq^{p%7ZML}@ z9H3D5eY0NtjCb;0;8@bott;k>g5Gq=GSm4zo<5Ju8MVuiQP0Yv{Q&jv8$6eHCD?=P z^y=u)Jsh8x|Dk@wy8Y;(6JNF8k$qr#ibDSDhzM*vyyZeqXeU5S^j2r})G}-d{5m6! zpInM*uJ&0&!rJU8g1jTA+kEhbrsjBdKtgee`xQf9)3_f8>DnJK9uuUo?6XRGhz zK-womD8%03Qtj>UXW2kVYidJg2})_FH0>lfKOrx3Xk-X%dIs3#x)uQN#k+(;r&!9Y zH5j0giBb^a)L+cQkmpVRuH1K99&!`k1&ro5kl-lF$?eTP5qJcqCTeLd4sj=J@?I@?ropEj{^|Po=}Elc10t+1K_@DW9B$uK4H>Tojq&HMyyiI=#NWR;5HP(V zY>XppvTi3%Wi2ux>+`{XLj?nX-q#Q7dDO-k)-0=ThMk*zb2wyCn znd#)@B=ps*ME%DQdtf+By8**-Mi207*jL0r2n!4Id3Mgt!_LsJd!OTE%MeU8>o)Ru z{HZ}o?;aXLhE+tw7vz&JFcaLrTmVF+;_#HH#0P(98-MWbuDJ+}wfOij+_Me$&!TM8 z0TOKqgyzaIpQfWf70C10#iwN6P8q@OiNW@84?VU2e5&%9?SY2dP49aKboIo>#cN)F z3Utn*Kp-SC76B2Ego0wit{ME97z`9{%IB8%wi6C%yc-XUe)uQS5F!|iDZ!U5LE`7-kAC^!V zFZoMGI{tiSRgty+k$B+}xbT6RE}&-^LQ6_Yz%>avXWcx#NI9N`wlGZJ8)9Kv#o_?P zg(D0FVFAu*FrIl7ssU0Y@Fb_>6r`@yzuh_Yj1wrs0ns`A*$vHN3jjTE>|x;E;n|m! zk%5%syIudi|Xf6I1-j`d!k~=f`g6EB(P|8N7OF;K;^l^|$7nqbvB%zjOsU=rXaWy`?>52P;UB zl>kATpX7}V4ct#YBZb?&g_lr~y8Q{1YYPjSMsF*vlR4~m$$v(A(Rv7ocRhWZyJD-} ze@Dz`h;j{WGlIad9-}-4JPdP0+!g_W?CR#WAcFLg0?8LkMt_(+f>jLjFqt>qsud`3 zz}~YtcL1C}bhIdL-$p?)TOgJ_Qdb9SJLureRaz)62_K;%KuHF;(2%7OWb(7Htik)V z+y}_zZ-|U0L6=_USMRskh`$Fe){uT&3w1EN6Q$*1a{JTzZnOCe@$cwL3=7zUj)f@+&=8zw z<6K{ads82c0Eo*FMAWLRdc?)WdD`vl?2y-iE(mH@z_%;ZG@iX9|3_46Rr#hg-HXv4 z^K@i{td+EtG@47$H27N@isQF5Iua}o%=WnK;)u5l`>#FE3~Dj1u8v!v#h>>1eriez zGZPaDF)<=IHntyjbBlE!<-*NP*k1rP5qoAp8npr!8$@{EtN_KbZGbCfe$vJ-L48C| z1&j3p>J?De?#dB(G!>Jz`gzz5{V~p43=v#2+)vDAs;!hE)tvzXCQJ0`>k zev4XUpufMO%OzI`RLG!q0(Y}V%!&2&K43Z{^>b1i5T?^#{q^z(#X%I|XxmV~dZ)AW z-NS-uCM5+roM4#BaucMqUV$|gB zJLk!ioiqvu;Tu?hWMpJLD+e%JwS=ScXwJG`EknKhbM#cW&m7`wR@^XkyHBZTiG%BU zIQ|F0C#;#Fb!?hY9E~F>*I-8shvG|Zfjkse zbOBl%6jV7BTEr~R!U<~tInl~X3^fD`m}Eu4-V*ba49>C=8tU;P|5=;X!RGC4`8uBbC zM#EELy2LC*gp?P{9;i?%pG2Zb)5+Q6f8lHCyN|g@t^e4~)B1av>NEY1e(RCU6e09A z&OvTKS%Hw1oRU&9x;Zc~5Q2s^M(UYi1taKwF!!2yS_v*%R-ksKwl%4xPN|JS34%)B z0cw|~`u^QJ+NsKsGoSl)X4}fzfA76ZmhK7BMKGbMU@D;!8+$W8*rZ$gdx|qR>2NGT z_7obYR1`|1qt6d87|Xn_OH$-m>-(&PBgGxf+HD&yYDKrPietsJG^F#f1gRcWN4fQt zn5r7ut~BtX zAHo@pTRGTE|KN1)7ICCh&}FWY;CY?NIqIs!@IWh+74bul8i9j?a3zmu4^(GCxv4rK zpHk)C_v!Ak#2f)$uK-Hk+j=+3(GBv%g&og0)(|jT#)8X$ql6mKM$Sp%XFQ~1rRX2t z(8>~9JdojWU`01PDw{&5oQS_KK$tsy^?Fn=UI-(Bm3R=BF)((c|B9yDqrhZ`Ha}QK zP=TRAk4uRt21Vbj^k}t>P_6$1Cbv%LBgLUQEm^q?(vq(^?x%*s2ZdW*`HUDAoxEl_ z$E$wo8PlP%U+9CFI7xBcqWr+R=&m(%LTz(QGG3EKr?&7Gai*U5YEp{kAwm;lKD@VM z;%7-)NXv77sGMGKNl@QqIvC`BPwe*XwK!kJ1~qXP)hE`frzWd^WQ(mLRyl4qRFoy6 zkm3DqGr;NLv|1UuDLZ}VS_{0^$?m?cc|bX}(fKk&7sUZFkFa>B|Bf@*R=VcOJAL0d zSq{JTckDq7uQhv3$`IeXCgbdgL@&E{QI5FulQHy_dxugkiWYO$LLHTYIQn8@ zWyXjlk?@6H(qCrFHTQpYs))so{*6iYrexzLBYT$ifKS+}MgHJU1yzD>y2n2>Bc)>E zB$-G?LdJNrl%JeYeRwuJRipQRMHFUrH2llq-v0Yp-|dbJTnk#Ze`#+bn=yM)exfpA zsGvq*hK_{bsP(5guMNB<9we00B=5}5C3UR)go)EyU zY@unkHdJPGDP=bm4jV%d5e=C#`WPt+0y_Bi^J_eN9`AW(1N?=J0>Vn! zf09bLQY#s=eG%Kf{b60uaMDXWFO&93b3~^pT6b;rS!E&OVfqE`YP5vvAjwCrwcgQI zb4ITR?mm;%QBC7Y2K8%`pS!t-0t6lvE%8!N4Qr0xX~lEJ(-7#;Ou@0_{QGq9@n&ER z(=~C?J=Cb1{0b^1ZG?i`Vwe*v}ZC*Tz^EctpLo8+wpbsP>XtR{eCc9<7z{T z{IA>btMAyvXkOns`BjL>;6QgOMXpZd!zl_dBhbBUgkOXF{y>A)`~H#KlM@c)cmI7X zVh=hK|L2AN?=QDdTj59$zWv`rM5cyH9%Rn`_n@Zxzu)lR*Ld@PZ|{FEqZr|x|G(!W zzhHp1@xSN87e8+2&9MIW>BxWo|9l(Exdjv};{U#bAQi?kV??|ZO8dxeuaEA_ zFXY3{S}jG%vE1@WHMO`*^jaLNjYjyOso_YW8&-mKiTSqj?62HFq1oi4!6nlJ#aWka z)c;(x5fq{1rA_KyZ3F@pQ7q3UOUOQ-zB__kT0rT3@JANg zY|MJS5UqOe+w!n^?xbm@+6Rm#{lN10SZVoqbXKG<4P9ktbaI98b|qTrSF6t=eYII+ zc>$aa(VIl-h%#L*11)voH&!*{t33_<3N=l&QEbT1^c2m(5=YG=tYc#6@ORQn#`R_k zYOJ({IQqDMfzi%VNzUVBtOgFGI&uZYt>h&^FY|(er%5blrEAfgBwg{$NI6B#x=?25 zSPjr6AGSKNzZ?5YLb9z8?-$&c^aCG3AkQK3{6k6i=90w|#VjnjdhaedxsT>O+{%o> z!Dw$1yRFW;J*Aa4*KT36w$lryR3~>4_Rp3XtNv$wprqLM%v`D-@NpK);NVzBXel8i z;xNo9KR>Cnn@V~rsuURf;- z65hER^T?{qIN)}?g2n~!x{vf){Vr7uVxADMMk3X5Tdgy5uiedE-djyhbgw!$LsW;Q;lg>4)307iNY z|8ewwJZcpA^v(&DMenCzx&AoPHbh)aY*liC9iUDA>NsfL4{IGJWpuz$^b$DoK0JQJR z^grn;2Hb=b3IE}lZcpKH+$;2A);L5$p8qvHBL*W{96||&G1xM=070mo_zE+Qz928I zvDY^MC()y()@8-~2Vh0$<9k8wRozZ(Qx11s?UD50t9eoKbu2;qZLdy)-SrP`&-x5m zu_C;eObOmB>8Ds3kXlTiR9EWQjlAqJ<3+5Bs4NVhBu4FN>*#>OH_Q|MFei3x7C~+f zZmb9}@^*Y5?hCCz1jtn(F_TLdT+uMMR3#_*_d5n4BX@?i7W@L5>>SRCUvu9hv4 z`aXSkX9A}8Jyi&Btxp-_(EBs2-T&=@(0=My~nP5p`Ous_?`RY;dBx=bB5YB z(R->21i3k?z4e??W{NA<)q_Q8w2kj7^-f@mQlVth&_J%}tzJ!9CWMT@H&umGm33|@ zT>Yd@IH&#M!;!pR(FKJibgAGmJUm<$h;#EMSTM67PEJp6u*7G?0pRcs0?gE*kvc)W z=3;Jpj2%~t6^+7Urp8>2H|QPNBn%N8ib{MK`Pm>k+V$su>=-Y?NNUx zv>xEq=RhcSj60(9JBCnFT7UDaJ3N$w&tGbsndPC=-9TlN8S)Qj;^ngoWt3)PE2tsw z4~j5&=rA|-T@AOC2#|dRf^{_bj>ylLbd4PxqC0xgwYl(5@7-4%jWv^Dauq3u)wT)6 z=puG4tdx&5I4xLp)1TIB4(5GY*?ZHKs_7EDpw-~IzxWf{e!x~<{rn^7`g{&)d}#`?E)UEoxWArytK$-&er;6>mdR;(lKba}vc?w2fx`NDXSA{=J*fas6fV=2@1hl&%r;-MO5W=P~H5asQ~)Meck z(?xEyuYPn&5i2zo!;*;WbxSK``(ji5q{b-`lz)N#qODz^!3z9cm_N%7dyW5wktmes zYpbjF0vYS;>u_$3J&PbRi-xzEnVr3}xk(Ar5OQ+Xi&{`HhyFOUMzaBOgmn|}?awd! zp-TgZ>cp%puGAio=cgtjVu%;97tW|)y!kC9>@!&`1G<4Np4|O~3j{(1sMaTUl9ktsFoG z`6@j8zDR7BB@!_Q$_#~CcF=c(UZN(JOJotuUI9{2(2m4~T6DaLY07$)2+9(`xT!Ui z@ZB>jeH{gz2^JQXOx5==4h5;Tv(wYjXF^|1f?#S7WJ6FJMDjytW{gk%AknDCjk=ne zzz~*#8Cl2KOOG=J1A_@@hXI%bnw|h$IsFT#GZg)#WMuxnzUWw3uU@?p6M5}{+V$O% z=0|7fGM^mUK*YDWWDApjISDFUjsIRD-Z|(w5+Gu-X?DH;%6V-~Ps?BL;XB?dw3DW$ z#aJw<;`&llKy$Y=NcrliK=Q{bPAX_sq^BDg8*eWyAtzEnLD$yY=V&$24VeHEPfCIq zva-7RZ#_R2BHYN@2e|E^AOX!7$jsef$>JwmV7+^SK-B2X>k8?sI1t8HwX2hK1rMVL zz4XJ@zO?uFNT~7B|AHA|5de1?;_E`)52XIc-*t1H5g59_y);}20KVX;{SjPW(Kkpj z?$~~`Z-#&bXmA+kE|mqSdVy{!13f)YJCp=fl!4wpKG%PT)hWU&?}LeF0(~0C^FMmKbym<8YbiVZz&7nAQelrDQ(v_>8yi?f{uM=tg87(lKyz(>}UV zxAZx53t(8A+hCgkjmUa{cznVGunaR9lj3d9`!mp4N$SR`ONGAGceu=9J~K5;b_6L^ zjKA`Jk{fjsa@1)~pYDiA%AE(XU)KI~^ir50<=IY|N7I_z&HA4%K!9}Q@o@5#ZEwcg zW?iQm-E#^uG8;k6g{7rx#|geSOPvP^5H%z{fQtUC1+aX?*_N;`LCoWmeZ^A?3+PUk zaRaha0IYZ;{mhh`ii(Py95&?8&S7F5w+IXAeNj;?wWyZMZ;2r)X zGu5xXAeM)L(GGN6bY^+y8~LJdydflgcf)QIZSo|d#&D&g*hL7>#m80hM9jXXHOtdA#~Z_0YF=)UuM3M z{i}(T%`a;6^OppE@RQTjo^iheL%R&!ISa6n^v>}<9DM(mVBujC2zO~~6JO&Y@2}(( zl$fC8Qii?@fJeZ9)!on4Xm37UrCKQUbaOieLu2JS(4vF>I*$7K>S`T4xiRcxQM0)Y zBE<@Te#ixIn2492p58BU{8k4eHX`B^#|?bK&Pkr^jQlAgN0HmsB2@oorr+fI+l9Zf zdmpp^QSGJ;zyIiE*SeJ|eo#2?mrWZUOPikZ;4O|nYqCnM?I1me&Zp)i&TnjHmu%O!Qv+V(uFf}Y zh#XrzsY5l2bi;AV;tERP*Ym{5ovxE_+=oH@%G|4>A{!8X_~fx-q~8YCn*Xut1vU~+ zmvngZvb$(dXUo;my#+c5Nm#*fCX^dghd230<^eJ=3=I>`*io2idrq-&s|hK7a3?*e571qCG_6^;)W{Pf*jTsdkz(9G`aRFaG> zW0Ib&dV=I!i-}gl`gb+7h8{HlyrCjo?uQtF-H|)z_lIjiqFlf7MZw`ymr8vkRhy8B zP#K`V5O7Y6$fW|s=TwB8ls8|C+PIv0Cp)+JkGhnUB=+lG@`g)Lg)Ygu24MT&|FpS> zc)U7Rvi%`|WBxhuY}B5~z^+WVL(Y9Su8ta&^Dl7Un&%RFRjJs1VBn@_UdSbo z-jvTEvI$V)pMqo)m-Z9Ta`%RbUg?^Kq2YZnypoK~%rpWp#xBOYL0dSegv|pR`^^DT zis144YulWD^zR=j&L>bL($yvHdSc`B)olq=d7{RSNb{o(yAIGqPLD@nDZqpW#3!;Y zHWn7>w%EWP%StFS%xdh^Xn~;0(q^VCTYM=%9oI zsl*VBH?~(hn*ypbXPxFmL$IB%n63aFqRW4yiTespoW0$Vo*v*%$5k*;#vo<7+lUVM zgy2+2=;-fPKhZ+Ey}%LM@a4<5uOr`lv?8NBo&=^E264aQb*6)eAQwfCAh zDioYr%ME_HVQ}2cx#MOiZ|Y8Mg3w+c|rFWr?Nsf7D<_MMFE?xkV=s zbQO?G!y3?8fJ+-5?n;f6^ZwN&RfjVdXGj%FlIi;G$=f^8eFh>rVb$$R7otL|OkwHn zHs$NYPLs}{WpSTE>JM0y!3;u~zQYaZS#*cw*ttMKl$5HZQg-+Q5dvII61zfq>-0Hc z|Ey%T;Zq6iy&|Yp07kqe-)p%8Fk#8O+a(E_MW*KO)w0+*$&0CA8zBfND89hi4PYg3 zeTv_Yh1tp?sr>L?2*~|SP2%p_-roNJe4!kJ56~twCoe;g2`sFsOAKrHO3MoGjq|;iWj4 zOiiP)Dn*OHdIOiTY~dwTbp?uG@?zNNU8YvF)X(ocEpqS{344>2auFs>kB*E43m5X! z;=H`Y>V8Rv$zZMGWfob`5F#KWJB|&ycuD;^;Pe1i8+d|WNk3_vi6GD_9Rgrp7unvl%IH^fCcGoSIOVDTFD zH@`9LbRW9Ue)aDIiiCa2sVb@VPo0yo{B&p{pB_t3ked0R&Eopr)4#gKe(Rt8de~;c z%9OgK{o9t7YoMk^;F}3$3Oh~8c`BOxJCSwVE`)rQ{x^sq{um~)_&92h^g&CI1k`kb ztpKI6*f1RVkS6Wx_CX{D^v5H$b0FdGkGtoEtZXZ29l#quba!9-Me`IBO&ZSjZ)Wqc ziz3c2j9M;&(mMO(MM5Kz_tsg=NZ18(Xw>3Z0qTX!?0U#~HF%@PH>4gk}H z*I~%wqh?B3Pu_VgSvo45;8UWaqlrxxVeJ>}!I?*Y`EXj|g!|)9+sA&`*M!Qk z;7M`9K!R1Cqzq2E|fiWO7 zyFj(71^_sRr!ahfgbBtd&%^Qqk1DD7`S=#MD&1F9==nhk;>VCfTm$6%cVW!}NF%Z5 z>wY`cMfz!Aclpd`h!*ZjmX^z4xTd`F)W93=*2U8h$Ym%0(E+w$NZ=Bj_BN0FkQoLN zMowW_z{(DyMd^BJqCX~?d!v4y?k*7HP!YP~m~B6J%Rg~n-B8h{e1#K^kRRkMOiX=g zyx;A6Ni5!`rYeTJoN`moRa2p~g7Xvu!}rZNP!StqXlQ6SXu#EUh`5+IGlPL9tj7dA zIBph(PZ#Q8^ANngEi;0f@xDyK85@xQpR^=lr0pc1_)0_{w=5o0A~Ae2=A&eP+=0$$ z{5Ksp_F+w_oflbl3I~IC{|-4<47NE)Do*cBaJPxK=hJw;_W0v5qnrBxR4XZ8%eLg? z+^t*)1KjkjJBKBAT!c|uLG#c_eg%?q+COMm_>_#QYhlX(%BW*1?sG;*z?_?->mE1v zsv;J2gklu(_r?Jg@Vh(PNmk0Ln;2mWiipX_7(G=kkRyVuQ~Mv@M@9*LT9rMu?JCi? zc(3|P9dW1YsrtRR<`y*fIp)8WlgAo980MK4isHCV9v`7_rARiDL{PAgYz0;KS%1;Fm-#ywdajuS~lHU2op7&|?OTb}{s2syK zt|5vSQa|QYqspgY+d7V3MQ|YrXdL16tE0@&EoE|QR%KqoIu5%vVgE$^L_dYoU~xXs zLEZ;*&bJjyaHA#bb3)-~cdB1OWO_SKt4&3DP1~4oquZ&7$Hp3Z%R~vfos6bicwwdi zMOE9WsSH%R5&R`H+hTx5@t;a-2@DLhoGd6Qu?M|o-In(D_DSb^42fMrj=!Zb9ZuQE z?qs|dcKmHtyT~Cdoa*QTI)hIF5hk6Xz^}2eK$(Y(ii@2cH5ky7RnZ}LCl^G)j0ZXq zD8Y9}(QI;*pPQ+0+ujOaY|JTLHlS=lq_>mg8Wz6DV@evWvV4OPeOHMmv$&<3^chqN zqN1XRJ;+27P_XHKguMFj1XOaqj#lucI|2)f08s$#y1seNoX&>Y3S%MsTQ$<>VXc@tZ-B%ZIwfrYdW=o@)mXV zO>+KEq&$Br3!liKAU-Uc2`u?fNULo`+i_3@mI?iU10Rr?D6QUh^a!w(IT%Z_@U1U= z@dC9p2WfHVGy6<&U2bh=~)uRNOA(7 z&sPv*3w&wkUIe}KC9K#mXuTxjfvwa;y-SW0rV*${*^@j*r<23f06;IKpqn0q*>Gqc z4kwF)7J|U(0UY)?dZ=i@QBk<*k|esXKqJW_T0%ksSYB5CuVI%G{*1cowie%IiS$9# z$piEeAjc8&(5lC8NPU$8GE_z()8O}|sS|Xz!4h}_O&#cPvjK;X5D`&PrRC*yXGd^+ z*V|(P?B@})z9ruTL|^CRAmA-8c_~^+osb0O_`ZjGEdnXb?axE#flU1pL$=KYu2trea}W zphzn#zt*va*fw=11B#7xC=byPcJ^U$I2K5fnOpl*wau-Hd;&~-N=t{VoypoLD}%_q zStwE_oq=vKi-L&yc>oCg_#}YxqP9W;1j~x0_Y_t+sx&CG5UGosdnG`K@q7CDzUg+Z zc7y*ApXdllX{0XuBT^oN;HBV|VDoQJ`HcP=Mm107jq_rDEiP$uY#nMP%i>TqvVaN& zF#rspDW3;%Dur^9aA2Q^TDAc#(&^EoHmE4%qq&XJJzs#VRJTxk^xcPrT8)ty`_WZ= z*k!bdGKOW|;N1x?!k^h!rjuY;NgiSmTuQTR{oM8-o+&*dkI#l#Hq9< zSvxIUe>rKnK9@qQ&wf1zOzW04a+=||t!#7$9IA(STfj02Dx7lA2|gq!)1`u}02&*= zGKF{N;64G?9;|?r&(Z#6(6Uhe7M}I}?p$LMN??$hoRb6iP7K6nza#cyH}H?Gq-mp7 zeJc_#jx+nX2L)POzqNJuYClWHDn-29L?FCpm|sS z-0H%R%7p#xurRj!7i8P7?o`FBH7RX88p5Us55q|C$|;8x+f>=&1d!tf#7_(%P1p#D{jwQKuZSe z&yAv#0su}%6?dZUxG@a$$=j6S<{xjqj*N8wBrFY!&H?L)Zp5FyqRXwd}+FeKv0H zE4bMwf>O|&DB%9t-j0(@$C9lqu~0M4WTMrIXNGu3lldkh4@KGq9r2EU7G1yzsKH8> zX$kJ6RRtl#If1iQohXd;ccmBA(Vtbn@u6P~f(9H}_?s#mHlII3>iP7Yg}#z-Dn#r?hHRa)EBm zPw9sUkKc~y2uI%kkGJoB#Pa|Dy^M@(viC@4D6+TggzS}UB1!fR86kUQ?_}>SR5oSr zDA^$iWuE8#`JC@L=Px+#A9^>q-S>4}uh;AOd^{iHsSQR?=OYBk$Gx43-WeD>(%Em# zR2ZZf7#)iI{ak7gYslyS?Jck1J;+wvo2~8L_ky&!I`D>tu9cFU9DHB5cN&*sVqzGi zeE-6#gG$4hTrBlbD$ET~`N+lNhM8eYMcwakpH&38X+HjIJkr+(zM)7-`f`=GDqB4C zu0SqKadx9$USGFuB>T47kgL6wBZdSnh9Buo=0cxWQ$vb?Qw=KO6MMRw3eFtdUcvIC z_urTsSFECa{#MPe05n94z(nW4y2khB=H_l`uvq%}WKrR3D@GjB#qFxG#Ps~<%;@e- z5pWQmMZLF&9c?J@L2(c8;`9eJ3F=Xlb1)}9@g!AH9Xe-QAenb`;OiUoWB1%|FG_4@ z(tN&x`Q^6GxA|TFa-Gr=jf=5)>b051z4iU4#exnrJXBRPm`$q3A`7W@q_GwXcD-; z6{_f+kI}9Tvj#}sLT**asNDJc{6!@CF9LMRZ}YC{_`iF02`8!MG8PR0A~NqpShivG z0-_{msE##?)igAups#jWuKUD=(@Jio8FU;}amoCHBiG1M|M+?ZQ$2y{uh>r?+qlEk z*&()T4$y5V)Q$jQB&n=|>Sg=TyF^FK@8tP~%zBoJU)RDp5yGUzBAcj!OK?*)C@6yC zsSEbxd7!?HQ^W4rbF5yEW$^ByUEOL&HjygIM(91%1 zImAEoA2WUNMyvjKpnO6kc@~i<1_I|TdK&HV;UT^76BF1;KJY`tTTg-JidQxw3qPS(QVge!=F) z?g*$@w!vr*uoG}baDwLIzrGNVWyX>+2<6%Zw0WeLJ3TlB+M4ld^_n;%@cV47S2FB0Vt#Qep+qs$!!fZjE4JH0#mR0hC zm`bpwXH?Tv2K!+r7NWF?hTllZXXXZ_*--qxNl~{arN4$VL=Ry2Fy>GX9VC+gMnu9E zAUfRmGJ6ZR?U4mZMj};oIy{|npIOV~LSnnJ%pW8YO>Ej*-yL|A9d+X7u)co%3eSJc z5|8t#p^&^rpB1eYoIMcd2II&hu--)Z%UquC1x!v(LJ1!k5g`gD43ZvwP7ECRxYwSU zi|`Bd4OjYEn{}jD-Y9&AJy&EXue_P3$3*t{jxKHI^G7d-E@#EsYv$R_%*-G`HO9K; z;)#TomKN+-e4L!3P~3n)0xDJ$Tv5>rpUl>TejJ;Da#4Wh#=SRnP~?7l0n`w%SmMM_ zA6YTMFg7w85bqF^3dzQT*k6*HpX&0YY~0gB14O`!#}gzNMveR|5(R`H(@Y}57>^ez zcNS4(PXNzIb1;E}=LDdy8K64D1lH+RnqIjPIU^$+w<^Gau0vO8efe}qXh^FQt(%HJ zkAx>{>8p7cKAtwGHdoxD-`5oX-Qv=eImr*gub2zipz$Nv8C?pTbHX)_0n{0> zlTdE~tUyVb0+)}Z(lRR}BLj4l5JoL-ngvobNFVR96U4&C-kYiZFopm4iJSh6i%6ed zy4iG;GDfE>W*sfYq%f1MpDyS6CN{#9H2)(}+9F{hudaF4_j|OYRUa{T1R7FrWIE5S zuC88Zbb~eI@3!$U-}d1NppJ8dV3~o9l;{{`C&^iTzb-C5f2SMszL61)%P-=QG0ZDh z;$;0u>DOW1!%Y5{xdPqOqqGS!HkT_gXhUd&NN>Q;&keT~F&a*T$x|_`xPJyn^$oIg zYto)o>K!-}>O}y_9$cg3cch4%qM>a^jnJE%cubmzk%55$u)~>dDYPIl)gr)BEVADV z(AyHPulm~=2!GR#$&1DGcT{82`^B9XgV_5D?@1ZBZSAC^Gw`f!m> z;wlXE@~gJHCXZl@#%%To!pZG-YQh)7Cc37)IW@Lv57C&f(bJzpejgo7Bmj9(SZ#j3 zNs9Ys5$^gw1xqp|8=A+ySq2C`m+0!6YN<}5X@1#hK3_O`+j;E>R zV=BQvd3gOr?BAAs?eI%)iXF-npB}KoL}1=LS5&qg3?}J-PvGI7uqe+dAvlsrjFwvTcOx3{;d5nyApf}*=yVg$3`Ii7rOsJhbd(+F8?`K?>aDgM*Zq>h_ci;U!P3 z9XCW}!Jgib{s@XjE1f(IP*aujsE2%k4GZ$r@e%OWE?8yTxet@4GK*+Z5ZfBf|I4&9 zpDs7npC={J9&KxH?{7&oYxgF}oOxoR)D80=vEAGh`adtd`@9;F;81PG;uHzJ(V?~HBFEAJu{fkcK*5<7fZH+5)s}|on z1L^YYOYe1*K;LQd4=7K*E0^nVJ@g$F>qBh|PHqolRsdq@?>79#B99FSL_F^@%GkAa zU;&kZLVFD7%V3xis|2{PT7C2bhy#!}tn_|GyN|a^_)=SK#cDdMJB={OUgQ6ZupDa1 zlNfRdFES+5rOlZ`u*gz}%pb+6INwQ5(pz8GwOQnRpU|Vy!tLWXJEL8XhB7DUY5b2l zLBs@@6ZBm}SxuGu-=c{IEQajKI)r&BGjVsL2-2ncd{UX&{q|hERRTcbfIanoDu2+@ zk_is=vnS0i*EXl#{Eshyh!2zMOEg5wH-t9S==uXr0x?%>6>B$_>W(JmhuS(x*_FJU z)=}P`ngk_IY^^)D?Z?-K66=> zTFct4SYi~S4okDTg&TxfQhEN3%;#NeY3oDX_xguh?a=GP2kEjCY&rXZGpzyfQ31dZ z`tIKh9n@&*ha~M+ghwy}Pg?s1*#s(ba_C4D_z)6$vW9GA?t+CXn2xzgvx$qR9~DAz zdSCKTM>naA{0TABLev(0F|FohM1a zyhR*AoHoO&7)Oj7t{kXiBT0&+5yufz%M=A*^kK50vykAak(0b;E)1yG50H6VQYNOC z-pZY!SLy~+)ez5B*R>H;vkvt-(nN?8?^b@BVV zEJjX>ezWx^{p?pA$d%(&WzXD}ORq2zYzXEYPHw!AcgrQ|AQ?W(Gdj6jAA6V*eyBlt zUl>!62D-z?#B#vQ0B}?K>{qa*%r>lSxMKO64ZtfT@vt^6*SiE&BOm}9&`@qN)XqY} z&ln7k40$MIiRmJyIb+ZoK%-A5_8i8P+cCEBk6%Oh53D*gb6D}<0Pu|GI^|s#_$@?kde-43rNI)alC)zL!ELivy@`xQ+-3}ip zo-&qBp8Tx+EG9J%_OY-V(F!@}=sr=a-!=!{70S#NpkN@jjZ1=E5KJrBqnoG83~RxK zp;PNGSdQe#{#XL*T*z*+wy_yIzmx6+)Djf==E-6=6)3q!u3LW1bp33!4FHLl|CuLZ z08V}n?n|yJe^+|w0bjMTaBDbU^YO+G?9#*8(w9IJ2Z^|CRy6~-z31TipjaDV+X}|r zVN-Fbser*R!-vB_4PdyVv%J?QjF#ReLh&>f`U zAEbb@FK|0tmW3jX#$SO&;dyFIL7l;yN>C-Ha5(lai6b$+*o;QqqmS^ zt3t1|@D=4$h3P3^$v}Tb8TWiZhujO#mVq^(c2V=22fCuigV_Qqcne5HU=ue&N5tH< zht)o>c!CoeobET;IxI}&F-tqSWujB+>NKBCL-oErpmkFI4IklX(I>?_f%UB+v9OL| zm>DUE4<iGw{ z9InBdl~{0jMT2oIDsJ8$Em{$#yh`hiKH!Z29*fT^sD#A9kycxG7^qpKbdCY}z(YlK zhVu<>%3hwHp1MS$V2m91Y$X(OA`WcjQdkfaI{MCUxzN}+ZYj~K_~p;vaUQ1$AxKT5 z!0dG5Pa@`1`M!(gv3qUbD5H6Eac3DV7SZd1Xk}bx!fGd8a&je zeUc9QbxgLMEe$2*`pn8d-tn$*Kr>bM!i#Jez)8dMHEWrnmzU_`1F#Qy4=E%%T##(( z^JjyE*!=YyT59UF^mHF#hws;~&x_CbPsaMm{;*SfR(C~|N|rDrRx(;ejwJA zZZy8J&Izx>Gme2X<3RreQ&&7P=E^mz!-z`H=nu!tfaV18^Dn;&sQ? zzWM1l^z)>cG7>p9mAw8gG=v%DU;=|K@e{C6X$E?uk#@56nSTG`rfa_jTSh3bUooXP8d%KZnW8CN3b&c$J(Be5VR^$!`NKEFjYgi=u)Eioh0p%7xAKM9 z%7%?{&FxUC#}+pR(=^t|=;6!mVdqEV?!4+4ql)q0I4|4|6-zk~OPPdrT+GZ5@^ygR zOfOXk-~tu+A|n#}AIV~V@6yM#ipi2weLjO36}ynW2j{NTJXHK6Def0Eq;igTXfs8N zq4NV&R7~l>AKZ5CYW=xj9{>Es^0nmJzNxN!+GyMII)-b&g79&#-$O^!_s@Fc3B;w1 zF1E)4oc`?xGamU?dF}PSryJs)T5M8@)r|DmhVl>Up()$4Lt-*mlqy{Ra4Cw%~ zYoFjCq?h(@d-pd1yn&#c1_Y=>qnP{VO$~MRg6eqis)yMT%B$U@u@2I6z%~7}b*}@D zh)9?w8eGg~XJ?Td?GmtbtcEhrVSKR(%nEk3h1HzeRWolkHVi^8$`dPU@`YSciaYfXk zy7JL;sf(jaW`zbL(WN5~qf)F7&_jTb=6>{BcuR0r1J+9-2h;=b^Gb+~U6xS(1`m0r zh>Hc_6?DBo|1<-fXRYH^bo3(B{(zA#5E+71Wg_PTy2xjR5e>W4zy|{Fd$7x{VZxz! zbzMMUA8=YgKyY#a1XzQFjSTJKhSF#v$G9<#-YXm9UaZG zFaY5YqySq4=0M*$magJCH9;$wIMxEmM)OlO47&vcs^7jMqTj%<=KEYD#7 z;Wes*8Hr*@ESM(2_;OT=|K`mn@btt;(Bb65(gWP+L7}!``4oIqaB9c{MmRjdV!&(D zHLP-&?t+0V1N6yKQgA;33QkPCmOlKF7+lGU!OZR3&*^)(goIx~!vHs~bbQmw5R5b1 zV1IDyg~MBPWdhT!&8dq?TFwn@TRhcfAd3PGXxp|#m^`mMyrg`mWmKVy3l559PKRQcR zFiaX!da-6~Y3_=t;VQ5t|1qi1%6NJ2j-Fz)6%Yz5z|IQnN6`YIwPApj2EvB?(IKd+ zpb7EtiBQ^^ScE2x$EcGLn7_vjA+VzcKPhu}_d9YNpyB{TENPCs zzH`u4n^m>3;YeN|VO}O@Pbiq;_~|7=T+vq{+lmXuiOD?duT-4BiE3VD`V~kVsEysy z(>Z^LZ`sBBU0iv-bU^q1;b5k+kAuS&FkYbFNK&;1*IMu6UC7>!$Ws`y7~PUH{DI z`(m8Q7GlU+6KyI`ZlEKGEEU3RhDJw}M^J`kG0K|WZ>|{G;S*AHzWt8Or*R+ertpZ3Fviqvusn zsrvsJRYF?^xDs?QNFeAX07NIJpuoCv<#2Pn*Q=sN8;mJXFZbX9FNGQqU}4VvHvV~I zb4qzqM+x}TzWOu<8wHs50Cjd^=Q>LFVPHT-N9VgIFDa}pSzz=Q%MH$+u`z9^9e{&% zv@yD3=8)l5>ReD<%qZzC$j5iTTo1_NZBU>wWPIILWr`@QbCg%gjli<^`pEi`xzqfT z5{;?&if)bMUoK*%TWHXHg-53*j91^MT@p}=)4VjQx82ejdCXOc((?eR0uBn$^#X(J zu~^cxunDgAYjVjo zrhy#czyQ`OOqhXamMLXXGC0i7yh_sgYP1nUEEO#3M>*rKahRhbcnGCwwPL`a+UiXs)l+8NVnODLIz1V!pZhx9R# zZ}K)zC+(Px{V@#**_w8n7FQpkX5hG;mVI%|MIv*ZLnhfP%8e_j(kTA{SJDf9dzooH zS+qlK0o&$ssea6r&(T~bj3;dv@d9aIDrRgx(7`B$>bon02eV#nAM?N0Z1brYPE2pl z=_%ObR%;$B1)bhzbG8zED(I3;dqnU@%ZR@>I0rojn=RDu{!SfxrBJqEr#*OqzQV+xuqzP5sX3LX6>cM*<(*@mRLZcxe~RwN2|GehRLRrE&fzJ+cy%PF_N46=rll9#kN zt^UlxZLIXqqyFhC*)Wy>+MFx0JhUde=i@T{XgoDrrF*14hElZ|iviZX8b zvB*J7w+WkFok58l6$TR%rWGz5ceviDLqi&m9$ntn@1UC1g01@ls9SM@aGvQ@SC&{j zOc-?K(ZBO*nlL`z*Mj#(9yP8M(?%7$@?=Y8&9GfT#ijKTzGCL>Y&9h+b)ws>+YZVomD zwpvFRts>V*OyXZx;UhI{XiFb$mtUT=U+KXJW%*8hcjoEA>V1N=?bYwTgwz7qfy8|^ zY(e-&Ws%qB?)`B(rs&Q>FB71`hX*3ik7a_W9gV16Yl-|3|1Kv@p~ZRW%F@N@lIvBq z@>H_$YX#%Jw~jxAclywz^kE#gFM?I(88DHQSXiC^4nDjXgX-Cb(en5Log0Jv+XrS} zPe1)1Qo$=t;QELr1D)Ph>rEnjgWSlh^2mnbxADAJ6a-l`B6MXBjVrMeU{e(k5U86J zDSiPL5+7?tmKCzU-0oWZ!0PjJ76vD+_eEg}F#lFkS_*C9r;-_`o$}yiu&b&OsOnTd zRN@tzf&QaL?*r<)6)}osd1|asuM^z$pT7tWosIsVKfvGp!mOpG{=a|zhHE2y>;HT) z)(5hoC;!hkzr2yU*!Q0|_1_n}56{(qUiH5(o@MUN`rm)}&z~i_ek%U&pNjgQ=ZpV; zKX}^X3e_=9^G_~;CG=S=Bv7P)pmz>O=55FJCr{$I!CT>*?17kwj7R})SI01tDtL>^ zjOmrOVNKcngu}F&>)8gstsmotd$XiPSYgeovfbtlk4RHql>x=8o0`*EiiwP`-&Dpv zVtFThuS-{lYf`f5YVfz~?u+R&2ok8YwS1~G|Hyp>BgiC+b7_Wqbtge}gvk+uaMn5r ziNj&J`h)WGmBSS|EDKXl$?2qhTcw+Wu}^kWoOe>xJ`+3MCA7z6VnI{syfFQ@jFzgx z8mvWK^DkLw+vo9!jkYg78D38yi8*GVOb{A#RfKs1dYJ5T;JVD0Etz?$5E<{L;f!MG zUj<9y>4!dGU@jD{NK8QPeFxnP?}f9Zeni(>+-HU_pIc^Uc*t;x5@ZmFZhvKw&H3nW z+MMHWvLRE@?p#N2BYe>x_bS0Dr6cyYJF`$&BF-s`%o!sF9VRhCj9aGkDT=iL*lk_+ zap=uHb?|Ogk3>mxKU0Ym+LYG5xLNj1rn8Td+14IY1~ZVDC9BGsj+IjSdvNx$xcY|< z-tEc(d_{R#E|vI!2L)|@Q-2E4Dcn>$9WZe)+>M-B215d8xwa=dUCG5o8AQ6Y6f{dj zD-7-|b-&B;@y5VctnXvtR12eh-!JUJw2$=*2O*Y2Lc#;XlQ9Xpb3cSX^kLiBYOTOm zOzOMN+3@P2ve$c7g0MuYQ#75t9p#$Cw;3)U-~1tA{2lV;5lsu@e?F^MoYRymp3`P& zHPLGf%ubv~kiGHqQY&0NkZGA1Cooar=D*3hZT-7L(jj{JXw|EgQDSlphM#$Btjj)1 zNlYsv49rnTke@5;>@pH85^}066qj7P7rG<|%uBzi*`uSdsaMj3%Sy5>@#8UR$T@T+ z5M*aOQ=i&A)=`fTV|~u#f&FSVcg)hvF`E>}*m|Xh053tUGwx7-XquwRe=Z@T=p`Z(gsE_sM6_VE zIV^GWz0Ng~HDay4;F6eZQY$DC@Xxqw-O%v`us|P zOgJ`|O(&}+qCFG(MVE#Z2$cp~&@r5RxEr-hPGVAGJ*F}l1DU=KGGh4j)iFvJ@z4KwiF$eYEurQ6+u1!7u;3MX z5{c1){KHcYdVM3U<8OoA{&Xm1r9o%zHylJ~T?_fnyz<7azwz#?Y4TlZOBwWT{}((( zZ~Vc0q^jY=&z`wj2|zyBbg)f*i_=# z7R%qLP^|0)pmoq~dY-DYk}IPzqx}r{)ptmQCLL|U5#0uyC64?%6Y3JXwtd6$zvKG*h%5=hJk@y> zd6+(8kRqHUI5?P??zc&woceNz(lNzG@fF~`+CG({I+Xcv5~)IQuo$t-yh>0MURe8I%lt@N6sbU z2Q*LEW}+1tZdDj2jPcmuQQsb8bPeQxcK9GkMI**@IsEHV$}6YhtGLS~2oBEk3{O8= zn=u0=)L0cLajeW%K*c{arF=K> zrhxg(sUfD2ME~eAFQ3pay?d+oz}$e?HP`pM+k~s~QxQcbRwS=`xwPu~xmq|Kx}&`zhGMQJusO~Ex`RqG znnozmTCL_=%TBj#Uy71y6#tL4`Q#@YV(C^-X4Ofknft<@qC?Af5w1=^CwuFb!H+1% z$BIRVuaZJByd8?A8Q=aO=HMUOpK{Hj6#V#BxK|Kgrc%3HE&SMB<3<-E|(# z<2zyr;tI6xaE!tn&A)#)oy~-k5}Bih?>GvoNnq1qi+G-N9TkAE9kKMx*vzmRZW0m%elYhd~gcoIzArVG_5 zqqrZobE=F1#~U{3LvKG6_~gDr!(isuJq&_j$atZopa9Y19Vw}=gGSa?RvA)$Ep>H* zz(xR3Tt`P%%~VlgA`ybCH{o8%>{CDue>13AC)M1xCY}#wmisn*WUWih(C1zqk>y)qF;*%y!Kb_@G zL(Jg;r!gD)moyu#D~khIuDZ51DKi^jlklOh-i=n`wlE(Etbkym{S8q} zws}~X`#|@qCJqx>tTofEu8)z47a1B6o@kF8IeDk{{3bm5>}4|6xo-m`2$u>HbrMyr zVFLt&vFAW{O9Oa_su~(MIXHeD9Bf0q1l-Ced7jIRi9^hy zAQw*j_Dk)~Q9-hNs1i<`)`yWtUx=F$8rQDF1(kGNi{gj`ro-c}ZOuMGpmdv4Ir;nd zV{fky>k*W|fOLX13T{1!ZOAb1lPEp@Lmn(X?Ox%iKojF2j>Gy9!yJ=};<9o13rYSg zSEVTfHO-dMqx9&ui=)FcbmK~fx!K!qNM8`&4>V^X`)FD4?Nix9+MGq4|1$<&XrzO{ zWVm-XkI6|$s)0?G5)boum>BHZ-^RkHicL-3(EbafT1Z8Lp;s)MJrE9K{$Rm1sOyAm zib8NzFe*2zEcO8dosBG*Kv`K^69BmSFF1G9Zfzu!d$8t|Hv^_NU_Csd3mRtdg%PI0 ze?`Fyj9Ju-owVOFbd~4UYux6b@{tG!q7=8-WJ_ zmm}WV3+kdj1Y%bq%_U-hISCUR9Ds;$ab2V$6!X(0yq*CEd2kh~Wsr{kxFtbJOw125 ze64GlY5)uCmrnM4{75W^GD%1wu?PqVIF{elzResGp7*!^MmBIdKQdfZS{l0l^eNR}z&{47XHL>!o$1g1I;PU(MK~NxlhVsla$Pi%&RqTv>TNxoxBN8ZBx@< zn1#CGtOG6OE)L1vIELRa(1z5Fdsz~*)@45r!I!e`uD=kFEOsV1z<>+Vl= z95tv3Onuat8qXZ+&AS(t7u`CS5->kED7N5~v3k_OnEDS`TR=6Drjggq)l^Va)OH!_ z??;h6#rN7)IPCeUPL6iOVW9_Enl?(Vrp=lk=YfjR(Fl@3VpTL&ta4M1nN(u&}q zgsDVu*YY#_(n$vto`=DFq?1*{GcE}fNjUIO8O~l=ucM+07_st4MSzELoPiBu3RHSH zydqj)^7Q6&AO@j`Y%raM{6R_E`qy$?Qqh=V?ZG%`EYCjjd~7)K+2jk#_9aF~hV!o( z3k&TrJT&x|MpyU|5t zY5XZsvQoz~t8!s}?Wlw*WL>0cEYhzD{SpNt9Tkw2ZL^&@s?T?7> zMbU|l?QdcSf*#Nk_CGk4(Se;72&`>vOu$WHgb9Ia2e5@TWq>(8%9tL8!-ZpDv#(J6 z1?Xyv-~tBY%9P*r<2Rv@gE9lSVqB$@TsU%Y*l(c!1+O&VH~0WY!lrh%xh(~CT&u7C zaKzTYVqmt~o49^5kK{`+%aAjxY4LoOW7arxbQ#*a#N`u60&3O<%R z$Y4h8&Onw1es5^zaK%B6SriIWYtKI$S_nwpzk^vFF#GvTTXY~z3Rnu8&&6tICER}+ z9J_-*TOCjk+3&qMTbI5Rg+T+0uX8^*9{ad85W$cHpNTa7%Wq<6R(jdDkT+7~Sc>MJlmy9t&O{r#K}rX}ui zv)2Wx506cGv}GVkfVlcOpoQT311tfk=0E_6kM#jI08qNC?)U5|B*gRmpP!Uo7nVZ@{i3>~28Yd6bIN-CV*25S5?RPZRhXS#_jC~9;v zfK{Im9}lZP`OO<6O-_{l|~Q17 zjScAp!V0RE(AJL6%ru2d4AL#I9Rx)*%+XKavj34ld99$V6m_|NK@Beva3O)HgUo05 zA9(r!@w8MRpySk=eg&XGU7S}7AAb7PLE>u8c-MsIm>pNPfdV(OhW^g<{j znds zAc6Ltox-UE0GQ?R1Hh4krJ~`%6LXKDzWs6+|LWC&zP_i@TX3}JT!96?$q=?3H&|j) zQDfeP-a7)a4zxgx;#SZ7JTW z=DsX{oqBZKkhSgDO7&o}NP6C7^5@%c`~FY|b-bCAeZTMW;q&WMcS%Pny2qkZqlsTy zem1U`mUsHewyT}O@}QIZ4Z7yx_D&EWZdL*{$NguS9?c>>2`YYy$m@H=4KP(x`-h@8 zNvLF;>y&)0oOD>cqFp)tfUgV$11j+nOGW>_(7tqXx`z!U!5xivjQ8-Usdf(jeDKdc zd)Rm5lJZtr>0YN@cBOR8bNBg~Gq2j9Q7QQqU&=O`-<#>VmM8y+qK>>=)ydR=MA)yf zmQN3_bZBg}>7nt<_s~bBT}=lHpiKGjs3J-;cZz8axkyz2o_dx4;^?lO9dX63JqEA_dtl$G?HRh!qGo6-vAorE8ynzq_;0 zJl_5bcB9Qgs8u$zX#SNXpN*+5(5e$E#pUz%?>8j16pJ|*rt98ak{fN*Y#_l;@2sO9 zO7E(>Yd=R~zCe<*w55Mk+wj*X^m^&D%YN_Nuk&%@Kyb$hgaW=r#7s3_5J=!CxmYtT zkWn)P?c;s!`<$P+%}GEK@N4G{@R(dkLc<;ze;IS6jkklM@E3^#O?<_lT1P{3j%5CFL_^6yVu>O55L1$D_#=WFYrBSOG3w#;#R@OnRqS}U81|RWMYX= z>^_|7ly;vsD*8AAw{Fvm&XR1zFU9g)A8@iL#r#3$k=qF%3v}bF0&JnBrv?g$m`u=( zUrP9j^w9@=+ESkv{rmEfF|DmruREur`m15lg-7OfM2l-0xGpK*9=>;w4WOSGU0;i^ z6A*gY^3F&@v@SFGBy&ab;rZ8JH})TZb*8JVEE>WC=`yo>Y2Wwu-UA*2WI;el7jhY4 zAD!DjcejBwXTVDlP5Y5Qa5Cz(R7_)lQUEfX^G_`G^XCqW%DzFG`Qk?qTiy-T{m zD`-qWQ-Bj?m*kUYR`K{_cpv}S!yv(GR&u^vF4v&ehTzb!P;OhD{tnp*9VlD{prtt` zYp@@GNzvEWmt#3Z*Lx2o#z7g+!b7>#5(iTC>H6+Z7k8p$-SDi$$a zyS*8brWP%qpvCMfT=5zh@w`G3pD+|m+v**bMV-n;s^x zzgU!?kMqGog8>nN#&O5%l@icozec{i(=7#Mj*d|dbdi21I;6G|z`6x9L;wU6*h-*` zB$E82301jP+_5M0h!GIk z0m%fX4j(t&s6d2S#){es|CW+M+xO+waZ%w`k3Wx>g76&Guc@*{ezahoeiG4dzqVp~ z&*qiiR>u1B>?8jbr7JJW@exEN5T}nkuF}hTm;o(c4CRNb8l1`;0y~yFlnE%n4_2w@ z+JoqVeK@4C0--L_SsmSf)lM3;;3jMg*UB?g+3O-_>+CfJ7 zTrig3C}6b}6BcNJAoqS55~9=Ev*!o4sEKP+HfN1=H=q-G; zej$S-P?&+H4`j$5m<|hr*CudN?8jhJW>Z0#7S=|-hkpXrVaOLCRGVn9{D;7bQgml$ z^Wn&WjsqyLFqiqhz7FN$cQDh#y0;LZ+HfW(^ZD`fjGSMNVgvD*@x#U4J?o-~8B)&3 z;YWvk)hJ>Ai}KP9lf&CDZv!bB0tNh_pa;7VKnhX8st%S{z)zf=jSc9Q>FVsD=Y_}f z&)!mY15R(LG~nN0^og`VE^kr{-hl+zq9wv3rR$B$SN{sApr-BKHoN z0s^>CN=mw+90?#TFMzZm#BV(MW2Qf>GzOVmgb}Nhz2jLKSu(2sv%5I&Ds=#OfTE(_ zE-kp<3DRV(1H||Y*LLgs&n>h)>!zFa^dY&ChiwUF_*y#a}aOA1yGw%<8`2Y*c`|Phv$~lb3@*Y3Rq@30U0H!U1WKT{< zN2l;~)4af>x{vMlZ88LSv64R=Ao=ca?1{n)mV{+{t&PMPGOQhrKY;xq)!f-J7bjUm`244OUbF{dz6U_}DV;-lBiJ+p36$B^8<;9DZ4Q0QVXc!$ zYNzW6e*MfbH<0l@Ln4=-g*7ZQt+>jjlWPXjgDC#p;)oKD$dYtcWQCQB)CLDG{nec-n>_ zmc0J$S0^+!V8_)Aa|e{S1hg+=nH8{o1!EC-gIo}LnH_BrKE6E=uE7t>=UF-`DI|&S z+?*W8!ef{(!>&HoZq3OQMcU4O4`xwLiHW)H`Jnb>RoQUtzxn#b0mK%Kn6))!gZktl zFRzXlK@v(@itTytnXx?Mf!|N9hoR_b$d{x?I^Wnak=@6PEv>_u@9P~(?qt5i?uz+O zBE#pAk)joPA+d6(`B4vIC|_SiH~uYVIwl!i>S)3`DY`E`GF)%aN@kcNGakNO|F9={ z>_VH*EJ>&4@op6WDO3^pq!a@S9~A`-6~!?$@Bpzif*%4`|E~;|BCa;|mw-~@AkVe# z>3&hi{Tg`f-p=m<QDl_d#K_(HEdbI_by~0tKoOP`4zaN{KWu>eat5#vhw%{uN zX6#`=(-xr8!pI2ul1tZjhxW;q6Z#}JnzOeY-)TSiUoe8E9=PcepI_wxd@&06{GY8! zZZR>Su`iBXM_H9&Bm?ui9bBgXF#ZOz!^;KN&!OPD?=v8}BDcc!wT%QcH>3EP~z_L520igRWNY@vy514QTRNNDF6eqbSSLAhNnc6 z!{kN2_qeFf-+r~7Jon7z6v@Ej`$b$hm8>lPR$j>{pXhE62!#&nQMBFOJ}Iyl?Y*B? zVPlj@#L1mdX=8MGu=K)(?3VPy_(qF+-_{P~rFVCKjeD7) zqEgYjDdm<)*c{;$X3+EWJnf_g%VA#bukO_wDR^to8|&|kV9SLzSUQEzjrc9}fcvlW zV#{j#{AV~5pq$nyMxl~WT4w|d3=h-avg!kWp*=YvNiAJgmmZ&Q>~gM5<{gABsWu;N zh@F#&aWSwy&WnVKe6xSQp8Z?sP57{)RT>kfM3@d&nGPg-$IEwJ%1lEJu{FUc zHKU(uZ1CuRn*mnbVM^I}ge6D*6P;IjgR*GD2P^-)wHbcXO!Rm1+Rds2-snJ}(8c2R z^*zBHz+v8Ni??Is8W)NE^H7d0_K){nomIj#B6a}tL6IutJ<0&ack>;7J}`cSKnpp! zZh(5gXGs>L5M_ri_-hF6hkf7_oc{nu%z0Hu@(enPssGCa5p1>tnh|F(OO|t{|@8}j`vp{WP~pu(DKXdJgomIqq;*@t zKcE&|na2M;4esLi`fatYoHUZ@sw?GFe_-q0Ph=zRiyW&uOB=Sq~<+kR0S9D&)Q-8w$;R2{!4T=f9Tke!3FNM$Z_UrfjF+u+oLa8SJoW5| zdnx(UL##Xz(muqlqHKpLfv*A+p~u3No?)7P9N@j;2QTzS{39*?jZ0okp(})s>WLT^ zXijmItud{t5*^q|8LLTnf;56?>|Jt*D05bPCk=uEXTmKLJ|Sum(s4L(IAd5Y(dlHG zu40Avebs08lK9juCTP~He%^$k=8Frh{Jr+u&`IQ0OvgNEc3mAHr)>0Zxy8)Q{i?^X zne3@15u2q2s{(t7d(JI1Y8mv#GelPgC)d*R76|n5LlA@O#4oCoz3yG2`8{SZ6J-7Y zULitaw$MoTMo~jdLJa_5PBT?qy}e4wtANeJxK-{`q-v%{qs6DIDLB|vEAK+4>;%Vt zx+lqP^WnTuv&(LAU?;qPgNR5q+4>|WV=<4vXJXhHWK)2}!DJWCQ-mMf71qQ>nw+WZ@vbn>J@VFqpL1^g4sPev%ptLtzj&6uiOzM|lKUOy#&ADv z<$F4RxlA~iv>0&c$3niZOY<4u!>GYg6zVb5Rt)efxcH$30*&JR`Hu6S|Ha%}M^*KH zU!$88kS-CBZlt6`2?^=$l$K5b0Rcq?1XQ|Hq(NF5m68-GX%qozC8h5=-|v0L{oQfz z|L@CSd_wVf&fd>{p0(zjYfi&yCZNq%t%cA-0gy|T^0|rZF+|aI&a2WIibG?@LwGcc zA+~hVH{`kr)h@ZqUFYu`$_ z9*;?A41>0g&_=X>rtoQ6y{>oJUy^Botcorm}EP8Qm~px7qKXO>Pc<9iK=3 zpoqt$Bm)zBh(zHH%mR~fgdw~PHlc5KaTuX|Ik5jg!WrbD6=I!KRDRD^&X(a8BPpYG zR}(o=Z}@jE!UQfCjnwZ^D&gAXB{l{PhFqnabj*r zCcPtofv~q4*J~wXqNB&we74!A-rv8gzv@SntilQ8zz^kO?>BpzzT;{r9KBpN*+^>$ zDkDCh2YISSnE<#d6Jt`}^#uVSFwL|*jb+#O&LaQ*QF9{t6h+DB6nO5^%jb1u+||O6 zXz?k{qe9!E?SZu2FGq!6RH*NNIBm-AWxKz4x^sG~p+$kD@cna^m4i^{vM-l?bgPj$lz)C-V5-|tdA9g zAZZ5}@^L3$OcM|Kt@kG7pjcwE7z!KXgwnYX*LGz%Ta(O748od}1aAgBUEAelw9Vl( z9Ts)UTO+^UudJv-|MsdpgDPvmy)=84+RS-81Zx*i#Wt!Yq zXI|2;(agr}ei^uNcu4O7iVxifPr|?+_3PL#cpL5U4?NPuo8G_o=u6<-PCJ6hMoK=5 z&FUIJ)`avvL+-Xw@ysE6cF?V$e}AB=4p^FItPOgeVrw0b%g7Mc>W~l`5gzg0EqKJT z+u`x$T~D`)MEm$A-~RgZ$#0yaUX;8|3i1qbty1^Gdtoq*()i|hsiffGq6o_1b*llE z(3jxf`>@IAD+u`h02+P^{7Au;7zOZY`fO7pj`jG{nGo;Mb?yb*+9@FKrA?i2tg5pV zD#l4bTbSwTVNw7%WG+k=P8>`Kf{#;_3gIgFYKjS)hm6SblN$kW<^WME8AMzzA2ix|J6=Rp!G%|1y)H?+c^xRciDSr>Z2f?5% zHp1BA5D|WKi%v#1*zT!3i=6zaKhxHpY+e>yr&D6(TT>C2nyUq|iA z$RjV}cu0!JIGjxwv(pOH0zHaGOgF}g$<9AdwWoPOtqHrxL=&}rO#nEZZVRRY4fKvP z09tv}ERjj}!fB<^oRefwzjh}FNxGUXI_cczG@oUg;nJ@=Q;^JQg6kuW&Vfc*w4E}dD} z%)(+GYyic<3ledhD4V)MALPNliZX8m?#K{9^_}41`{VcZ%j{OkSEVJ6^vK0PhW9pl zaxe)o356_{8IwnYQ1UEkvdTiAkT7>?w)08SvAx}t^MvVI`hzzHl_n2PPMp%OZgc#6 zzW9WO1CK*8xjEDBaR_Y-V|wUsir}_v)9j5#hg9!WLBA|PZ+mEx!IDP+OfMC*xrihi zp#o^Nu!P*Qe5F^g)4lA<;~nnndjz!D0?OOj5muDz%Iml4r&@PS?c-$2DSMGP6Ly5H zDx!Y6B`Bb);dqJNl8!d;N2K<(?BE~Y+LKl+df(SMR&bVHpXJbzwO6y9D05CrViw}+ zlLG@Pk}C)x1aBWs1+Nt(Z=Ms4OHKg$20ZYG^Tv^q{Hy7(L{tn^kDon>m+kxbq3Qy3 z0&rmABY?w${;8Hoh(TG$%q(L%+%fxDv6TsMC?Qx1f%8K{Ls>wi=PA78n&bu8sx;OL zRri3z<2y}hO-&7$pX^SUx{t3sX)S;!3Ls}}r27^9B>;$XkMY>zAHXm=G~%yWzJMjs z9dlzI)li9GL1Bi>m9R}rN`;v7J~^dG0o;_{Z+z2=u^J)A2Lm5=@I+j=E<8r+MuzQz zF7Jl9G;4oJN;!9UqCAk~t4^~wv=#E;s8r&79I|Z+9#+6pbOAKH(Uei03n&NSyE!vY zt#T4N+a35H$w&J|O`XlnUp_a(3?E2NOHR5#wIYE35ag0wP9=A6ls{Q zR2j^6deK&r#?$X47WarOTwqPnT|d&erUI8o(aYk<*7b!z5?Hb&By>`vK*%YA{0P$VIcROGTybF%4TWMt&~;^OB^ zzOg}7aAIOTTvGuiNOzfS-B;48BXsvFOg-ygh%fFkV*w;@dULD zvTL0Qkt*{WWxojw_Obf|Gwq~@@f@o1L~klRrwPI1Ja1f2_H>O9_QmIZ^ps-2%6Pc? zta#l(MbLS6n75$5k zT<-2Wuu#8WaiWv*e^0%dFiZ!QAX?JrpbjhKePeeFGY90W6flJ)(oncQ(FxFaPM z#Tm189Ym4iSEZX+Uyh2bzL+WM%L-Om3aj7QA`ULLp!m7za+T$VDP?m zu51N`2g6a-_1G9>iO_48IIpa(LegSMzqz@&jkPuC+A8hGz<{EpK_nwHvw!ETGm;Fb z4_#3bOh%AygYkwFyKA|D^UkzWyVQnZvsmy+ZEy)y*6Q86cOiCTiy{})N3bn@H>?Q8 z)jD;5r?Ns5t^86o#LabjeyD|gHjendIQjkCv=3Bc)#iyyE@-9ZM|^ufSKQ5?!@m8i zxhe-$C~r3mY4xWYbEVuu(eY$Ibz$U;H>B zF;Rcn&3)BSZTxjz=EfJ%@rQ)ZP2-tdom|=LTK%hopOt82dqA}2OjPlfD?5{`()hH- zhXBhD0fxjqJ!^se^^fbMr#Y2o9{dYcV0{xA#aU!2{TW51K zVk0BISj&(^kStg0){?Bw%gZ(Ez8~BcHDq2iFX8js17sB7{Hq){1SbO6{%VusGJ=Pg zs+w9$W(D;_H#fHs5F~-*Z|q?gjK37TeGBO?K=XiRz~IJ3RW%4%Sy@<$J}W`vgxbwP zKTv%W3|v7a=eE$+x{_+sx75P)?}fD0Gp3sN)r3@}7*(4FvjfR3*V?vLx4Ir+GU0TN zKO0J4ZvMj~O&A)9)YCL%Q?Ml)4J6+pc>4IAMqQ@gsw_&kRt6pu1bQI4f>tmqJKLR~ z3J#5C)Yc$igrTwqJ$;@3Uuz;FqHb`so(2Kq_^K&1&H(I;G`L&(s>}64!HG&2%+zqG znSzKDGBPqyD@ppUC%k^GH8=VkY}z0W!wf98tgLdr*s3TiYx`AFAXgMmH|>V z+!@W5^m7BZPiUvXL(?%#%G^uIBh~ams1aK>0d2ZP06vVx&`J3aL;n&wyIYn38!E5M$rboj~jm~K#IK%wyZ z^=lBwMCD0$jWz8KzP(GcRl76W0(1w86y5Y|y&&#&_m3E0j zkS?Yg4y_6Q3jVtKeEXfgv7U!6ISI~E$uS11-4vg5vWs=&XU;^ni9fmF*A))^{^k#k zM@&#s@$n5$O`!}Z0qd)*XdZ)u1ZE`+!1NndIZ&-*)y%)mt%RtDA*n4nd39%J;hj27 z-3?Ud2*&Tmwzjr_P^iyb92e=nPd_*daJ9ws-9?+nO+hb6mR-eYA>+u?G4VK;p25GM zIJGPu=o;84?ul4LPq^|0f5OZI4-0XqeF z=L!N|OY5{BQ1>15gmB71fBNWf(|E3TvmqZ9^m*q3{O};>5{@6(CQ8&9JVBudv@QtK z9EH;0n!>YeAWqm<&hNb5u=@q8(_FPS!2_XXBo8Ick>R)(ZCoCW<|r(m^VVfMrrM$J zdb}(bzsWbgM#7mgTspm1-tD8bS^0hOEDxYg1gUZ~*fZszym&r*81rR^E*gFrVw0iX zxpU_ZF!199Ll$OcEnxfxTs#=R*Ns>8jiVk;&@w@?>qAVCKyd>kiouKf z96K78Y|uTyi^1<0l-7ZKA1cc#(f(OBV)DQ4g zWqBW5)QfB$G<)pN_-UFeQGk#RdRIwi=BGN;kPW%hWjF;Z?Pzn#8)jhOEC)rMzUVWx z08qsOH>Ihe5er&4=%r|W#oo7D=PU}Y!N9-(?3a#MYb)qy9Uq$0>5w1{|IMpbxi<$-Dd%DYZmT{~i0q-P7n$Qa|*q$1pn`mmeFeE);y+&Tj(@-RI zY5WT9=;z%g-=BYGEgI#{7~U7S3+!aw{6|$pc7>KP0l~_VK|+)IU$4KtVds5az&@U5 z&5*oi^Y9nl{1M}ef;L6MVeO+fJWm@}sg&d|TO1eVPfL%~y;+Q2UZu#6JUQGE=Ti24 zGCA@z-&#B;!uqFK?|erfO~i9jypF^<{qWs!if*@Aeav_KUABVR!#V17BL|ju!@1ZJ zKYsW-uPyG@@@nc^L&}jFWlLr;ndJrValy@*j8|TGFDa0(559a7SC@M#beKxB^LJX~ zI4JPt{Hxy@fHX`4jg_CDzqGW(t>gm^nlPKK^|ElFHOoY=HlTh!E$8Y!dKD%)E;#`) zU8gmJR4?UBBCSH%!vUkI5+lJHUs-&xG!a=mV4!^Jg7pPo?N;G6BNe(M|GbeaF}>PC zAD;#s`Ro3a{8=E4NzYfZ8m7C$O2z<4B*Kq$hob4qRDJKndomaB#b%eJ(PT zq-pp~)NLGcWtOJE+Cnn9klZI-tFX059$0cUxCA%ohqCxhrLVEWG?G$3Xkt1;Ojhv*xX`_|f5Bp|o>fm>tPY(>|JbesTGxnh1Ot zUnbY1yD#sX@zr&^m>?eTsFDygOg+)Z_VWYFJIR^F~mQ%EVEU`hQg?YMmBW|vS zwoR&#id0R)izPh{v@T4&D`~{jE(TwvgIun+t0vbgAG4>{MJTi7$Y9C>kV*5smrNdK82Q3!UZu?KZ21*oIe$~z)HcGXJzBKN#l|&EpT})INUAL z1ATQoYokH&J*B`BYr(I$I6t{tmNsMe=FUGDkhGX`J2_%MqP;;NjNW2-?WyH8_zO>V z>Z2Eyp9n_PMMSbOP=DHpoOpWVzA+YjFTZpPb#1VpGTyoK7{;I4&vsQoGi+vNcB~1i zB~%zOYA)|klBFe{Ey#ePAFY<>g_Eg42Hh50)5EX(s;BLa`B@8ObNw8Z4+M-xt>5R5 zhJ6i|yCzY`;>m=@tc%pDX9iw$2l{#YZ5#a;eCKZnDpM;1*(^4~v%Gmg2W`2YI~{BHl7fBeUdOvI-_ zK$lX`PN0z-!6DAu!@BGJ>A{Um-hJeLHCA*oxoYf*rd(!3?)8yV2@!pqxf@Xhh?-$a zIs7oPI2WOMHP6%2rwJ}eZkpBXKN+heDRJk9j8=2bd)8&dVsc|i_SkCnIUb$FL#O*W zk6XouaTq#3eo;8$JU$^24`lD3W2PBKDv4D>(fJXqlrrLV_0}ejw$=}!JrUiDzdM=6 za%^8_)L0HpGp@4o&rmGFx!w^YXQUHxCuMJa==fkIND`ya)5i-gc5W2ETg$M(p77rR zF7aU2(O&r;5(haBqGj~EpAag3R{bWgCz+-y1UH`brepR z-6zLXnd3xS#p%6r$hlbCIRi{xUdTF1#<(Aoj|3nxvCmj3IG&z~w`7M=vyO1syO8K! z!?1)<)7tQLciwQcP|qCOz5wR}?n}l+X8E>{!Zi1a7T*go!>v!Wn)Sc5jtcT^u-jp= zUW}ko63@~?R6=LZjJPqhLis=W%lXjOWW3)DR{TSg7a2hB&;EjyZjLXd?rYPLaKS&Q z8H)vpXc?CU8PTi|BnRO~m%<1my+OwJm@CWT-Bw$)GpV4cf?984^P@E>VJh)FMMX}H zhE0ELM_vIB`9<8ql^`8Y7c{I;G>>pc4zj)S9)~x-hWF1WNy=Z`jf;WXr~lklg@6x0 z4451lDMp+>xom`8D}Etk9>7G*L`;lBf-^GvuYILEFpL-tM-xXgf^1m|i*v_tG~|?H zLQW|gv!oDvI2?iyplXLI)V`+i(3BIaP@SfUe~)pUk(h@SomGkvoj8m*ON$^EJ$G3! z_e4>(_^Kv8Wq+1~?!(~Pou)E=f5EAV(88zNuLIxiH!Iy+OCM}MxMF_C*Kki7tuCc* zHH08HjYf5o@5b`1|6*mzb*(M5dl1hiFZ*l+FIGB520bl@&v`fFtL z9VBg!t`QpE=&wjF*i`S#WPhY0#(<7S7x6q43FkZd8H$6Thss1C9T-1w|Ac?=%HbAr zzuv*`)3SOSdKsx~%oLdA?2ZbE4^u5TCv^a zhJ`MfQYU+!S4n!NNp z$hNe&@}tr~De!^RWxVstwm$veF{CN%dc$E?tJqp9eihC~)&RpBzWysLf@bF(47roZ zAyzQxMyMRBOrlyYgoyq2?ColM}}! zdv%rfhU`=S$cV`yp@nK~wo%#m(c-_%w9bXB*vwk|{{r}qEV_rBnnDW+=nkObiryO^ z>EJV8F}ar^7RZ`{6aMFq;l5Fud)9Q{;8T-U19uDL3a*Ztz=sk^+MmP0-WIjJDHJ>A zgnZ}JAk1`uTws7x*J~+iei#;_f}no{qpK2blWpMRYH z66tpaK2{_>%;aUu|$dz#BjyKgu>ppzz_eAkS1+;MD!-$c**ge&UX@MLf&zv}v zBC;?DY*wQZafr4>H1Xyb;kv+zUrUsE8CmoEPq>F|*Q?|9Z6Q3`{dIf)2YJaFnqf)( zmr!`~J5H&bc1fr6TBbI5eDPKJ@oa?dXn;571`t@h}!#`RH8BBXq3oc<)$)3;7Ksjp;`mArc17Y^AzXra*imCD+b^LJH;5dIEz z9-RVPkyR!_qL&LJk{_{>5qtQNwv$BxEr&eRBTLFmP@n^iZRbAW zP596Q1}vG!Ngw_mcJJNCFuy|J;X19iC`>NKL8JLyBUlrMadA>FMRZ29mMb+iuYpEM z3>`02CQ^|7UGNJ}W+*Fqnks)Pc7F%86Ff{*Hb8}=W{_Pa#+FxC*;!c3S|5!M4)TbJ ziGhX1bFS(<4rq};d{;N+g7R>PjP&&NEq9gxYj_|QQj(Lw^Mmrn4fCB&J^RS)AwuVU z0dpDh^ElE2^yF0Azkk{v+#qUw*XgzNhUF?FZIA7jN4w!o9XZS5Cx!*x3hao{ZvFPd zwPe-(dEeW*dNe>_0e@IqQ{xRz9a2(mQBkpUx6{(nz%wCT*qM@qgoK#*D}$1nrlz#d zZ!KBbFrY|K+AUDk&84j`IX!A^JCtuK54=n0w z;BPHA6%`e2m=Sx*l`^;jn67zSka6iaM7Z>HcK!+ad`8`Oz5UjZN*(nlG!IOUzXudv zR|`nvf3C@g8AvDNqsiecr8-(CU|@xMbdWRP5%WCzq-wpQ!T4tcTbTuJiyjel^(b{Y zAgVw|&9{N41z<1YphGIHkOQ`a9~45M75Zp@>YUTv(a`~Wj7IjI)-rl+acD#iK-mEv zz?-TtgACJmQz&X_1LbF_bN3@a_^34K^-u8XDBV1a+Jn3<8#ph3msC|%(K-eP%RyrK zII=ie+5X`^{?1(7`hL_+jL>uok5R)6@if3W7hVX)fAgl&UF_Di`Dh_JIMu7opi)sA`l zro|#XNgmB|Yk-ohA=gum>buEhhTc*6K?K6tl1@gxT!y9n{&k$2)YBze7V!|gGVk3;C~On$?!;(TvcZq1qXo| z2ol^C>;r!cO#mGLnsric237z@2{1yNPzkyO&#R*w0Q=fCRI$_1cryL1;-La7`Gm{O zv+Y7Tbbim4Ro#5I5wk3>Z?{&rZl=dHHjmU@5;7(5F~1RjAYjx`W0)`;-&e->ayoCK$tOO1s$)6&idkzqnp&bQAsq znqJ5r4m_Ik8*loz#q!Qchtor?Dh1~FeQ+eTwWXR9&*vH|)Y={oKCGVpctYMSpDKRk zmVjVeldo1LM^xU*YjA0YJ#pq~_b!CGCw7$~d zi=ta(=pCS7Y4+WChaH3cG#3^)Aft!}rD>^zPna6M?C%KrS& zo?t{L2eC8P3?mi`oVO>UMzh+v_(pHAn!K={oxSq?sUqTUV_{&CJKH1pTC3{t;f4BYQXNQx1pvC7X_r0EIbQvqGr43l+n!$M)+YTpt1qsE(&lTafBgWe(D$$-T&VKweh6* zU3qOYEcXB`4ckLUb5(vOfeEasCqYwy(VWzh9?b~0d+~k3QvW*|S7A<1X=KuT& zmM)hdk-0`r4)K|m0F^P@J zHpwDx?%SZ>|K?sDBa164*ZA;p2u6^Oq1&S42SoHD7&4bV_*duu=9AlA=QXb*!8^Zw zNV~@G;WzWf)KqFYpGQkg-1oMpp7a5SA$O_nm(;Yh6HwYiP!XIUfLDY60?-R^%{Hg& zK%ajC<6bb%+B6t{0C}R4d53M z(C^b7bnk&UY^xQM0Q)nwBug5 zt)xltY3`@DNTlCL$t6qp9}J4W8zt`ay?D)LV4%+WMDjb{kYOWXNds(F!;mqYeR)=R zqUkI12J*dOdm=#9o-hqpQGW$JR5AZL2*3f3r-hP5PfrLASU6I1HC_ZBFDJuEn36S2geU*U#c!VHEazIXLd#; z0SWs%9Dpsv6M18u^1I&NtA;a?i^|`*8bhM2I|}ac%Gb{mB$0v4xnI8?sBRdT$c2)8 z^)%pdzfYbf(+9@{d<1+)a+AKT6Ep)cZ?Y0%-_|X|NK53we2MWx z%yH7-sL!_Yby6Dr8y zgPg?*ar)iML0r_7O9~KHlO%O;;1alk;o;#SR$WmPhkY^T@Ib*(lTJV>ECq}X07LtG z9y|nrQk6leK6#ggaKj{jbq)8!_;jF`P`)#poA`AobwuD-BPbnTD6`7oTz6u@l$#uEZ;gN93 zpD}3ns;iz(&Hy!_!RJp$^MRFKl;dkY5Deq$O1D>`{%FA78z|f3<%DmjoNWkPF5Itw zOqObU;aoOGBj|dzbM#X?L^b5`x8=(-J&rGs1`+mY>D)|Ct`=GR(H%p-@z;^hAu|c- z_+hisxL1=M#A|b6YB)Z))r`@F4g3A;C7}~ky%>Xhg?j-ODu#`pB1o9I#sX1O4u5|t z9OQp0@t}odlPk#gw&3O_Wtcm`ltoZ8nn56WJN)RfE?+uca<1VnC@{|4L4|lQE4UTm zyHRWyMGO4qx<0K>o1?#%f$1iyUg$=$TpCn7+)vD(fdM?m*Qlk9oEaft%=?&M79BUj z-VYDC{>lqD&M`uPQU|@}c@=o+_EIs2<6pn^wr@^RNvTwL#l*HIt?0nZ;P@#QT1vcIK}^?#-|p-L7<0*>5tZcy_MakuZ8jD zF8bEXHs@l^yRGxl5x0t32hl=JO6;K4vZ<>}EoW)K=KX!^eC@Mo&AG_gm^G~T_Y)>z zSCD&>mIAI(EErI3{}PL7%U*}#4#@#M<#>6Ary1b7CY;(+JwMZ7uJf(3@L)uNhQ$>G zj(V76Nxfy)2O3I?M{5NjKoFLYX!`HK+d52<@a#Mhg(j?U6ab@Fu$aZL5DYDZidj3n zew|wWnUn>-=idl^1=@cG1sXybxV<yYMX42SHYiJKN!vLI%_viNObo- z?C!g8-fy3Tg-czh7_w)vHMY93VftBu;!gRXjd;l}Mjd2Sy@ZexDj`SW;k`H(kel2z zdXJJ@0J07ZUg_Zc*y%RK7@vTC-q3{={ramKo$|zoKY#D>?(2(6M7&^IJY(`0AF%Gg zdiv*G7rDzgZ|%wD%YTHw`qs~#x)cn=Di7<_*h>l|GwHWKxp6b=F+AdXJtjsZb3e(a z{B*%9Xoo@? z+&N7WT!b@04`CP4PdX#8U=dXPs&&miT^SUB<)X{WRxE?9ViU*ENqZQ_++{KUN7|OTI%& zD*5PV%C&0}2WRZPlx%4-^|utzwQRl73Xc#(4r@^;d^BuP4QFt$@#NS!-+Z^BO=7BM zbLClQ3X67fhh*ks%mXe%XRY)&-55D;wSH|`u*oYbNf)$%MGQ;&4p;(!)B+-PPXN#b zCn~6dZ?m!4gEj{w>%I~ySYyJ`%A9JsImmr{a-9i+@LN9x$jp?I)1{Bt?b{kyyvy=l%VPj(;2q-c4*C?CsUM zBH5T9{(iON9Y0peCn}5`0}HHx$u{$okFOouaX(w%@%Qelmk#Og6ZNi;&2WDIPMLj3 zIK`Ww-jmK@*z#iS#RpL~{A1JhKO@02TQ4uiKVKDNztM|C3)FB3(ckkwFV_uqGlhM_ zw50JDj90NDX5iquizD$cF?(%Uj1VbF*WHKo9pSfypw0v16pIEoa|nQfE(BqamJXRH zGxn7z?i1U2k!pe?XoUxF5?{@JvF7!M+ok+*Ss{OWmp}h?XmG#1_u^#Q@4lbYMC1nR zB+iJ=(U-AR<^#{CG?xxJ{EeMO1lJ#?KR}_kZc6#??vV2N8*C=P?g>m!=fQbh&*DCi ze93&4yzOAV+PAPE4I!XVX&5M(SGs_f3I#c&r>6%iTU#^rcslK{Q83!VG65$Hz)=SK z`&HG|x5kQf%||WUJN06`}yFjQ!5a$ec;C zeC;noZ^}fl5drO!qlH63|t* z!=UblgiJ5g{D7(rQo*`FZ3N2{R5qyOE-??Q<=jW>QPfWmvy^c)e?oG{iRrm6tD6E` zBI&|QpHic*Iy^jUA_OxvlQ~P-oGzD=_w?vr3z7|u&p4m5Zj|&dy1NOw7A37~R~Qr~ zu{yv|KOh0*3dU8|UGO}b2S{H5;LhvUy=Tvyiihs$?4yc^`!B45* zFTmH!D@{7&v6t5_fEA@dkE@#=_H%g-o|1#)ZWGY()}LuURF3`B z*Ij9kkHdM|uF|uS_Ce^cBu97nmBWDFUdG?QgietcjJ`CO^1lM&z}KrtUr(P7o-CS0LDval!E<10T0#=Qf{+i z@x|6#6L*ZCl$JW*{$}~Ab4twmC!&smhArV9M^M(G5K?|xq@ zGn6QBor)dIAT-s>tsf0T(QpS+3;`&95E@&Skwmmf#FnZY&8b$&x-;mg(-Ja$);ZyM=$b879v{O2?zW0Y-}H?ru#ZT6%iqFc#}HRs@!R`MVWk`+6MR2aj$i(6p9`6ls$C zKIk8~Rn#Cwo)pi_YxIHdVmKz?T`4-dbHKYcAM{a=C=0758dYRqigisgWai(RLm*g- z$lPZTgpJFz7wfi0?RK3eHq=C8i`2ohzt`4|L4=9On3$MAe?P9g4i#fP92np?;b3oX zkafi!M*r7^Hejv^sRwti^!6obQ!$oMfOjhJDRykyB3M8 z8&((?6vlajb6G`21^8-K_0YkiJP+%HT;WT9ziFM5zK;)#sTOh*)lEYL_`%0sXzck+ z>(yF~sB8d=-2o%gXJ4r1{@()?$p9b0!~;}cLs?dmzGpuGFaTxiC4jc@2)EeLd&xz7 z^-R*?5-80}%9B3Rv8@x3lPSF~K~QsPN$PrhtQhnBf!j^(1tgT?Q(k3-;Pn*rDC0Q1 z7!3h$_hVNSx$LKx|B4393aVXznK42E5Fw=E@960%1c~4L{5&K^f9dU|0<9@HK5)B$ z+cY>MQjwD8ar8mf2mqVZ0REGZ{Mgz$2gjGx!4+sY!K!Pjl#`52%^#pKGzLaS6g&#Y zEmV^z5EcBKAVV&1+zC`7{f{)M%qKg?)W`(si|Dx%Q_G&mI?>|X*`9^D%K8W|+UVdu z43-ZhLg?|b0+AEk;@g@#%%94+BZ_jtmH06%L&O8rn$Q=61+>rl7$-mfdwAo3K_mPy zV8V5pHcAJpR7#37a)ii(UbhY@t-L;3{hT2An^o`YCw>kAr^Xg53Vo@=K*y- zKswMzTHL<`e_=oi5vwXA@Y=C6G@JxL`rpY(EVXb+QIR7oYl?|WnEC)AFtUhR$#{6{ zaBBb$fS%UQj(MJZ#VBGS%$#=WKvFv%)@{_Ade3LIRh>>$zkxyK9j|5iu_J_ zt6Lf3?pTPN3^#w4O_u^?Oh+yHQUrSR1FUxxRIE`QiLHiyUVr0yS-M@KSebRN38x>$ zwk&(1Fe1tI#}PDWi(tP138Dos+ll;ISwW?5!gzxf{{SFt@X7S+2g08{V`XEDr4iM* zapMjT4}_AByNHm<0d@kS?0)I8dz|OsgMu1{{PE)>6(BG(Al8-m4L&bp(8YoUZ=mE_ zrLMJ`^5m&$fxJCi!A&3X{%Jb)^TE9@xfO}+_+c*$djbhJHcTjmRHDDARozS4TWgBF z>52aC zho`eQ?g@*9%({OqOKzDA8oBwdb$(vmWZ+A@`jgY|dJIR(_s1hpJ3sJB4$#^HI0Xl5 zp`x?X>HYs9`JXZL#{@6!DnKdx{QRyU`6DSn=l}slO1cX*EDD^Z__nmR)(8)ka=c#F zUH|{31z^K{VpV4Xn;}ZLcXk#e*9+d1utlPrGIuwijYE}3(4-%-Cax})$BiP^bwXTiNA8p6;2{*8=`Y#fxGac370Ca&BR*rECy*sSFctA%7CSaP_` zH{0*cHx=4ml6cF8ZSM3yyzcM6@8e&RUBRhu3qPnQtY2=h-FH2K~Xpl5WG8qxZMrX|P0uM*KI zn5ugdokI%IVi){jyJ%+>Er7j_k4ai%G4w_?^0cO>*gLr5Npn8Qa?we#Wuu9#xvEC~ zh(GK}EiF=jJcH7-$Tbd1%X5f*2gwH{$#Pe=gG~yw^q}1a=I3B@s%9mS+etb7%^O)x z0vw$8M}Fn8X;hF}IX#@zS@x{UnJIv;F!|E=TqXAH-eki?y9dIi9KGt^glkQf_?fc{ zA)cF$S?!G!S4I}!KGP?2?fg0&ypP5tHo2P00zYkLD&sBf5Isf`N`&d-qi?w)@ zk(w#fza}ystXP!RiV?wdYhwLKr8`!@7|=lYoN=QuFmOn5K%1Z_C#SpmM1;|OL#E#3GeRYY|WkXrTkWhBVL+Q`s`>+I)Naw-%j7nwvR;GwaCT3 zjX@_L#yogd%R7`|aTl@DyN4DYKyz}YG-^$ZWVAmd#HroI_52#}A1x9H(XzG6({+>m ze1DJ)eCc;d7I_ksDX~_p!GNymPiJfV#?GFC;-x+>r~b`qHvjjktIBd(Foc{+BL15A zAIIew<4mKai$hy-C5zT=r2~@^5}aW)EVljuHg3vNms|a;Z0ln>*Y223Ua;mlzkk7_ zou9)I^~Dx&tVuXZP{;Caa3K+`7{C1h74qlcAb7iRl{UFs!oP8NV}z%<>Gzz*BdqQ# ziU}$T)K=@}q}Yo~ANOcSJnTNRD%E}?MHuj@HV@njYpFQW4mXD1?h^$Zcqr&`qmtg#zWy4PeZp7?rDSrTr1{dL;uCBz(TA(W-Q_LL7AKJv06LHJe z9=VG-`nn zJ{0Qd5wqh!#TfoI_@)HvYkH*kgRW3Z&GYul)p6*|d@eu!(1cI;AEE01x!TkJDW(0N zLfH8cT$v*pHd7-{>>3O1Q_;f0=L~sAzK(e)%@aWlCBF?D+0`%vqKh)A(#T>;^GBZ% z%k>JtOAy>`dn`W`Ygn+HEl7oaJHfm_EsYXOrTLL{Ezw%O-z!(>kXoRb{O&3iw;`(Oz?JZ8j z0i%w$_<4m4TI-5oS`2pF#E2E5l54h`p3ndrYwn<4#hO;h5jMdDWJLU)XQ1wi=K1zT zlZuB)0?$P5fEGE^PZ2~Aty{~p!ru}viT_YC%`ZN|Cgl?2`hxnK*rI8)&jp;{9Avxe ze(CGB)e)uRfGG{_9nBd-9ZO7qMImw@wU$4{Tc0cc^{_ehwEpZJgBAcCK?=m?s+X?k zGPb7J{zdmG92EMMf zbXB@o`yCIiK36gm+NRECkgvQMj|`@qzZ_;>(z@NwR=;mT(_VDD_4Ie+ zwxnOEKf>tKbDMZ820BxaBEfLFKUfw+`s<~Hg!mB#=z{xWtK_kZ5hzXoqUQGb2kdms(wxkLIo47h_9uk{tH?mCCTs0O=(IFsUvTunTC` zjZ8`(P2$msh@ScTZu_l?0mZ9j_S_gq`Ex0lC_i)5wBC`@nNX&O$kIn17YkSu>7qN$nX<|JhtY5dJf70Jrk4l&CN32WV>U)n^052 zi)+UnzBOO^`hd$`3TT54%6D3;iqhn$MJ|K2cLF_BM@xnof7itCYg!6Lha$_>QNqM$ z7q^fI>AgNIydiu{4bMD}1d?YYnXy_9)+@^c+42nra_Lomzwk-qMB`bMlkAL>Ze$XV zy+-x$RYm~)f)jv2;eSPxdlBsIl3jU|qkb2|lRfE^>ycmj%*@5}7{m2~elIqT&}rWJ zQ8D>s{8NQxvdD#T5SOXbMvv*yC4I1q@PX69qQeXLkaT}#I%7b050n?k1lAVX+iyE! zhJr71Z|^+>aD6_D-~;_0)~^DTfa$|XGq9B?`v6zR(_jfzo@cb|A387)jwtc_d^J-@ z5ev7mg1x$+36}C5AJ)DP$zopS-|O^ z93=jF{Kw(+S2*KgohhB<+SRoFs+a%x!av{lf=_j^51$HpR~TD^FK8uweN3pujc*MH zsEa=)B`@54nL3D#P-?fmcJ^}b6AsB{51KM}x`S=>?&kR;U5)sf%DyQ>PUH*n<|I{P z$SduiAL-aSZx$|S4Zn7dmQwNL4arS?UL_yPc>`Kf^x2pXBo|RMvS?Nvw(q**@k3## z7xHm+Subn1##CH8S(LIUP`%fR?DcCs!^4%}*Z(B+NY?!-bNKPwu-tVGx2R9E8UGfi zq&#^h>mm_pGq&8a-IMB1H}hOXuC1P@+Qyt7-B1;V*GQj=dMmh~<1j=ggsvG{D8D3! zdE<}F2bv|D(r4mm3?OV3gQw6x3#X}CES^*jmxRM}rL$I_@D3lU>K~YMc9(jz?+D2u z<4&%C@IsL-BO`Nn>lm;JkbVsH_j^Bl=mrLUcZqJxKe*fd8Jvq?*b&~auebMALc-Xn zsGnlnNv4_bw82RoUG?YBAL#dzE_+=>c7WrCC(xw)F!u!=%4Lh-6b2B17(#Vr3^+i? zfilDkM2PBXkd^?6000Y30g4I3%JW|FCU4Zr0w~uo-)r_0;##pTOZ$A=)p!(p5=kM# zlz{V@WhO65_H=ENlwonZM>v?&k>15co+#r{LKOH+0O1c{{okXb9mwcvZVo)w9L^B; z2Kx-~PXWOlfXtBlk#=M+M18b#d3m@kc_%6_82KmIJjwS=?;b-Jv7GWfbjI9B@--bS ztgFQPVlK}$Gu zxYZM>h2h|Tvs>VpVV*P<{LtPZj?bc{Xv7ivWl)WqvrsAs!0@uD=%f|@zvE+Hr~`mn z5asazyd6BkIdM?Ne8ER^hI^O|wUWwlAf}8*1Hj0om0L8YDPD{+Rpxna(44dSSBptKGL~c5%KK8`Fasa1iM^jdAoWgXRckNr@U%0Sa1=$;~19*!n>j(Z;t^duLlu8a>czOBE=GGbEh`%8ti!FC)7&9u5 zZ0>$olBXuH(7U9o%`C}q{ZCqj`;F)0oq7i|4T}4njY?V-#Y;QeX6|PwWq%-tZ{l;p z>ox2%IF3(wvUZM+WF|oc;dWg~irZ?pNJ>f~CTAqf5SR?bCBQ;(wSzO%%*+farsn;zYF^hW-*x*ZWu`gn)k zjbD89hDImjo!zyrRgS^;5hDW!j@D^*gmLG6Z_co&dFYp+GctWJp_eD>n%H$oq#rPt zu1S7gc9dJm2`cG#F3dGk_(dtv)YA81zPMu0eh}W}d{39q$YIS4F=m&QtwLQm90s`O zIuEYhw3ax21})o_!18#<10aMhUlfxdx(s%OtGK`A?x zgz@vLs;XE+YwN5=Gsh#jb=PiUx^Pd`XtS+VW6#S%__P`5?*(oe|oT~ID`$4P7Er%9owoF$@*q~*O`Ik z44#I2PWkKsCuScCPMaI$_ZptiW0dF=%PQ3D%Xqh4cflY;3seI)G1i$gfQ7CfU0p3* zyWV{RrWIf|!D&&*%iW5F-V^P@s_3NdQM%4t3BNy^(>>2%D zb4u-Q;*G$4ZD@&9NFs-gv>U-Rqc;8N9d_nYHM`6*P+xar&s@563HrA-OQF?Wd-ob^ zXjF%GO)t^xb(3Br;q(Q^>JWGehGJs7R`C%jq_g#M42wtyf#ohOE{6a5jBXA~a%`;r z_L&Wola87mBc34< zQi)SyISnRK3p&3G%NwRDt_iR>ii_!t?2)q?=1ofY)0=+P!FFl$tl`>y@?Bf&&an18 z+*$U5x#Q!Qu}`g7DcfI1`$0^&)HYql(F8k*%bTjE}hhu<)X*iML& z9LrBU5pz7$a1B_(k2gr-svn&~)XI+HfPbUva&KO!DyU>`TG$)1cW?3`%0^k zpj8K%{QHkyxZ;O?JZ}gOk=bHr^;8 zkN@1&_m4^(hUr#iw(gbK#mG2V{P4#Qhb2`U#UJR7c>e;_uF1kxmz$Doufn&`f3|@` zGvet`@$r6%oC;G5@wanZr;-*|T}bvxrXgpp4nA{x>o@QHE|5oa1bbbdO}!q?m(8Cq zPFIjFWd}}lWbf51GT_TE4iVad-Pq5e`3^{$Jg@Ams)G;p10Fu)Np>mo#HF8#+e%=` z%4vodw+G2Xm+qZtBE1-U0)V?HQgvf=>CSoLUkEJ8Yg5u&ax_x9d!KV_(>m?_yr~0t% zuz5eb|6Gru;gX`kw!-kKcVSvFji+qQWpmE{syyBwBXUbOYHs4Bx9$kb>e*0fp1vxp zRv*!II?qxZQ-fiZ z&z|`h-5_Ji_ABsFCVZT0Gq%6JvYb*fU(_+hTxe?a=-X?mGf&+bu21_<=^q!-6J@dL z>28uJsxS>d)BA+cDz@$QvOZ12;b|N3o=swT9kboLH`cK2J$Fo^`}IxHdI!$W<%jeC zteU-2sp>4?Dx$Em(J8VWzn>wW{o%U;VGe^EZH_fe$~3e$!8S40AqLS{hQs<2C5cF> zxh9j61Ov2OLP$`sZT?joI648>`_2WX4a~J3T54ruFsyj!9@{G@)96}+;ImiHE!X=2 zQ{4iYou$r`{PV%#c!EFxWvW*~A~zyAR)F+}C0Q|xQ2NBg7=jqT`W@QFx}#6ABAkCa zLQ(FM{rjumwxK&kj=c7zQDe)S!MJqVi9pLqbx1rTCw%=>E43Oz+veyp@Vum zRKq^|dA#}=;B-Bw_P1O zt2dVVTNL$ACV5co9UZeBa_jAXJ+I@v{EdeBNw$8TuhAAUOH}v>vZmnV;i!{7yG6j9-9I*hthi zPS z?RAV`$S%^B@Mo<+ZeOTsn8R847uh@aZePjYME~FCZNz--}(!k(^%-M6S@W0ckS=+>>QU` zUkU0>h)foYX@nIqsZNw+h}l9YNis<(01^Z{g83E}7m+^A=dq4@vU$9@1u@Mo+ST(k zIg9CB{QPQOFJ8~HT>5qH#>*3tQwqWABq|>*!koEVHX&__zDIBwm1qR(diNuP3(5oU z!#9Y|vv+bGTfUhh%t__r<1Zv8OZ!*64}KevGEw?qLI^fypC@t|QK6iIYhbx9XgbtU zo5|y%Gp(u3cr?1a?Ld4uFvVbjBZ1dt_dZ4cq0>q$rMfNhzLqIpmRIkd8Xb$fp{lEGbCFN@DGfEj z8z4Ose&O;b$xN5R{e*d;L5X_Vqfz$`mrjg2WUS8*w_J$Ze~>^^g<^M4#?i}A3Qfib zhlLFl@JdSl#eqq7oX^auEWPyF7hnM5A9HgttS9n=y$=!&HJQ%rLuqw0FtK^N_Yb_y zYe6u6PYk%)_k$1Vt4mWQ@bpTJerdqZ>zJ(7m19zsVHL-_&?6SUh2wO2w^qBRY13cb zNEJ$o_7_qGgK~Qv6T9t)x(t*;ztqBI3jrqt z_;L(KzZiLw7)JJto$qe2c;vRbtRU1?Q#?%DS4V92qz1v6cIvQOdK~3^kt=Ow z#@Ek}<*0!mGyrIPuRry4fl2_X0C*^|LSF^!aL;PYBLwt8gb8s*JG6ojR7{PKQxJ!;sG0PB;3M zqd7nDobXLw#4*qXQcs!|A^}rqbEHt%p>6kjor_i;f}L}iU-VvIL+zizT3foTYqjh* zP@-LDvwuPX%1`rDJ?g>?86ABWsnqe6JA;klmdo_YqvAI%tZg6eyI*T^)l*w3ch9Z6 z-8nmUQYgzjXR_>Sr2#^}!Ih@@92~2FQRDpiA<8dwt=v4te=s}2 z;r5j7kfn&9d2%Shxhdt!ohCLPB9*f|dd^Z~V$(Oeg&h7^;qD5z+BT*Bti{l79$L!L zkw0m?vg;=bU+kRJ)6@)kScv<~mk1xMsB?!f5(LCIZ|{$M47)AP4y@xub=H4nFvZW$ zZwqZMge#D|k-eqJc6=B>!^?~#CmY6mdl5gP07?mQU0E47Wf}jM79cB&Y)>~(`4=zv z$z59@a94%mMNQ2Xt@|FY?P;^x53Fh`s0-t$5)+u|g(hXPj{RL0oE>ctvHF;5lAcg=)knG(&ki@x+JT)wqyF+bG1tt0Qr)sJXd0Xz^V}WJOtuwb({$ zCb?%jsOc)d8Na);!Y36YQhfF6mwof=F=EcOO9L;MqTGWn^&imbdoDS(nzOgTrIfU#dWlZ)4`k0$_|Y-4m^fzfIYxUd(F3f^b1VV6B4p|t|gN9gmE+f z1C}M_clV^rWj@hL%Rft@oBvE<*v;%QqMJ_3^)j?O?9!k#=aoA$yY(6!6TsiBQDa~D zuyN;EXkb)TvxQ|4l#8t>jv468y+GRE&{t(&EID!d^j-qGkj--!K zzy;o0W{;=Ys%;wuh(}FDMO4G{(mhGlWu~_-UN+0opF#K*G7qheFsl?=U@&I*@O4}5 zcACc*zxMjk+NTOXkR039mKbu6Z5yHYa7A`0%BafriPS<@Du-ofS@!6ed5S5a+%tps zzQ>$%&l>%_Z9w$q*sni)EZoUd&ssN})}iFp*w*R@$KtujB8mXL0gtv`ys(JhHW zj6C_T_cBlO*>HcxkMd#?#+Kva$47!a#_;e3qwH=yUR< zb9#bqYAe|+bISh*oE1C!ex?1~_Egz6{lsXDB&{BWobaiw6l#>hdRq^Ey?Kd^j&qyW z8Sdcv!~Hx&p;F@Oj6fH!0xGZrRd4q^V#DKo_kI|OWf$9z4^zvWJ$DW?u?MtZ@*#nl zn-DmMbX&n>L0;Yo#1?6AGhxE(v$-G;YuE`UmQN=IokrS#*GSa$442~kE^7##y(ZFA z0j3S!{3PN2NH2BQqtO*0B$U-yrWsAn6a>2NNu+rrE*DSCYEGp$A9h3=s>y8<(?c~3 zPr=Q%H{VpnCAGZcOQ!%IpJSiJ^bI$mfdUbshdR%6o*V4^)uH00D{!1UnfYZ%MHchu zs0+b4-=eTR!IFVOjyKA>ji2{VPhXR(%-9Q!hX(UomCl_LWDQi?42$P4Cn+&gcX>l_ z{Z*OsHi^i^gkUrEr`aISK!9OiAJk5takXg99@hrtXXK|R7anx+c$t-avtpFjQbz4h0b2ncM zrpKpm{2HMFai@Rr0;sn_UFZ^PeNx_#~ zb8J;(q}cbio0aym39m95N39PaE)0V?3 z(dj&uUE1EaINifSK-kt&=6a)+xorc@*7AQA#e4U9L6+ct^Cm1iw-dmgj(h!aD>=+3g$3y?*-*GOKF^=Q^rK3f+5Xtut-Hm3(KS{0n$tha zC$5`@JKBeM&U(F_?KD~4x+{x$I~7qFXnv%o@uT}L(O(XULX3rX=oAv#o~b->7B|5H z)$mi`k#zWn{*T2Sx9Tvtz8yZg7zbY=ogmIdosbXZif{nAX#EXJc6LVw`H(nmXl2nB z9u`31{cAWX+Vj>eXd&toAY><(>(Wkqa+?v1Bx(!4~onG8&&r=c- zu}5{dnP4Rm5)$ITXbSZmnL8OhaLm8q=mQq%o*JRBe$?g`(G-gGeMk{$`9=48T6jSYH?-VFZ`vhYV zke9m!Be>e``!czazKq;Pf}CWlsHo_1^JadKe=zVT@e#-6xyvQ}RuKNeDRbxnAI`QV z*i>$)24Oc;@Q|%tUPKjm7^A%AOn@xygZuZ3VTInc}*{}6V;?SWK z^!d=SS}WPv%>g0PuZR6{?9uX@iqNJUM0a#|N4`{3zWD3%1O4R{aTbc#T6V;Ze~OKsDlqszkc;`K4jeC! z;kw=wjUxt4sC8ln#E6(y6)-5cubStKbX3%16w**NBQtfa`8W+}Q18V>@J5(chCkfU zjG+L;QRw3l64l_Krmo)H(6E7kS`T{)B5n!}iA)_*l>QEM03L+KXf@KP%R%n zu9=`g>ua|()rC_7@$<4DFm;ZDue-3!&xo(f&KKpfsiXs_z9;eVP4A!Gz#uCIDUoO7 z7*qCX+zI0&cCqI!CZ)G5k57z^R z{lj_z!sf8gP>;_HRIOvjCXnO?_hf$ey<0H{h4xiYfgFnSq9}85D@V5rYD30z>RNap82!MH#j8Za-qT7 zH*bbQT21l3Ukd#^-TrZ@xedj(2LXmcxH0^yl-ubrY&vht>*rs!C+ugSNXerAO z3_Lqu-#l(27^FUfn{P;VC?wIAflze+JzEw6i5nBduNVcR5) zLNA#2snD^f_4{qV1@0A>3;S&l$!=xlBv3ub-WSLo)Z@)e9#NaIbi| zw-k!Bias0OP%sd+nd-`2$s5mxDRV|;MZ~JaUN;Pqk#d-6T>GXq*g~3M-m5ugtC*d9 z*Mwu=l|!@12Y5+#MAnJJ-Z>w6sOiGibfY%E+u5OVUMM@h#o0ufskDb@!^7E-#awlj_6bi;RN&gza1m^oI<+60YrVG8$-= zJAEds$T{vd1RqP=oe%uFTy^23p26o42iCGT4T0QZoEjIR-Y(V&6Vi!cl;LR_`nCc` zHuiUVepSy{wYBmWCMzN~lXYI2g3~keW0$Q)w!J!;nao!w_txG(C{@ch6sD!VayGu# z3Ih~gUOUD(?K$zGv!JDUN1&@G3p)x_XMb%)l6K*)%VLXxM;8n&PrVx#EY~}uL-&S{ z;NB@;XPKl|W`69!#+Qlq*Nr-lPw1)dsN|<7_^i#a&M`){GhwF^7qyG-Yu}~UUY754)r5dcHz-Y_@VGXLuHfk^k<8o&RlqS>pT8uRH#rR}M?Un!^P`)}jP{G5f|Rw=cteMYE&R+*Jb z>vcE5rO|@DDGigy;obT7VHfpHUc<9ed}h~_?i}>j<4+-|d?=-_a+vq{TJ^}boCrHV z_xyB20PaiQaom4H&Zq@zg``V)TFQY3ll{;8oJg(Q4Eds~f4joEgw1dFqVzZ16cmI; zdZv<|rH`lgn@XvK5Sm|h0-n}*UOopPr&|3H3__xLzx+})h! z&*TXrXJ}!gd4d1e)jbFd94_|wrFNolXooRHm;>EGhP|TkgIsmD!do}CxwbIK(zoHP z#BPgj+t1f`qU?x}^B?Q3?xQ8unt{wdI@33mH&G~myE!#he3#Yc*mb4x=ZXgY8zEgv+wDyyEyT-@I>F&rTX#7Dvwx~s&n7VJ4X>mrzNi-tDk|X z1|J{3Z&k@QyTL0MU0-M!D3hI`k;xnBz2Mj#)f@VIjTiS)BkuLm`>L?WNH{=I{IlH9 zTc75MK-++L7or-6XrMxpv57m&1*UhXktn2*HBeOaD~^9(vJR6x5zIBNHk)a1<(-&_ ziHWks1^3;_Is>AgwDk5zm~#J_$%*{I>Cc*lii1r4l2p*Pd+tmbaS+&PEft*7L1;1% z+#gOke49IzSt6UfFgCY*gp}#+?da2Wlf{#e(;DwA!4;YmjDp+2d*;YO5a@pa2>JgCIxG z$mIU}@7ax$OV=Lc(s+p8kv(y(^J+TYxKAjg0KXGoK`iYNvScDKk<{**fLb%1QRhh8 zgd1nB_uo;@FTc;ZH}Y`??~oNGt{n3JTy?jGqIgY>t!2yelJN4;BS0%+|i4b46+|y18xd-n4gD-?3V!) z7ajKWsb;)6y}cLxUVj}Oh55+6e<&LlzZ$(7f1FhEP<{KUtv;0Mf~=ge)KC}_TGP)m zyf?G)PzU=25EAWFX)-5#Xu~#m$k!E?vsWGZ)mM;4UTtSOa=*Vv4_x_osRuA)NTi@` zuNx|AvPT^D0nC}6Mm>q7wtQV+zUW-X(6~bnHHT!WqsW9YN0cU_04ywqpZlT$9b~&v zj`r_caU>`$GTL!K;Zbm;|J}P7Xsl;Bg>Kq9Kkz?5SC@y_`avWxe4qIDO z)xD<+JHM==dq7tekcgq#O$5f#g*}>T>!20@;#G#7CM6~16u$ruk2^*o@r103dWP?S zvm^_3jQl?Vi@Jr(WCXJC349Z^g~1ST1uIahfv-vrwmoS)&mqFxsk8<|raoA8>02K< zZ{lFy3EfmyR@Hms)9ycpyw?_KD{pN0D> z_+<7ye~UWGqhnb!be$HT=-CXpQA#BE+bI(yhOIs{Z`#N74ulhOybq*(G-7BaD3o@z z3;vp-_|MqEG^J5p1w(;B& z?|99RBlM&uQ{05$O46W%3uA71IXeMuGThLB_QInV%M83TK-L8uFybShu<((|M@j7C zI(F>S+qVax#)N1UIX}?DkfYsNlJ3Q>rKH|vOXn_2(~Q3FyCL2#mX?mhz^QyCPhaF# zX+`;$xw+>wF@nD`OTVo>B|V>(Y<=Mo^Wsd;?X&57xT!UibtoA`m5xR|Y}V_e!|qaA zCKxuaL$+0C7DPIl8Mu>;ez%OBDKUZlc7;Sg{4HpCgX}As>Qr8$7bW>M{nvxzM>D zJm?8G01@6fke<-1NO>I5F4*C?MpvLWc=^m8_W?@v(5M0~mbu zq%q;6m455bd@Wg5i*xq-_X+Vc#+A29AVnpQbkA1S8%u+nrGaeV14w93kmj&h)8h@d z-nnlGj#s^a@8BG#q*Mn+zMis*V#7wlhm-cm1;BnFXD>#If%ZCmx-vJ{;p`J^UnqF` z-rpbyvL+Lh&=xY|EABXQ&fVS3Z5eC;=+O-0U%-sc#Rvy_4&ePjh}o-)$y$<&ZQ6kO z?gs?{m-fJ3rx*mm|51@~Z|_Y^ zomQz{5=COR@a7qStLz>vz6F$nx(W3TO zfFk)&Y{0_ngxxA!9%?Ik$J*=;2s@@n2yD0$Fi1g`r87x0QMrXWe^-vt5RrKvrd6kt zv9M}L?KW1#$SZTgP@WL=o^Y6zfh8MdvOgt-@@?LOr29o{hU6lWUd7M+#vcqwf4M&m z^+}kY!iBSE?@zAQ;5DthW%`vH@$vwp96oVxrls9=(rgRyp_yNpD(o{#u^j~DmM5T! z%h=TKJ{tGr$*$BpqOqorOF!2QYVY^}u933*JfdLi6DbIZce8a>p!FqU5H1D;|dUc|z zB|X6))VxLGar{Zk2TG~O#@+oQL@eBs)GTx|KFnu1lu1)arxuL4B@H(8sJPGM=RK?0 zNEj48zd9XUu2~yXP_ERlyRasU$w>1219DHXlU2@R5Fd(?oZpz2m)CyC4U&a;?=|<@ zRC-frco1}GAF_hA2?{AHJls?`h7D^K= zMn)*B1AP^UPm_yqrOMit`WQFFQZU-Gh!?X+-|nzfuCIAONgfHQFtbSg9&I*XY*0%n zpms%+Thp}2E@RBc?}bdTgsCzugJo8q86I6VCB_XVBok48ArlQlfXl21^wVxpIU#6a4x zmG-vc-uao>XU%1nBo%Wx(ahGn-S6jCB16U%89zgtvpoz5AgX(MdgNIF!dph3pT5gGiuno_m%tsncXz?Q0!lSZI^#;nrH9&L zi*)Wx3h{Mz);md1_xw(#Hm720_uwxE49aUm>D#&XFldP>?ItufaosKyagfet5gz+q zm9~gu_m{7M&IR<1XgrQbo%0XIdj-#9s8~@WAkh?7IM3kV(cO3hd-Qg->GZ?|eo-Iw|i_{uCuK0^d5mlFqhlUfjJSA&(@k(lQEVSHl=`z|Ldy%t>l8cL~! z(XA%!3eAR3^?F}K85R#5?HyO7?K_|^&%(+Y{rIu=TUE3Q*G@#?Fv4-wR*&pv=A zIzg;*y}Ov1UBE%1^kzHA&CkNu5=9X7f=Y5SI{dIjEl-x{^7Do+)s>EwLOQ3j%)B*| z&zeix?#X(09STKusk?XY=8Ir*OC*48_`w7;cn@3=X)3^%M+iLQrBtHews*O1mc4TT&E z5BqUd5436lEAlYTdVPAPM^yLODjO}fFJ>dVK9{XT>=&sjoixa!Is%{|A9U(k^^ zw)V)4$KxkZB~Wl4IB*sCZ2KYoSJqLDW@X6|7bYR@_(baq_Zd3<4{iN{@5K6r|AQ@B9OR>Ipej4GIFf7ed-zj?WbTii0rDfyKv zU%Ey}6#V>oef5W`@VJ?W0&O3oez-i3QQj5#!cE-CsPi6POsfB@k&*oo9)o12{JiO) z-%Y4A&CjK$dMn&2!BY4|nSBBd>?wpD~gM;2Vk4l+|Do(FMkKAaiyMCeY>W5}-in z+J!z*R70yjT=I_HqR?RJuCrbCrp}lGn;vGuYh{=vRqPmdGDPf*@2XTUzFq+Pwx5_n zlLU3NE_wXKxIO12#t2~4OY9HO466C(W2XN2!vyHj7C>>cfK^z`4l z*(Ps2ykBY1a$Aq(6!qWC>|RTr#bgd@c7sC8cQeI#(d-$SA8mwt(3g!5q&}|L6Kldg z+8=6AQ%|e$T!-qWq?1JGm0PRcEAA^zMYJjnHR{YeZTk#lsOk3m$fdhgJv1{OYtZU9;kPfVr&eGlaV?7tT0+KZQOJ4je<<6-A7AfO|O^3_R6dfm>cUz9?^^Hu?9k{@< z&nf2u)|_5uc3K*lI6R-Fe>qQwwA&H<1o}nTjx&LP?N~$iWNZKEdReiiSqOq*CI@kw-?+qMPXHaRLhxsK1fLdsI;HC)qHp zjzm05OuT};PRyJ6ZKMya{pGRLb-k;m18t87*ffmebz&3*VU2i2F|qfp>-tmYdgavC zzj$;Y?uof?_c(9}6o4xNCyierSNxgn@bq_Xr+kE}9Vq2LpCiasAlE1B0PwI;v0=X8 z?xg0JYURdMBKg(Cl{faW{p*vTc@pJhcF64D(k%EQI_$})TJV7TzRtCy!B_Iesxp4M zhZ_nNwQt&|bFZjpyDUNAA6|j!H0X3?6_uy%BfF7mg~kE%n23Ue*Zvk_1|J01RyQuS zI&~5`z(qYECK+Mpf-3=df;kGb!&`y40Kjn87p}B${q9kv!>_Tr7@=@mi{i+kiTv7) zXLsm?Jvd^XW&NT+C1Qa^$UbNOw9RDhzbEBs!Hp*+0jpkoBEWSbs~=|gKp$d$RQ06U z$4h-jb_PUm9YV>o2<5{zHsMDMTF^*`hB|<oy#1yY@Q= zKk`JY^@D3a$k-UG!fxEk+#7N{Ao3(Jp0V#5&Pz~f>B0+u#b8pVXI<7uHv9n?D#L&g z5c1N>cq>beT~Jb1rrWlS#s@7{L4ly~;st5xYUBt)W6BjYrm>k$B`9CQtO*84!oWC9Yl7G&4kJEfkr_DT$$b{dr+{xBw!*SOA=0BH=7YdCZk^%0Z z2rzy|FtD-3#K)f|3(Swc9D$#DF?I=GUrM|g;JJaH_BlNUh!3y$If;P`UA~n~3Y}Ck z=va|qi}G97@AIQakH{=9I5 zz>EkFFDWZil$S^PYMt{hov!iy<1Jh-eK%0kNKhQyMz@uTT{Ax@mTTweN6VkqD($=b ze2%CM^V~33_PkwIU~bmZeX;$o&Lq!SxnSwwYJRN{Usd1wpdF1*zUn;=?X{qvHdk~N zmq@f#ba8Qkf(4hLoQx+r+mX%YajDDz)Uc|G3LhUI3PeB+=T+}s&E8{q^r#_Y^3P63 zTe?JY5Yug{V23xi{)8p#Zg#$=QYb4U7ZTr)mmHTXxJ}pDD(CzUV!y@%t(a@4W6u0+ zWD7hEVpLq`gZJ(~-v0j?xd|Dx@A3Km9&9V(sr5Ylw-K252l`4aqhDMJmnJp$zYiSizpdC-wWBa%_p3*o z-^D6JNNJ+v*b!0a_K6RHv0;?d%6v2?$@^&!lF8Ec?DrF8-f~Rsu}vCn#^~KhYha}MX^U# zmAxDb%MW`0^0!wNtQbW0TdMy`=wZq>>-mxOrjV}gkmhYI@oaXUdlh*m-ZxT1Rr7a- zMKuk5p?mFlDYVB=<(TIG?gLD7@VG##QWCt?R5x{ats8p5a{~a zuy6k68>t|1hYP z#R5yUWKQ;NRB4=Ng~bcM$oVN9dlIf%I#&8fv*>2%oU?EZANdfs6TDv>!jKQIMTEv8 z)w*)-&1P)9uvou=S$3SX(2-#bjDmnV*C_v*l4s=4{)3xbUTd_8on6$wS0x!MmaW*@ zykvZwx8i=OjMEu;+JsYbI`Pp@((z5p{e=*pQgL~+v6l3-0Z=Fu>F!`oo5)IO@DPu|Kd$?%CfQ~NBOE9c> zI{^bK$b?Ig>HvN!Ud-{mwXg`PGhSXx#2#QeBXjoi--f~;-$;S3$7h~y{d!mZ*zd9M zhl)kTjLHAuWod<|=(%mFFyxK?6P4npf21$DF{VvWroSSgOa2l0r}WJKpQ+zF8p_(* z+LL9w{#_3K#~KUmdiBzOG literal 0 HcmV?d00001 diff --git a/packed-bert/packedBERT_multi_label_text_classification.ipynb b/packed-bert/packedBERT_multi_label_text_classification.ipynb index 1a27389..9251035 100644 --- a/packed-bert/packedBERT_multi_label_text_classification.ipynb +++ b/packed-bert/packedBERT_multi_label_text_classification.ipynb @@ -132,10 +132,11 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "![GoEmotions dataset (Source: GoogleBlog)](../images/go_emotions.png)" + "![GoEmotions dataset (Source: GoogleBlog)](images/go_emotions.png)" ] }, { @@ -153,7 +154,7 @@ "source": [ "Let's initialise our training configurations. \n", "\n", - "In this notebook, we are using both data parallelism and pipeline parallelism (see this [tutorial](https://github.com/graphcore/tutorials/tree/master/tutorials/pytorch/tut2_efficient_data_loading) for more). Therefore the global batch size, which is the actual number of samples used for the weight update, is determined using four factors:\n", + "In this notebook, we are using both data parallelism and pipeline parallelism (see this [tutorial](https://github.com/graphcore/tutorials/tree/master/tutorials/pytorch/efficient_data_loading/walkthrough.ipynb) for more). Therefore the global batch size, which is the actual number of samples used for the weight update, is determined using four factors:\n", "\n", " global batch size = micro_batch_size * gradient accumulation steps * device iterations * replication factor\n", "\n", diff --git a/packed-bert/packedBERT_question_answering.ipynb b/packed-bert/packedBERT_question_answering.ipynb index fa97ddc..19fa668 100644 --- a/packed-bert/packedBERT_question_answering.ipynb +++ b/packed-bert/packedBERT_question_answering.ipynb @@ -129,7 +129,7 @@ "source": [ "Let's initialise our training configurations. \n", "\n", - "Note here that we define a 'micro' batch size, which is the local batch size that would be passed into the model on the CPU. In this notebook, we are using both data parallelism and pipeline parallelism (see this [tutorial](https://github.com/graphcore/tutorials/tree/master/tutorials/pytorch/efficient_data_loading)), so the 'global' batch size, i.e. the number of data elements passed for one gradient calculation on the IPU, is calculated using the `device_iterations`, `gradient_accumulation_steps`, `replication_factor` and `max_seq_per_pack` (maximum sequences in a pack) for training, such that:\n", + "Note here that we define a 'micro' batch size, which is the local batch size that would be passed into the model on the CPU. In this notebook, we are using both data parallelism and pipeline parallelism (see this [tutorial](https://github.com/graphcore/tutorials/tree/master/tutorials/pytorch/efficient_data_loading/walkthrough.ipynb) for more). Therefore the global batch size, which is the actual number of samples used for the weight update, is determined using four factors:), so the 'global' batch size, i.e. the number of data elements passed for one gradient calculation on the IPU, is calculated using the `device_iterations`, `gradient_accumulation_steps`, `replication_factor` and `max_seq_per_pack` (maximum sequences in a pack) for training, such that:\n", "\n", "```\n", "global_training_batch_size = micro_batch_size * device_iterations * gradient_accumulation_steps * replication_factor\n", @@ -318,6 +318,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "a47ea927", "metadata": {}, @@ -326,7 +327,7 @@ "\n", "The first step is to tokenize the dataset using the tokenizer. Note here that for packing, it is important to **not** pad the dataset, so `padding` should be set to `False`. If we pad, we will have to un-pad when packing sequences into a packed sequence, which is inefficient.\n", "\n", - "The preprocessing function is outlined in [the original (unpacked) question-answering notebook](question_answering.ipynb) for more information on it. In this case, we can import the preprocessing directly from `utils.packing`, ready *without* padding for PackedBERT." + "The preprocessing function is outlined in [the original (unpacked) question-answering notebook](../natural-language-processing/other-use-cases/question_answering.ipynb) for more information on it. In this case, we can import the preprocessing directly from `utils.packing`, ready *without* padding for PackedBERT." ] }, { diff --git a/packed-bert/packedBERT_single_label_text_classification.ipynb b/packed-bert/packedBERT_single_label_text_classification.ipynb index cb37161..6e024ba 100644 --- a/packed-bert/packedBERT_single_label_text_classification.ipynb +++ b/packed-bert/packedBERT_single_label_text_classification.ipynb @@ -101,6 +101,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": { "id": "rEJBSTyZIrIb" @@ -134,7 +135,7 @@ "- [MNLI](https://arxiv.org/abs/1704.05426) (Multi-Genre Natural Language Inference) Determine if a sentence entails, contradicts or is unrelated to a given hypothesis. (This dataset has two versions, one with the validation and test set coming from the same distribution, another called mismatched where the validation and test use out-of-domain data.)\n", "- [MRPC](https://www.microsoft.com/en-us/download/details.aspx?id=52398) (Microsoft Research Paraphrase Corpus) Determine if two sentences are paraphrases from one another or not.\n", "- [QNLI](https://rajpurkar.github.io/SQuAD-explorer/) (Question-answering Natural Language Inference) Determine if the answer to a question is in the second sentence or not. (This dataset is built from the SQuAD dataset.)\n", - "- [QQP](https://data.quora.com/First-Quora-Dataset-Release-Question-Pairs) (Quora Question Pairs2) Determine if two questions are semantically equivalent or not.\n", + "- [QQP](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (Quora Question Pairs2) Determine if two questions are semantically equivalent or not.\n", "- [RTE](https://aclweb.org/aclwiki/Recognizing_Textual_Entailment) (Recognizing Textual Entailment) Determine if a sentence entails a given hypothesis or not.\n", "- [SST-2](https://nlp.stanford.edu/sentiment/index.html) (Stanford Sentiment Treebank) Determine if the sentence has a positive or negative sentiment.\n", "- [STS-B](http://ixa2.si.ehu.es/stswiki/index.php/STSbenchmark) (Semantic Textual Similarity Benchmark) Determine the similarity of two sentences with a score from 1 to 5.\n", @@ -170,7 +171,7 @@ "source": [ "Let's initialise our training configurations. \n", "\n", - "In this notebook, we are using both data parallelism and pipeline parallelism (see this [tutorial](https://github.com/graphcore/tutorials/tree/master/tutorials/pytorch/tut2_efficient_data_loading) for more). Therefore the global batch size, which is the actual number of samples used for the weight update, is determined using four factors:\n", + "In this notebook, we are using both data parallelism and pipeline parallelism (see this [tutorial](https://github.com/graphcore/tutorials/tree/master/tutorials/pytorch/efficient_data_loading/walkthrough.ipynb) for more). Therefore the global batch size, which is the actual number of samples used for the weight update, is determined using four factors:\n", "\n", " global batch size = micro_batch_size * gradient accumulation steps * device iterations * replication factor\n", "\n", @@ -709,6 +710,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ From c2041d18afbc2558ceec3b9953fbee62d5a04ce9 Mon Sep 17 00:00:00 2001 From: Arsalan Uddin Date: Thu, 9 Mar 2023 15:42:42 +0000 Subject: [PATCH 03/11] updating notebook tests --- .gradient/notebook-tests.yaml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.gradient/notebook-tests.yaml b/.gradient/notebook-tests.yaml index 0659cc3..1b54b94 100644 --- a/.gradient/notebook-tests.yaml +++ b/.gradient/notebook-tests.yaml @@ -167,4 +167,26 @@ useful-managing-ipu-resources: generated: true notebook: file: managing_ipu_resources.ipynb - timeout: 1000 \ No newline at end of file + timeout: 1000 + +# Packed BERT tests +packed-bert-single-label: + location: ../packed-bert/ + generated: true + notebook: + file: packedBERT_single_label_text_classification.ipynb + timeout: 10000 + +packed-bert-multi-label: + location: ../packed-bert/ + generated: true + notebook: + file: packedBERT_multi_label_text_classification.ipynb + timeout: 10000 + +packed-bert-question-answering: + location: ../packed-bert/ + generated: true + notebook: + file: packedBERT_question_answering.ipynb + timeout: 10000 \ No newline at end of file From b08c363ce82ca60852a3befd77e744254bf209ef Mon Sep 17 00:00:00 2001 From: Arsalan Uddin Date: Thu, 9 Mar 2023 16:39:41 +0000 Subject: [PATCH 04/11] removing notebook-saved local path to fix test --- ...dBERT_multi_label_text_classification.ipynb | 10 +++++++--- .../packedBERT_question_answering.ipynb | 18 +++++++++++++++++- ...BERT_single_label_text_classification.ipynb | 9 +++++++-- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/packed-bert/packedBERT_multi_label_text_classification.ipynb b/packed-bert/packedBERT_multi_label_text_classification.ipynb index 9251035..7bede48 100644 --- a/packed-bert/packedBERT_multi_label_text_classification.ipynb +++ b/packed-bert/packedBERT_multi_label_text_classification.ipynb @@ -1139,10 +1139,11 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "Let's load the checkpoint we saved earlier to run the inference on:" + "To test inference throughput, we can just use a default checkpoint to run the model and evaluate the speed of packing. If you saved the model locally or on the hub earlier, you can replace `model_checkpoint` with the path to your model to perform inference on the fine-tuned weights!" ] }, { @@ -1151,10 +1152,13 @@ "metadata": {}, "outputs": [], "source": [ - "model_checkpoint = f\"./{model_name}_{model_task}\"\n", + "model_checkpoint = \"bert-base-uncased\"\n", + "\n", + "# Load checkpoint locally:\n", + "# model_checkpoint = f\"./{model_name}-{task}\"\n", "\n", "# Load from Huggingface Hub instead:\n", - "# model_checkpoint = '/{model_name}-{model_task}'\n", + "# model_checkpoint = '/{model_checkpoint}-{model_task}'\n", "\n", "model = PipelinedPackedBertForSequenceClassification.from_pretrained(model_checkpoint, config=config)" ] diff --git a/packed-bert/packedBERT_question_answering.ipynb b/packed-bert/packedBERT_question_answering.ipynb index 19fa668..54acd59 100644 --- a/packed-bert/packedBERT_question_answering.ipynb +++ b/packed-bert/packedBERT_question_answering.ipynb @@ -1049,6 +1049,14 @@ "ipu_config.ipus_per_replica = 1" ] }, + { + "cell_type": "markdown", + "id": "0db86630", + "metadata": {}, + "source": [ + "To test inference throughput, we can just use a default checkpoint to run the model and evaluate the speed of packing. If you saved the model locally or on the hub earlier, you can replace `model_checkpoint` with the path to your model to perform inference on the fine-tuned weights!" + ] + }, { "cell_type": "code", "execution_count": null, @@ -1056,8 +1064,16 @@ "metadata": {}, "outputs": [], "source": [ + "model_checkpoint = \"bert-base-uncased\"\n", + "\n", + "# Load checkpoint locally:\n", + "# model_checkpoint = f\"./{model_name}-{task}\"\n", + "\n", + "# Load from Huggingface Hub instead:\n", + "# model_checkpoint = '/{model_checkpoint}-{model_task}'\n", + "\n", "model = PipelinedPackedBertForQuestionAnswering.from_pretrained(\n", - " f\"./{model_checkpoint}-{model_task}\", config=config)" + " model_checkpoint, config=config)" ] }, { diff --git a/packed-bert/packedBERT_single_label_text_classification.ipynb b/packed-bert/packedBERT_single_label_text_classification.ipynb index 6e024ba..363a5f8 100644 --- a/packed-bert/packedBERT_single_label_text_classification.ipynb +++ b/packed-bert/packedBERT_single_label_text_classification.ipynb @@ -1,6 +1,7 @@ { "cells": [ { + "attachments": {}, "cell_type": "markdown", "metadata": { "id": "X4cRE8IbIrIV" @@ -1279,10 +1280,11 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "Let's load the checkpoint we saved earlier to run the inference on:" + "To test inference throughput, we can just use a default checkpoint to run the model and evaluate the speed of packing. If you saved the model locally or on the hub earlier, you can replace `model_checkpoint` with the path to your model to perform inference on the fine-tuned weights!" ] }, { @@ -1293,7 +1295,10 @@ }, "outputs": [], "source": [ - "model_checkpoint = f\"./{model_name}-{task}\"\n", + "model_checkpoint = \"bert-base-uncased\"\n", + "\n", + "# Load checkpoint locally:\n", + "# model_checkpoint = f\"./{model_name}-{task}\"\n", "\n", "# Load from Huggingface Hub instead:\n", "# model_checkpoint = '/{model_checkpoint}-{model_task}'\n", From 91c74f9d217986bf4ee16527d60e8d99319ebdc6 Mon Sep 17 00:00:00 2001 From: Arsalan Uddin Date: Thu, 16 Mar 2023 15:06:28 +0000 Subject: [PATCH 05/11] Updating with batched inference pipeline update from latest Optimum merge --- packed-bert/__init__.py | 0 packed-bert/models/modeling_bert_packed.py | 3 + ...BERT_multi_label_text_classification.ipynb | 303 +++++++----- .../packedBERT_question_answering.ipynb | 289 ++++++------ ...ERT_single_label_text_classification.ipynb | 288 ++++++------ packed-bert/pipeline/__init__.py | 0 packed-bert/pipeline/packed_bert.py | 436 ++++++++++++++++++ packed-bert/utils/packing/dataset_creator.py | 23 +- .../utils/packing/dataset_templates.py | 7 +- packed-bert/utils/packing/qa_utils.py | 8 +- 10 files changed, 921 insertions(+), 436 deletions(-) create mode 100644 packed-bert/__init__.py create mode 100644 packed-bert/pipeline/__init__.py create mode 100644 packed-bert/pipeline/packed_bert.py diff --git a/packed-bert/__init__.py b/packed-bert/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packed-bert/models/modeling_bert_packed.py b/packed-bert/models/modeling_bert_packed.py index 023b634..81dabdd 100644 --- a/packed-bert/models/modeling_bert_packed.py +++ b/packed-bert/models/modeling_bert_packed.py @@ -159,6 +159,9 @@ def forward( if labels is not None: logits = output.logits.reshape([-1, self.max_seq_per_pack, self.num_labels]) output.logits = logits + output = (output.loss, output.logits) + else: + output = output.logits else: output = self.multi_label_outputs( diff --git a/packed-bert/packedBERT_multi_label_text_classification.ipynb b/packed-bert/packedBERT_multi_label_text_classification.ipynb index 7bede48..ef4b706 100644 --- a/packed-bert/packedBERT_multi_label_text_classification.ipynb +++ b/packed-bert/packedBERT_multi_label_text_classification.ipynb @@ -18,8 +18,7 @@ "height": 1000 }, "id": "MOsHUjgdIrIW", - "outputId": "f84a093e-147f-470e-aad9-80fb51193c8e", - "scrolled": false + "outputId": "f84a093e-147f-470e-aad9-80fb51193c8e" }, "outputs": [], "source": [ @@ -36,9 +35,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [], "source": [ "%pip install scikit-learn;\n", @@ -60,7 +57,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "import transformers\n", @@ -82,7 +81,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from huggingface_hub import notebook_login\n", @@ -100,7 +101,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "!apt install git-lfs" @@ -116,7 +119,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -132,15 +134,13 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "![GoEmotions dataset (Source: GoogleBlog)](images/go_emotions.png)" + "![GoEmotions dataset (Source: GoogleBlog)](../images/go_emotions.png)" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -148,13 +148,12 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "Let's initialise our training configurations. \n", "\n", - "In this notebook, we are using both data parallelism and pipeline parallelism (see this [tutorial](https://github.com/graphcore/tutorials/tree/master/tutorials/pytorch/efficient_data_loading/walkthrough.ipynb) for more). Therefore the global batch size, which is the actual number of samples used for the weight update, is determined using four factors:\n", + "In this notebook, we are using both data parallelism and pipeline parallelism (see this [tutorial](https://github.com/graphcore/tutorials/blob/master/tutorials/pytorch/efficient_data_loading/walkthrough.ipynb) for more). Therefore the global batch size, which is the actual number of samples used for the weight update, is determined using four factors:\n", "\n", " global batch size = micro_batch_size * gradient accumulation steps * device iterations * replication factor\n", "\n", @@ -175,7 +174,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "zVvslsfMIrIh" + "id": "zVvslsfMIrIh", + "tags": [] }, "outputs": [], "source": [ @@ -190,7 +190,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -210,7 +209,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "import os\n", @@ -241,7 +242,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "IreSlFmlIrIm" + "id": "IreSlFmlIrIm", + "tags": [] }, "outputs": [], "source": [ @@ -252,7 +254,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "dataset = load_dataset(model_task)\n", @@ -273,7 +277,8 @@ "execution_count": null, "metadata": { "id": "GWiVUF0jIrIv", - "outputId": "35e3ea43-f397-4a54-c90c-f2cf8d36873e" + "outputId": "35e3ea43-f397-4a54-c90c-f2cf8d36873e", + "tags": [] }, "outputs": [], "source": [ @@ -294,7 +299,8 @@ "execution_count": null, "metadata": { "id": "X6HrpprwIrIz", - "outputId": "d7670bc0-42e4-4c09-8a6a-5c018ded7d95" + "outputId": "d7670bc0-42e4-4c09-8a6a-5c018ded7d95", + "tags": [] }, "outputs": [], "source": [ @@ -314,7 +320,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "i3j8APAoIrI3" + "id": "i3j8APAoIrI3", + "tags": [] }, "outputs": [], "source": [ @@ -344,7 +351,8 @@ "execution_count": null, "metadata": { "id": "SZy5tRB_IrI7", - "outputId": "ba8f2124-e485-488f-8c0c-254f34f24f13" + "outputId": "ba8f2124-e485-488f-8c0c-254f34f24f13", + "tags": [] }, "outputs": [], "source": [ @@ -365,7 +373,8 @@ "execution_count": null, "metadata": { "id": "5o4rUteaIrI_", - "outputId": "18038ef5-554c-45c5-e00a-133b02ec10f1" + "outputId": "18038ef5-554c-45c5-e00a-133b02ec10f1", + "tags": [] }, "outputs": [], "source": [ @@ -391,7 +400,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": { "id": "YVx71GdAIrJH" @@ -411,7 +419,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "eXNLu_-nIrJI" + "id": "eXNLu_-nIrJI", + "tags": [] }, "outputs": [], "source": [ @@ -443,7 +452,8 @@ "execution_count": null, "metadata": { "id": "a5hBlsrHIrJL", - "outputId": "acdaa98a-a8cd-4a20-89b8-cc26437bbe90" + "outputId": "acdaa98a-a8cd-4a20-89b8-cc26437bbe90", + "tags": [] }, "outputs": [], "source": [ @@ -476,7 +486,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "vc0BSBLIIrJQ" + "id": "vc0BSBLIIrJQ", + "tags": [] }, "outputs": [], "source": [ @@ -504,7 +515,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "import numpy as np\n", @@ -532,7 +545,8 @@ "execution_count": null, "metadata": { "id": "DDtsaJeVIrJT", - "outputId": "aa4734bf-4ef5-4437-9948-2c16363da719" + "outputId": "aa4734bf-4ef5-4437-9948-2c16363da719", + "tags": [] }, "outputs": [], "source": [ @@ -570,7 +584,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "max_seq_per_pack = 6" @@ -586,7 +602,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "problem_type = 'multi_label_classification'" @@ -626,7 +644,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from utils.packing.dataset_creator import PackedDatasetCreator\n", @@ -670,7 +690,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", @@ -714,7 +736,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "packed_train_dataset = train_data_packer.create()\n", @@ -731,7 +755,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "packed_train_dataset[133]" @@ -768,7 +794,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from transformers import AutoConfig\n", @@ -790,7 +818,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "scrolled": true + "scrolled": true, + "tags": [] }, "outputs": [], "source": [ @@ -807,7 +836,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -819,7 +847,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "# test the model on CPU\n", @@ -856,7 +886,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "scrolled": true + "scrolled": true, + "tags": [] }, "outputs": [], "source": [ @@ -876,7 +907,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "scrolled": false + "tags": [] }, "outputs": [], "source": [ @@ -913,7 +944,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from optimum.graphcore import IPUConfig, IPUTrainer, IPUTrainingArguments\n", @@ -930,7 +963,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -945,7 +977,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "scrolled": true + "scrolled": true, + "tags": [] }, "outputs": [], "source": [ @@ -994,7 +1027,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "scrolled": false + "tags": [] }, "outputs": [], "source": [ @@ -1017,7 +1050,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "scrolled": false + "tags": [] }, "outputs": [], "source": [ @@ -1052,7 +1085,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "trainer.save_model(\"./\"+f\"{model_name}-{model_task}\")" @@ -1069,140 +1104,156 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Faster inference" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This section demonstrates how to perform faster, batched inference with a large number of samples using packing." + "## Fast batched inference" ] }, { "cell_type": "markdown", - "metadata": {}, - "source": [ - "When training, the packing factor affects the convergence and hyperparameters in a similar way to a large increase in batch size. However, for inference-only runs, we are free to use a bigger packing factor to speed it up. Let's try it on GoEmotions with `max_seq_per_pack = 12`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "max_seq_per_pack = 12" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "metadata": { + "tags": [] + }, "source": [ - "dataset = load_dataset(\"go_emotions\")\n", + "Packing can also be used for inference, particularly for performing inference for workloads. This section demonstrates how to perform faster, batched inference with a large number of samples using a super-easy custom pipeline which batches and packs your input data, performs inference and returns postprocessed predictions. \n", "\n", - "encoded_dataset = dataset.map(id_to_N_hot)\n", - "encoded_dataset = encoded_dataset.map(preprocess_function, batched=True)\n", - "inference_dataset = encoded_dataset['train'] #Lets use the train dataset to have more features\n", + "For the pipeline, we need to import it, and initialise a few essential parameters.\n", "\n", - "# The dataset initialisation and .create() can be executed in one line\n", - "inference_packed_dataset = PackedDatasetCreator(\n", - " tokenized_dataset = encoded_dataset['train'],\n", - " max_sequence_length = max_seq_length,\n", - " max_sequences_per_pack = max_seq_per_pack,\n", - " inference = True,\n", - " problem_type = problem_type).create()\n", - "\n" + "The `model` is the model checkpoint, we are going to use the locally saved checkpoint generated from training `go_emotions`. The `executable_cache_dir`, `problem_type`, `max_seq_length` must be specified. To return predictions organised by class names, the class names for your output must be passed to `label_categories`. \n", + "\n", + "The pipeline will automatically determine your model's IPU config, given that the checkpoint was trained using Optimum Graphcore, which will be the case for the model fine-tuned in this notebook.\n", + "\n", + "In this example, we pre-load the IPUConfig and modify some of the default parameters to get the best performance out of inference and leverage the benefits of IPU parallelism. The micro-batch size can also be specified, for which the default is 1.\n", + "\n", + "When training, the packing factor affects the convergence the same way as a large increase in batch size would do. However, for inference, we are free to use a bigger packing factor to speed it up. Let's try it with `max_seq_per_pack = 12`.\n", + "\n", + "**Note:** Packing brings huge benefits for performing inference on large amounts of data. For small scale inference tasks, such as those which more suit sequential inference on a single un-batched input, the generic Optimum Graphcore `TextClassificationPipeline` may be prefered. This won't affect fine-tuning, the weights generated from fine-tuning using packing will work just the same!" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "We can see that the average packing factor has improved from 5.68 to 9.14, allowing an approximate **9 times** throughput speed-up from the base unpacked model.\n", - "\n", - "Let's also modify the configuration of the model for inference. For speed up, we can replicate a one-IPU run (`ipus_per_replica`) over four IPUs by changing the `replication_factor`. After this, we can re-initialise the model and the `IPUTrainer` with the existing arguments." + "Lets list the class names for the GoEmotions dataset." ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ - "ipu_config.layers_per_ipu = [12]\n", - "ipu_config.inference_device_iterations = 32\n", - "ipu_config.inference_replication_factor = 4\n", - "ipu_config.ipus_per_replica = 1" + "class_names = [\n", + " \"admiration\",\n", + " \"amusement\",\n", + " \"anger\",\n", + " \"annoyance\",\n", + " \"approval\",\n", + " \"caring\",\n", + " \"confusion\",\n", + " \"curiosity\",\n", + " \"desire\",\n", + " \"disappointment\",\n", + " \"disapproval\",\n", + " \"disgust\",\n", + " \"embarrassment\",\n", + " \"excitement\",\n", + " \"fear\",\n", + " \"gratitude\",\n", + " \"grief\",\n", + " \"joy\",\n", + " \"love\",\n", + " \"nervousness\",\n", + " \"optimism\",\n", + " \"pride\",\n", + " \"realization\",\n", + " \"relief\",\n", + " \"remorse\",\n", + " \"sadness\",\n", + " \"surprise\",\n", + " \"neutral\",\n", + "]" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "To test inference throughput, we can just use a default checkpoint to run the model and evaluate the speed of packing. If you saved the model locally or on the hub earlier, you can replace `model_checkpoint` with the path to your model to perform inference on the fine-tuned weights!" + "Lets initialise the `PackedBertTextClassificationPipeline`." ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ - "model_checkpoint = \"bert-base-uncased\"\n", + "from pipeline.packed_bert import PackedBertTextClassificationPipeline\n", "\n", - "# Load checkpoint locally:\n", - "# model_checkpoint = f\"./{model_name}-{task}\"\n", + "from optimum.graphcore import IPUConfig\n", "\n", - "# Load from Huggingface Hub instead:\n", - "# model_checkpoint = '/{model_checkpoint}-{model_task}'\n", + "model = 'Graphcore/bert-base-uncased' # This is set as default, comment this line to run your fine-tuned model from the lines below!\n", + "# model = \"./\"+f\"{model_name}-{model_task}\" # uncomment to load your fine-tuned model from disk\n", + "# model = 'your_username/{model_name}-{model_task}' # uncomment this and use your username to load from Hugging Face Hub\n", "\n", - "model = PipelinedPackedBertForSequenceClassification.from_pretrained(model_checkpoint, config=config)" + "inference_boosted_ipu_config = IPUConfig.from_pretrained(model, \n", + " inference_device_iterations=32,\n", + " inference_replication_factor=4,\n", + " ipus_per_replica=1,\n", + " layers_per_ipu=[12]\n", + " )\n", + "\n", + "pipeline = PackedBertTextClassificationPipeline(\n", + " model = model,\n", + " executable_cache_dir = executable_cache_dir,\n", + " problem_type='multi_label_classification',\n", + " max_seq_per_pack=12,\n", + " max_seq_length=max_seq_length,\n", + " ipu_config=inference_boosted_ipu_config,\n", + " micro_batch_size=8,\n", + " label_categories=class_names\n", + ")" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "args = IPUTrainingArguments(\n", - " \"/tmp/\"+f\"{model_name}-{model_task}-fast-inf\",\n", - " per_device_eval_batch_size=8,\n", - " dataloader_mode=\"async_rebatched\",\n", - " dataloader_drop_last=True,\n", - " logging_steps=10,\n", - " pod_type=pod_type\n", - ")\n", + "The pipeline expects a **list of strings** directly passed to it. There is no need to tokenize, preprocess, pack or postprocess the data to use the inference pipeline.\n", "\n", - "trainer = IPUTrainer(\n", - " model,\n", - " ipu_config,\n", - " args,\n", - " eval_dataset=inference_packed_dataset,\n", - " compute_metrics=compute_metrics\n", - ")" + "As a test, we can load the entire `sst2` dataset and perform packed inference using `.predict()` on the text column to generate predictions. \n", + "\n", + "For datasets with multiple sentences, these can simply be passed as `predict(,)`" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "scrolled": false + "tags": [] }, "outputs": [], "source": [ - "trainer.evaluate(inference_packed_dataset)" + "import datasets\n", + "dataset = datasets.load_dataset('go_emotions','simplified')\n", + "preds = pipeline.predict(dataset['train']['text'])\n", + "\n", + "print(preds.keys())\n", + "print(f\"Number of predictions: {len(preds['predictions'])}\")\n", + "print(f\"Preprocessing time: {preds['preprocessing_time']}s\")\n", + "print(f\"Postprocessing time: {preds['postprocessing_time']}s\")\n", + "print(f\"Throughput: {preds['throughput']} samples/s\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Using these simple optimisations and the increase in maximum sequences per pack, we can see a massive throughput increase to approximately **25k sequences per second** - remember that to obtain the actual throughput we multiply the packed samples/s by the average packing factor - highlighting the benefits of using packing! " + "There is minimal overhead from tokenizing and packing the dataset, but the speed benefits are evident. After increasing the maximum sequences to 12, we can observe a much higher packing factor of 9.14.\n", + "\n", + "Running the above pipeline, we achieve a throughput approximately 45000 samples per second, demonstrating the huge time benefit you can achieve by using packing!" ] } ], @@ -1235,5 +1286,5 @@ } }, "nbformat": 4, - "nbformat_minor": 1 + "nbformat_minor": 4 } diff --git a/packed-bert/packedBERT_question_answering.ipynb b/packed-bert/packedBERT_question_answering.ipynb index 54acd59..396b22d 100644 --- a/packed-bert/packedBERT_question_answering.ipynb +++ b/packed-bert/packedBERT_question_answering.ipynb @@ -55,7 +55,9 @@ "cell_type": "code", "execution_count": null, "id": "aa8d39f7", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "import transformers\n", @@ -79,7 +81,9 @@ "cell_type": "code", "execution_count": null, "id": "d4c81945", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from huggingface_hub import notebook_login\n", @@ -99,7 +103,9 @@ "cell_type": "code", "execution_count": null, "id": "afa6ac5a", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "! apt install git-lfs" @@ -122,14 +128,13 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "89898522", "metadata": {}, "source": [ "Let's initialise our training configurations. \n", "\n", - "Note here that we define a 'micro' batch size, which is the local batch size that would be passed into the model on the CPU. In this notebook, we are using both data parallelism and pipeline parallelism (see this [tutorial](https://github.com/graphcore/tutorials/tree/master/tutorials/pytorch/efficient_data_loading/walkthrough.ipynb) for more). Therefore the global batch size, which is the actual number of samples used for the weight update, is determined using four factors:), so the 'global' batch size, i.e. the number of data elements passed for one gradient calculation on the IPU, is calculated using the `device_iterations`, `gradient_accumulation_steps`, `replication_factor` and `max_seq_per_pack` (maximum sequences in a pack) for training, such that:\n", + "Note here that we define a 'micro' batch size, which is the local batch size that would be passed into the model on the CPU. In this notebook, we are using both data parallelism and pipeline parallelism (see this [tutorial](https://github.com/graphcore/tutorials/blob/master/tutorials/pytorch/efficient_data_loading/walkthrough.ipynb)), so the 'global' batch size, i.e. the number of data elements passed for one gradient calculation on the IPU, is calculated using the `device_iterations`, `gradient_accumulation_steps`, `replication_factor` and `max_seq_per_pack` (maximum sequences in a pack) for training, such that:\n", "\n", "```\n", "global_training_batch_size = micro_batch_size * device_iterations * gradient_accumulation_steps * replication_factor\n", @@ -150,12 +155,14 @@ "cell_type": "code", "execution_count": null, "id": "0ad1b478", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "model_checkpoint=\"bert-base-uncased\" # Default uncased pre-trained BERT checkpoint\n", "ipu_config_name=\"Graphcore/bert-base-uncased\" # Default Graphcore IPU config initialisation for pre-trained BERT\n", - "max_seq_length=384 # The maximum sequence length allowed for sequences in the model.\n", + "max_seq_length=512 # The maximum sequence length allowed for sequences in the model.\n", "gradient_accumulation_steps=32 # Gradient accumulation steps for training the model on the IPU.\n", "device_iterations = 32\n", "micro_batch_size=2\n", @@ -188,7 +195,9 @@ "cell_type": "code", "execution_count": null, "id": "b882a5b3", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "import os\n", @@ -212,15 +221,14 @@ "execution_count": null, "id": "b37cb293", "metadata": { - "scrolled": true + "scrolled": true, + "tags": [] }, "outputs": [], "source": [ "from datasets import load_dataset, load_metric\n", "import evaluate\n", "\n", - "\n", - "\n", "dataset = load_dataset(model_task) # Load dataset\n", "metric = evaluate.load(model_task) # Load metric for dataset" ] @@ -237,7 +245,9 @@ "cell_type": "code", "execution_count": null, "id": "2115928b", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "dataset" @@ -255,7 +265,9 @@ "cell_type": "code", "execution_count": null, "id": "311b8b73", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "dataset[\"train\"][0]" @@ -275,7 +287,9 @@ "cell_type": "code", "execution_count": null, "id": "628bc41f", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "question_key=\"question\"\n", @@ -308,7 +322,9 @@ "cell_type": "code", "execution_count": null, "id": "aab94819", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from transformers import AutoTokenizer\n", @@ -318,7 +334,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "a47ea927", "metadata": {}, @@ -327,7 +342,7 @@ "\n", "The first step is to tokenize the dataset using the tokenizer. Note here that for packing, it is important to **not** pad the dataset, so `padding` should be set to `False`. If we pad, we will have to un-pad when packing sequences into a packed sequence, which is inefficient.\n", "\n", - "The preprocessing function is outlined in [the original (unpacked) question-answering notebook](../natural-language-processing/other-use-cases/question_answering.ipynb) for more information on it. In this case, we can import the preprocessing directly from `utils.packing`, ready *without* padding for PackedBERT." + "The preprocessing function is outlined in [the original (unpacked) question-answering notebook](question_answering.ipynb) for more information on it. In this case, we can import the preprocessing directly from `utils.packing`, ready *without* padding for PackedBERT." ] }, { @@ -335,7 +350,8 @@ "execution_count": null, "id": "2263dfef", "metadata": { - "scrolled": true + "scrolled": true, + "tags": [] }, "outputs": [], "source": [ @@ -383,7 +399,9 @@ "cell_type": "code", "execution_count": null, "id": "6bdd1b9e", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "max_seq_per_pack = 6" @@ -401,7 +419,9 @@ "cell_type": "code", "execution_count": null, "id": "bfda406f", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "num_labels = 2\n", @@ -446,7 +466,9 @@ "cell_type": "code", "execution_count": null, "id": "e66ed06d", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from utils.packing.dataset_creator import PackedDatasetCreator\n", @@ -496,7 +518,9 @@ "cell_type": "code", "execution_count": null, "id": "113b58f4", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", @@ -552,7 +576,9 @@ "cell_type": "code", "execution_count": null, "id": "bdcc161d", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "packed_train_dataset = train_data_packer.create()\n", @@ -571,14 +597,15 @@ "cell_type": "code", "execution_count": null, "id": "c966cd9a", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "packed_train_dataset[133]" ] }, { - "attachments": {}, "cell_type": "markdown", "id": "a1d8ce9c", "metadata": {}, @@ -598,15 +625,16 @@ "cell_type": "code", "execution_count": null, "id": "254a0f83", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from transformers import AutoConfig\n", "\n", "config = AutoConfig.from_pretrained(model_checkpoint)\n", "config.max_sequences_per_pack = max_seq_per_pack\n", - "config.num_labels = num_labels\n", - "config.problem_type = problem_type" + "config.num_labels = num_labels" ] }, { @@ -622,7 +650,7 @@ "execution_count": null, "id": "3285aaa3", "metadata": { - "scrolled": false + "tags": [] }, "outputs": [], "source": [ @@ -637,7 +665,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "d6070000", "metadata": {}, @@ -651,7 +678,9 @@ "cell_type": "code", "execution_count": null, "id": "02aac4e1", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "# test the model on CPU\n", @@ -681,7 +710,9 @@ "cell_type": "code", "execution_count": null, "id": "11502853", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "model.half()" @@ -712,7 +743,9 @@ "cell_type": "code", "execution_count": null, "id": "f4c6e4d0", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from utils.packing.qa_utils import postprocess_packed_qa_predictions" @@ -730,7 +763,9 @@ "cell_type": "code", "execution_count": null, "id": "5420ef6a", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "def compute_validation_metrics(predictions, raw_validation_dataset, packed_validation_dataset_unformatted, metric):\n", @@ -778,7 +813,9 @@ "cell_type": "code", "execution_count": null, "id": "8b0452e1", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from optimum.graphcore import IPUConfig, IPUTrainer, IPUTrainingArguments\n", @@ -808,7 +845,7 @@ "execution_count": null, "id": "141a2e2d", "metadata": { - "scrolled": false + "tags": [] }, "outputs": [], "source": [ @@ -824,6 +861,7 @@ " lr_scheduler_type='cosine',\n", " pod_type=pod_type,\n", " gradient_accumulation_steps=gradient_accumulation_steps,\n", + " dataloader_mode=\"async_rebatched\",\n", " dataloader_drop_last=True,\n", " dataloader_num_workers=64,\n", " logging_steps=5\n", @@ -853,7 +891,7 @@ "execution_count": null, "id": "561a41ca", "metadata": { - "scrolled": false + "tags": [] }, "outputs": [], "source": [ @@ -881,7 +919,8 @@ "execution_count": null, "id": "c4cbe563", "metadata": { - "scrolled": true + "scrolled": true, + "tags": [] }, "outputs": [], "source": [ @@ -900,7 +939,9 @@ "cell_type": "code", "execution_count": null, "id": "b9e60061", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "trainer.push_to_hub()" @@ -918,7 +959,9 @@ "cell_type": "code", "execution_count": null, "id": "625847dc", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "trainer.save_model(f\"./{model_checkpoint}-{model_task}\")" @@ -937,7 +980,8 @@ "execution_count": null, "id": "c65f6830", "metadata": { - "scrolled": true + "scrolled": true, + "tags": [] }, "outputs": [], "source": [ @@ -945,7 +989,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "7553b34d", "metadata": {}, @@ -958,7 +1001,8 @@ "execution_count": null, "id": "825dd9a8", "metadata": { - "scrolled": true + "scrolled": true, + "tags": [] }, "outputs": [], "source": [ @@ -973,7 +1017,7 @@ "id": "50eb0d90", "metadata": {}, "source": [ - "## Faster Inference" + "## Faster batched inference" ] }, { @@ -981,123 +1025,65 @@ "id": "6bda303c", "metadata": {}, "source": [ - "When training, the packing factor affects the convergence and hyperparameters in a similar way to a large increase in batch size. However, for inference-only runs, we are free to use a bigger packing factor to speed it up. Let's try it on SQuAD with max_seq_per_pack = 12, and sequence length set to 512." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "50cd4b0e", - "metadata": {}, - "outputs": [], - "source": [ - "max_seq_per_pack = 12" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c64fcd44", - "metadata": {}, - "outputs": [], - "source": [ - "dataset = load_dataset(\"squad\")\n", - "raw_train_dataset = dataset['train']\n", - "max_seq_length = 512\n", + "Packing can also be used for inference, particularly for performing inference for workloads. This section demonstrates how to perform faster, batched inference with a large number of samples using a super-easy custom pipeline which batches and packs your input data, performs inference and returns postprocessed predictions. \n", "\n", - "# Lets use the train dataset to have more features to infer over\n", - "tokenized_inference_dataset = preprocess_packed_qa(\n", - " dataset=raw_train_dataset,\n", - " tokenizer=tokenizer,\n", - " question_key=question_key,\n", - " context_key=context_key,\n", - " answer_key=answer_key,\n", - " sequence_length=max_seq_length,\n", - " padding=False,\n", - " train=False\n", - ")\n", + "For the pipeline, we need to import it, and initialise a few essential parameters.\n", "\n", - "packed_inference_dataset = PackedDatasetCreator(\n", - " tokenized_dataset = tokenized_inference_dataset,\n", - " max_sequence_length = max_seq_length,\n", - " max_sequences_per_pack = max_seq_per_pack,\n", - " inference=True,\n", - " problem_type = problem_type,\n", - ").create()" - ] - }, - { - "cell_type": "markdown", - "id": "42d720aa", - "metadata": {}, - "source": [ - "We can see that the average packing factor has improved from 2.2 to 2.95, allowing an approximate 3x throughput speed-up from the base unpacked model. This is not nearly as much as the maximum sequences per pack limit, due to the larger sequence lengths in the SQuAD dataset, but still allows a 3x speedup for inference!\n", + "The `model` is the model checkpoint, we are going to use the locally saved checkpoint generated from training SQuAD. The `executable_cache_dir` and `max_seq_length` must also be specified.\n", "\n", - "Let's also modify the configuration of the model for inference. For speed up, we can replicate a one-IPU run (`ipus_per_replica`) over four IPUs by changing the `replication_factor`. After this, we can re-initialise the model and the `IPUTrainer` with the existing arguments." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2b7f4fca", - "metadata": {}, - "outputs": [], - "source": [ - "ipu_config.layers_per_ipu = [12]\n", - "ipu_config.inference_device_iterations = 32\n", - "ipu_config.inference_replication_factor = 4\n", - "ipu_config.ipus_per_replica = 1" - ] - }, - { - "cell_type": "markdown", - "id": "0db86630", - "metadata": {}, - "source": [ - "To test inference throughput, we can just use a default checkpoint to run the model and evaluate the speed of packing. If you saved the model locally or on the hub earlier, you can replace `model_checkpoint` with the path to your model to perform inference on the fine-tuned weights!" + "The pipeline will automatically determine your model's IPU config, given that the checkpoint was trained using Optimum Graphcore, which will be the case for the model fine-tuned in this notebook.\n", + "\n", + "In this example, we pre-load the IPUConfig and modify some of the default parameters to get the best performance out of inference and leverage the benefits of IPU parallelism. The micro-batch size can also be specified, for which the default is 1.\n", + "\n", + "When training, the packing factor affects the convergence the same way as a large increase in batch size would do. However, for inference, we are free to use a bigger packing factor to speed it up. Let's try it with `max_seq_per_pack = 12`.\n", + "\n", + "**Note:** Packing brings huge benefits for performing inference on large amounts of data. For small scale inference tasks, such as those which more suit sequential inference on a single un-batched input, the generic Optimum Graphcore `TextClassificationPipeline` may be prefered. This won't affect fine-tuning, the weights generated from fine-tuning using packing will work just the same!" ] }, { "cell_type": "code", "execution_count": null, - "id": "a11a6f8f", - "metadata": {}, + "id": "50cd4b0e", + "metadata": { + "tags": [] + }, "outputs": [], "source": [ - "model_checkpoint = \"bert-base-uncased\"\n", + "from pipeline.packed_bert import PackedBertQuestionAnsweringPipeline\n", "\n", - "# Load checkpoint locally:\n", - "# model_checkpoint = f\"./{model_name}-{task}\"\n", + "model = 'Graphcore/bert-base-uncased' # This is set as default, comment this line to run your fine-tuned model from the lines below!\n", + "# model = \"./\"+f\"{model_checkpoint}-{task}\" # uncomment to load your fine-tuned model from disk\n", + "# model = 'your_username/{model_checkpoint}-{task}' # uncomment this and use your username to load from Hugging Face Hub\n", "\n", - "# Load from Huggingface Hub instead:\n", - "# model_checkpoint = '/{model_checkpoint}-{model_task}'\n", + "inference_boosted_ipu_config = IPUConfig.from_pretrained(model, \n", + " inference_device_iterations=32,\n", + " inference_replication_factor=4,\n", + " ipus_per_replica=1,\n", + " layers_per_ipu=[12]\n", + " )\n", "\n", - "model = PipelinedPackedBertForQuestionAnswering.from_pretrained(\n", - " model_checkpoint, config=config)" + "pipeline = PackedBertQuestionAnsweringPipeline(\n", + " model = f\"./{model_checkpoint}-{model_task}\",\n", + " executable_cache_dir = executable_cache_dir,\n", + " max_seq_per_pack=12,\n", + " max_seq_length=max_seq_length,\n", + " ipu_config=inference_boosted_ipu_config,\n", + " micro_batch_size=8\n", + ")" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "1debe64e", + "cell_type": "markdown", + "id": "42d720aa", "metadata": {}, - "outputs": [], "source": [ - "args = IPUTrainingArguments(\n", - " \"/tmp/\"+f\"{model_checkpoint}-{model_task}-fast-inf\",\n", - " per_device_eval_batch_size=8,\n", - " dataloader_mode=\"async_rebatched\",\n", - " dataloader_drop_last=True,\n", - " logging_steps=10,\n", - " pod_type=pod_type\n", - ")\n", + "The pipeline expects a **list of strings** directly passed to it in the format: \n", + "```\n", + "questions=[], contexts=[]\n", + "```\n", + "There is no need to tokenize, preprocess, pack or postprocess the data to use the inference pipeline.\n", "\n", - "trainer = IPUTrainer(\n", - " model,\n", - " ipu_config,\n", - " args,\n", - " eval_dataset=packed_inference_dataset\n", - ")" + "As a test, we can load the entire SQuAD validation dataset and perform packed inference using `.predict()` on the text column to generate predictions. Postprocessing samples for SQuAD is done on a sample-by-sample, unbatched basis so this may take a few minutes with or without packing." ] }, { @@ -1105,19 +1091,30 @@ "execution_count": null, "id": "81969920", "metadata": { - "scrolled": false + "tags": [] }, "outputs": [], "source": [ - "trainer.evaluate()" + "import datasets\n", + "dataset = datasets.load_dataset('squad')\n", + "preds = pipeline.predict(questions=dataset['validation']['question'],\n", + " contexts=dataset['validation']['context'])\n", + "\n", + "print(preds.keys())\n", + "print(f\"Number of predictions: {len(preds['predictions'])}\")\n", + "print(f\"Preprocessing time: {preds['preprocessing_time']}s\")\n", + "print(f\"Postprocessing time: {preds['postprocessing_time']}s\")\n", + "print(f\"Throughput: {preds['throughput']} samples/s\")" ] }, { "cell_type": "markdown", "id": "3c77e5ac", - "metadata": {}, + "metadata": { + "tags": [] + }, "source": [ - "Using these simple optimisations and the increase in maximum sequences per pack, we can see a throughput increase to approximately **8000 sequences per second** - remember that to obtain the actual throughput we multiply the packed samples/s by the average packing factor - highlighting the benefits of using packing! " + "There is minimal overhead from tokenizing and packing the dataset, but the speed benefits for inference are evident. Running the above pipeline, we achieve a throughput approximately 6000 samples per second, showing an approximate 2x speed up for SQuAD." ] } ], diff --git a/packed-bert/packedBERT_single_label_text_classification.ipynb b/packed-bert/packedBERT_single_label_text_classification.ipynb index 363a5f8..3729195 100644 --- a/packed-bert/packedBERT_single_label_text_classification.ipynb +++ b/packed-bert/packedBERT_single_label_text_classification.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": { "id": "X4cRE8IbIrIV" @@ -55,7 +54,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "import transformers\n", @@ -77,7 +78,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from huggingface_hub import notebook_login\n", @@ -95,14 +98,15 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "! apt install git-lfs" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": { "id": "rEJBSTyZIrIb" @@ -122,7 +126,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": { "id": "kTCFado4IrIc" @@ -136,7 +139,7 @@ "- [MNLI](https://arxiv.org/abs/1704.05426) (Multi-Genre Natural Language Inference) Determine if a sentence entails, contradicts or is unrelated to a given hypothesis. (This dataset has two versions, one with the validation and test set coming from the same distribution, another called mismatched where the validation and test use out-of-domain data.)\n", "- [MRPC](https://www.microsoft.com/en-us/download/details.aspx?id=52398) (Microsoft Research Paraphrase Corpus) Determine if two sentences are paraphrases from one another or not.\n", "- [QNLI](https://rajpurkar.github.io/SQuAD-explorer/) (Question-answering Natural Language Inference) Determine if the answer to a question is in the second sentence or not. (This dataset is built from the SQuAD dataset.)\n", - "- [QQP](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) (Quora Question Pairs2) Determine if two questions are semantically equivalent or not.\n", + "- [QQP](https://data.quora.com/First-Quora-Dataset-Release-Question-Pairs) (Quora Question Pairs2) Determine if two questions are semantically equivalent or not.\n", "- [RTE](https://aclweb.org/aclwiki/Recognizing_Textual_Entailment) (Recognizing Textual Entailment) Determine if a sentence entails a given hypothesis or not.\n", "- [SST-2](https://nlp.stanford.edu/sentiment/index.html) (Stanford Sentiment Treebank) Determine if the sentence has a positive or negative sentiment.\n", "- [STS-B](http://ixa2.si.ehu.es/stswiki/index.php/STSbenchmark) (Semantic Textual Similarity Benchmark) Determine the similarity of two sentences with a score from 1 to 5.\n", @@ -149,7 +152,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "YZbiBDuGIrId" + "id": "YZbiBDuGIrId", + "tags": [] }, "outputs": [], "source": [ @@ -164,7 +168,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": { "id": "4RRkXuteIrIh" @@ -172,7 +175,7 @@ "source": [ "Let's initialise our training configurations. \n", "\n", - "In this notebook, we are using both data parallelism and pipeline parallelism (see this [tutorial](https://github.com/graphcore/tutorials/tree/master/tutorials/pytorch/efficient_data_loading/walkthrough.ipynb) for more). Therefore the global batch size, which is the actual number of samples used for the weight update, is determined using four factors:\n", + "In this notebook, we are using both data parallelism and pipeline parallelism (see this [tutorial](https://github.com/graphcore/tutorials/blob/master/tutorials/pytorch/efficient_data_loading/walkthrough.ipynb) for more). Therefore the global batch size, which is the actual number of samples used for the weight update, is determined using four factors:\n", "\n", " global batch size = micro_batch_size * gradient accumulation steps * device iterations * replication factor\n", "\n", @@ -198,7 +201,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "zVvslsfMIrIh" + "id": "zVvslsfMIrIh", + "tags": [] }, "outputs": [], "source": [ @@ -223,7 +227,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -234,7 +237,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "import os\n", @@ -265,7 +270,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "IreSlFmlIrIm" + "id": "IreSlFmlIrIm", + "tags": [] }, "outputs": [], "source": [ @@ -286,7 +292,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "scrolled": true + "scrolled": true, + "tags": [] }, "outputs": [], "source": [ @@ -309,7 +316,8 @@ "execution_count": null, "metadata": { "id": "GWiVUF0jIrIv", - "outputId": "35e3ea43-f397-4a54-c90c-f2cf8d36873e" + "outputId": "35e3ea43-f397-4a54-c90c-f2cf8d36873e", + "tags": [] }, "outputs": [], "source": [ @@ -330,11 +338,12 @@ "execution_count": null, "metadata": { "id": "X6HrpprwIrIz", - "outputId": "d7670bc0-42e4-4c09-8a6a-5c018ded7d95" + "outputId": "d7670bc0-42e4-4c09-8a6a-5c018ded7d95", + "tags": [] }, "outputs": [], "source": [ - "dataset[\"train\"][0]" + "dataset[\"train\"][:10]" ] }, { @@ -350,7 +359,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "i3j8APAoIrI3" + "id": "i3j8APAoIrI3", + "tags": [] }, "outputs": [], "source": [ @@ -380,7 +390,8 @@ "execution_count": null, "metadata": { "id": "SZy5tRB_IrI7", - "outputId": "ba8f2124-e485-488f-8c0c-254f34f24f13" + "outputId": "ba8f2124-e485-488f-8c0c-254f34f24f13", + "tags": [] }, "outputs": [], "source": [ @@ -401,7 +412,8 @@ "execution_count": null, "metadata": { "id": "5o4rUteaIrI_", - "outputId": "18038ef5-554c-45c5-e00a-133b02ec10f1" + "outputId": "18038ef5-554c-45c5-e00a-133b02ec10f1", + "tags": [] }, "outputs": [], "source": [ @@ -422,7 +434,8 @@ "execution_count": null, "metadata": { "id": "6XN1Rq0aIrJC", - "outputId": "a4405435-a8a9-41ff-9f79-a13077b587c7" + "outputId": "a4405435-a8a9-41ff-9f79-a13077b587c7", + "tags": [] }, "outputs": [], "source": [ @@ -483,7 +496,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "eXNLu_-nIrJI" + "id": "eXNLu_-nIrJI", + "tags": [] }, "outputs": [], "source": [ @@ -515,7 +529,8 @@ "execution_count": null, "metadata": { "id": "a5hBlsrHIrJL", - "outputId": "acdaa98a-a8cd-4a20-89b8-cc26437bbe90" + "outputId": "acdaa98a-a8cd-4a20-89b8-cc26437bbe90", + "tags": [] }, "outputs": [], "source": [ @@ -537,7 +552,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "fyGdtK9oIrJM" + "id": "fyGdtK9oIrJM", + "tags": [] }, "outputs": [], "source": [ @@ -569,7 +585,8 @@ "execution_count": null, "metadata": { "id": "19GG646uIrJO", - "outputId": "0cb4a520-817e-4f92-8de8-bb45df367657" + "outputId": "0cb4a520-817e-4f92-8de8-bb45df367657", + "tags": [] }, "outputs": [], "source": [ @@ -597,7 +614,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "vc0BSBLIIrJQ" + "id": "vc0BSBLIIrJQ", + "tags": [] }, "outputs": [], "source": [ @@ -623,7 +641,8 @@ "execution_count": null, "metadata": { "id": "-b70jh26IrJS", - "outputId": "acd3a42d-985b-44ee-9daa-af5d944ce1d9" + "outputId": "acd3a42d-985b-44ee-9daa-af5d944ce1d9", + "tags": [] }, "outputs": [], "source": [ @@ -644,7 +663,8 @@ "execution_count": null, "metadata": { "id": "DDtsaJeVIrJT", - "outputId": "aa4734bf-4ef5-4437-9948-2c16363da719" + "outputId": "aa4734bf-4ef5-4437-9948-2c16363da719", + "tags": [] }, "outputs": [], "source": [ @@ -680,7 +700,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "max_seq_per_pack = 6" @@ -696,7 +718,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "num_labels = 3 if task.startswith(\"mnli\") else 1 if task==\"stsb\" else 2\n", @@ -711,7 +735,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -738,7 +761,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from utils.packing.dataset_creator import PackedDatasetCreator\n", @@ -787,7 +812,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", @@ -832,7 +859,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "scrolled": true + "scrolled": true, + "tags": [] }, "outputs": [], "source": [ @@ -851,7 +879,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "scrolled": true + "scrolled": true, + "tags": [] }, "outputs": [], "source": [ @@ -893,7 +922,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from transformers import AutoConfig\n", @@ -915,7 +946,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "scrolled": true + "scrolled": true, + "tags": [] }, "outputs": [], "source": [ @@ -933,7 +965,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": { "id": "CczA5lJlIrJX" @@ -953,7 +984,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "scrolled": false + "tags": [] }, "outputs": [], "source": [ @@ -988,7 +1019,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "model.half()" @@ -1004,7 +1037,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "metric_name = \"pearson\" if task == \"stsb\" else \"matthews_correlation\" if task == \"cola\" else \"accuracy\"\n", @@ -1039,7 +1074,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from optimum.graphcore import IPUConfig, IPUTrainer, IPUTrainingArguments\n", @@ -1070,7 +1107,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "scrolled": true + "scrolled": true, + "tags": [] }, "outputs": [], "source": [ @@ -1087,7 +1125,7 @@ " lr_scheduler_type = \"cosine\",\n", " metric_for_best_model=metric_name,\n", " dataloader_drop_last=True,\n", - " dataloader_mode=\"async_rebatched\",\n", + " # dataloader_mode=\"async_rebatched\",\n", " logging_steps=1,\n", " pod_type=pod_type,\n", " gradient_accumulation_steps=gradient_accumulation_steps,\n", @@ -1143,9 +1181,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [], "source": [ "trainer.evaluate()" @@ -1198,154 +1234,100 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Faster inference:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This section demonstrates how to perform faster, batched inference with a large number of samples using packing." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "When training, the packing factor does affect the convergence the same way as a large increase in batch size would do. However, for inference, we are free to use a bigger packing factor to speed it up.\n", - "Let's try it on `sst2` with `max_seq_per_pack = 12`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "max_seq_per_pack = 12" + "## Fast batched inference" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "To have enough examples, we will reuse the training set." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "dataset = load_dataset(\"glue\", \"sst2\")\n", - "encoded_dataset = dataset.map(preprocess_function, batched=True)\n", - "inference_dataset = encoded_dataset['train'] # Use the train set again, to have enough examples\n", + "Packing can also be used for inference, particularly for performing inference for workloads. This section demonstrates how to perform faster, batched inference with a large number of samples using a super-easy custom pipeline which batches and packs your input data, performs inference and returns postprocessed predictions. \n", "\n", - "inference_packed_dataset = PackedDatasetCreator(\n", - " tokenized_dataset = encoded_dataset['train'],\n", - " max_sequence_length = max_seq_length,\n", - " max_sequences_per_pack = max_seq_per_pack,\n", - " inference = True,\n", - " problem_type = problem_type,\n", - " custom_label_key='label').create()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see that the average packing factor `6.7` is not close to the maximum now (12), this is still an improvement compared to the previous `5.7`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's also modify the configuration of the model for inference. For speed up, we can us a single IPU and 4 replicas by changing `layers_per_ipu` , `inference_replication_factor` and `ipus_per_replica` and also use a larger `batch-size`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ipu_config.layers_per_ipu = [12]\n", - "ipu_config.inference_device_iterations = 32\n", - "ipu_config.inference_replication_factor = 4\n", - "ipu_config.ipus_per_replica = 1" + "For the pipeline, we need to import it, and initialise a few essential parameters.\n", + "\n", + "The `model` is the model checkpoint, we are going to use the locally saved checkpoint generated from training `sst2`. The `executable_cache_dir`, `problem_type`, `max_seq_length` must be specified. To return predictions organised by class names, the class names for your output must be passed to `label_categories`. \n", + "\n", + "The pipeline will automatically determine your model's IPU config, given that the checkpoint was trained using Optimum Graphcore, which will be the case for the model fine-tuned in this notebook.\n", + "\n", + "In this example, we pre-load the IPUConfig and modify some of the default parameters to get the best performance out of inference and leverage the benefits of IPU parallelism. The micro-batch size can also be specified, for which the default is 1.\n", + "\n", + "When training, the packing factor affects the convergence the same way as a large increase in batch size would do. However, for inference, we are free to use a bigger packing factor to speed it up. Let's try it with `max_seq_per_pack = 12`.\n", + "\n", + "**Note:** Packing brings huge benefits for performing inference on large amounts of data. For small scale inference tasks, such as those which more suit sequential inference on a single un-batched input, the generic Optimum Graphcore `TextClassificationPipeline` may be prefered. This won't affect fine-tuning, the weights generated from fine-tuning using packing will work just the same!" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "To test inference throughput, we can just use a default checkpoint to run the model and evaluate the speed of packing. If you saved the model locally or on the hub earlier, you can replace `model_checkpoint` with the path to your model to perform inference on the fine-tuned weights!" + "Lets initialise the `PackedBertTextClassificationPipeline`." ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "scrolled": true + "tags": [] }, "outputs": [], "source": [ - "model_checkpoint = \"bert-base-uncased\"\n", + "from pipeline.packed_bert import PackedBertTextClassificationPipeline\n", + "from optimum.graphcore import IPUConfig\n", "\n", - "# Load checkpoint locally:\n", - "# model_checkpoint = f\"./{model_name}-{task}\"\n", + "model = 'Graphcore/bert-base-uncased' # This is set as default, comment this line to run your fine-tuned model from the lines below!\n", + "# model = \"./\"+f\"{model_name}-{task}\" # uncomment to load your fine-tuned model from disk\n", + "# model = 'your_username/{model_name}-{task}' # uncomment this and use your username to load from Hugging Face Hub\n", "\n", - "# Load from Huggingface Hub instead:\n", - "# model_checkpoint = '/{model_checkpoint}-{model_task}'\n", + "inference_boosted_ipu_config = IPUConfig.from_pretrained(model, \n", + " inference_device_iterations=32,\n", + " inference_replication_factor=4,\n", + " ipus_per_replica=1,\n", + " layers_per_ipu=[12]\n", + " )\n", "\n", - "model = PipelinedPackedBertForSequenceClassification.from_pretrained(model_checkpoint, config=config)" + "pipeline = PackedBertTextClassificationPipeline(\n", + " model = model,\n", + " executable_cache_dir = executable_cache_dir,\n", + " problem_type='single_label_classification',\n", + " max_seq_per_pack=12,\n", + " max_seq_length=max_seq_length,\n", + " ipu_config=inference_boosted_ipu_config,\n", + " micro_batch_size=8,\n", + " label_categories=['positive','negative']\n", + ")" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "args = IPUTrainingArguments(\n", - " \"/tmp/\"+f\"{model_name}-{task}-fast-inf\",\n", - " per_device_eval_batch_size=8,\n", - " dataloader_mode=\"async_rebatched\",\n", - " dataloader_drop_last=True,\n", - " logging_steps=10,\n", - " pod_type=pod_type\n", - ")\n", + "The pipeline expects a **list of strings** directly passed to it. There is no need to tokenize, preprocess, pack or postprocess the data to use the inference pipeline.\n", "\n", - "trainer = IPUTrainer(\n", - " model,\n", - " ipu_config,\n", - " args,\n", - " eval_dataset=inference_packed_dataset,\n", - " compute_metrics=compute_metrics\n", - ")" + "As a test, we can load the entire `sst2` dataset and perform packed inference using `.predict()` on the text column to generate predictions. " ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ - "trainer.evaluate()" + "import datasets\n", + "dataset = datasets.load_dataset('sst2')\n", + "preds = pipeline.predict(dataset['train']['sentence'])\n", + "\n", + "print(preds.keys())\n", + "print(f\"Number of predictions: {len(preds['predictions'])}\")\n", + "print(f\"Preprocessing time: {preds['preprocessing_time']}s\")\n", + "print(f\"Postprocessing time: {preds['postprocessing_time']}s\")\n", + "print(f\"Throughput: {preds['throughput']}samples/s\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "As before, to get a correct throughput estimation we need to multiply `eval_samples_per_second` by the average packing factor (6.72). For example, if the inference throughput over a run is `4200 samples/s`, the actual throughput is `4200 * 6.72 = 28224 samples/s` " + "There is minimal overhead from tokenizing and packing the dataset, but the speed benefits are evident. Running the above pipeline, we achieve a throughput approximately 35000 samples per second, demonstrating the huge time benefit you can achieve by using packing!" ] } ], @@ -1378,5 +1360,5 @@ } }, "nbformat": 4, - "nbformat_minor": 1 + "nbformat_minor": 4 } diff --git a/packed-bert/pipeline/__init__.py b/packed-bert/pipeline/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packed-bert/pipeline/packed_bert.py b/packed-bert/pipeline/packed_bert.py new file mode 100644 index 0000000..f1fd511 --- /dev/null +++ b/packed-bert/pipeline/packed_bert.py @@ -0,0 +1,436 @@ +import logging +import time +from typing import Dict, List + +import numpy as np +import torch +from datasets import Dataset + +import poptorch +from models.modeling_bert_packed import ( + PipelinedPackedBertForQuestionAnswering, + PipelinedPackedBertForSequenceClassification, +) +from optimum.graphcore import IPUConfig +from scipy.special import softmax +from transformers import AutoConfig, AutoTokenizer +from transformers.data.data_collator import default_data_collator +from utils.packing.dataset_creator import PackedDatasetCreator +from utils.packing.dataset_templates import PackedQuestionAnsweringDataset +from utils.packing.qa_utils import postprocess_packed_qa_predictions, preprocess_packed_qa + + +logger = logging.getLogger("") + + +def get_poplar_executor(model, ipu_config, batch, detach=False): + ipu_options = ipu_config.to_options(for_inference=True) + model.ipu_config = ipu_config + + if isinstance(model, poptorch.PoplarExecutor): + print("Model already wrapped - nothing to do.") + return model + try: + model.deparallelize() + except: + pass + + ipu_model = poptorch.inferenceModel(model.eval().parallelize(), ipu_options) + + ipu_model.compile(**batch) + + if detach: + ipu_model.detachFromDevice() + + return ipu_model + + +def prepare_inference_dataloader(ipu_config, dataset, batch_size, mode="async_rebatched"): + return poptorch.DataLoader( + ipu_config.to_options(for_inference=True), + dataset, + batch_size=batch_size, + shuffle=False, # Must be false, retained order important for batched inference + drop_last=False, # Must be false, we pad up to global batch size in inference pipeline to avoid any division error + mode=mode, + collate_fn=default_data_collator, + ) + + +class PackedBertTextClassificationPipeline: + """ + Packed classification pipeline: + + Batched inference pipeline for packed BERT text classification with multi/single label. Wraps all preprocessing and model for inference, executes on text inputs in format `questions, contexts` of any size, proceeds to batch according to checkpoint or as per custom IPU configs, and packs data. Performs inference on PipelinedPackedBertForSequenceClassification. Returns postprocessed predictions in same order as input data. + """ + + def __init__( + self, + model, + executable_cache_dir: str = "./exe_cache", + problem_type: str = "single_label_classification", + max_seq_per_pack: int = 12, + max_seq_length: int = 384, + ipu_config: IPUConfig = None, + micro_batch_size: int = 1, + dataloader_mode: str = "async_rebatched", + detach_model_after_compile: bool = False, + pretrained_tokenizer: str = "bert-base-uncased", + label_categories: List = [], + ) -> None: + self.model_ckpt = model + self.problem_type = problem_type + self.max_seq_per_pack = max_seq_per_pack + self.max_seq_length = max_seq_length + + self.pretrained_tokenizer = pretrained_tokenizer + self.dataloader_mode = dataloader_mode + self.detach_model_after_post_compile = detach_model_after_compile + self.executable_cache_dir = executable_cache_dir + + self.micro_batch_size = micro_batch_size + self.sentence_2_key = None + self.label_categories = label_categories + + if not ipu_config: + try: + logger.info("Attempting loading IPUConfig from model checkpoint:") + self.ipu_config = IPUConfig.from_pretrained( + self.model_ckpt, executable_cache_dir=self.executable_cache_dir + ) + except: + logger.warn( + "Loading default config: 'Graphcore/bert-base-uncased' - because no IPUConfig found in model folder." + ) + self.ipu_config = IPUConfig.from_pretrained( + "Graphcore/bert-base-uncased", executable_cache_dir=self.executable_cache_dir + ) + else: + self.ipu_config = ipu_config + + self.gbs = ( + self.ipu_config.inference_device_iterations + * self.ipu_config.inference_replication_factor + * self.micro_batch_size + ) + + try: + logger.info("Attempting loading tokenizer from model checkpoint") + self.tokenizer = AutoTokenizer.from_pretrained(self.model_ckpt, use_fast=True) + except: + logger.warn("Loading tokenizer from defined because no pretrained tokenizer found in model folder.") + self.tokenizer = AutoTokenizer.from_pretrained(self.pretrained_tokenizer, use_fast=True) + + config = AutoConfig.from_pretrained(self.model_ckpt) + config.max_sequences_per_pack = self.max_seq_per_pack + config.problem_type = self.problem_type + + self.model = ( + PipelinedPackedBertForSequenceClassification(config).from_pretrained(self.model_ckpt, config=config).half() + ) + + compile_data = Dataset.from_dict({"text": ["I am a dummy sentence for compilation."]}) + + enc_compile_data = compile_data.map(self.preprocess_function, batched=True) + + pck_compile_data = PackedDatasetCreator( + tokenized_dataset=enc_compile_data, + max_sequence_length=self.max_seq_length, + max_sequences_per_pack=self.max_seq_per_pack, + inference=True, + pad_to_global_batch_size=True, + global_batch_size=self.gbs, + problem_type=self.problem_type, + ).create() + + c_dataloader = prepare_inference_dataloader( + self.ipu_config, pck_compile_data, self.micro_batch_size, self.dataloader_mode + ) + + c_batch = next(iter(c_dataloader)) + + # Remove custom column for compile - autoignored in optimum, manually ignored in predict + c_batch.pop("example_ids", None) + + self.poplar_executor = get_poplar_executor(self.model, self.ipu_config, c_batch) + + def preprocess_function(self, examples): + if self.sentence_2_key: + return self.tokenizer( + examples["text"], examples["text_2"], truncation=True, max_length=self.max_seq_length + ) + else: + return self.tokenizer(examples["text"], truncation=True, max_length=self.max_seq_length) + + def postprocess_preds(self, logits, ids): + ids = torch.concat(ids) + mask = ids != -100 + ids = ids[mask] + + if self.problem_type == "multi_label_classification": + pred_scores = softmax(torch.concat(logits)[mask, :].numpy().astype("float32"), axis=1) + if self.problem_type == "single_label_classification": + pred_scores = softmax(torch.concat(logits)[mask, :].numpy().astype("float32"), axis=1) + + pred_scores = pred_scores[np.argsort(ids)] + + return pred_scores + + def predict(self, sentence_1, sentence_2=None): + self.sentence_2_key = sentence_2 + + prep_st = time.time() + + data_dict = {"text": sentence_1} + if sentence_2: + data_dict["text_2"] = sentence_2 + + dataset = Dataset.from_dict(data_dict) + enc_data = dataset.map(self.preprocess_function, batched=True) + + # Pack the inputs + packed_data = PackedDatasetCreator( + tokenized_dataset=enc_data, + max_sequence_length=self.max_seq_length, + max_sequences_per_pack=self.max_seq_per_pack, + inference=True, + pad_to_global_batch_size=True, + global_batch_size=self.gbs, + problem_type=self.problem_type, + ).create() + + dataloader = prepare_inference_dataloader( + self.ipu_config, packed_data, self.micro_batch_size, self.dataloader_mode + ) + + example_ids = [] + outputs = [] + + # Process the model to return logits + prep_time = time.time() - prep_st + + model_st = time.time() + for batch in iter(dataloader): + logits = self.poplar_executor( + input_ids=batch["input_ids"], + attention_mask=batch["attention_mask"], + token_type_ids=batch["token_type_ids"], + position_ids=batch["position_ids"], + ) + + ids = batch["example_ids"] + outputs.append(logits.view(ids.shape[0], self.max_seq_per_pack, -1)) + example_ids.append(ids) + + model_en = time.time() + model_time = model_en - model_st + tput = len(sentence_1) / (model_time) + + # Postprocess predictions to preserve order + post_st = time.time() + final_preds = self.postprocess_preds(outputs, example_ids) + + if len(self.label_categories) == final_preds.shape[-1]: + final_preds = {k: dict(list(zip(self.label_categories, v))) for k, v in enumerate(final_preds)} + else: + final_preds = {{n: k[n] for n in k} for k in final_preds} + + post_proc_time = time.time() - post_st + + return { + "predictions": final_preds, + "throughput": tput, + "inference_total_time": model_time, + "preprocessing_time": prep_time, + "postprocessing_time": post_proc_time, + } + + +class PackedBertQuestionAnsweringPipeline: + """ + Packed Question-answering pipeline: + + Batched inference pipeline for packed BERT question answering. Wraps all preprocessing and model for inference, executes on text inputs in format `questions, contexts` of any size, proceeds to batch according to checkpoint or as per custom IPU configs, and packs data. Performs inference on PipelinedPackedBertForQuestionAnswering. Returns postprocessed predictions in same order as input data. + """ + + def __init__( + self, + model, + executable_cache_dir: str = "./exe_cache", + problem_type: str = "question_answering", + max_seq_per_pack: int = 12, + max_seq_length: int = 384, + pretrained_tokenizer: str = "bert-base-uncased", + ipu_config: str = None, + micro_batch_size: int = 1, + dataloader_mode: str = "async_rebatched", + detach_model_after_compile: bool = False, + ) -> None: + self.problem_type = problem_type + self.max_seq_per_pack = max_seq_per_pack + self.max_seq_length = max_seq_length + + self.model_ckpt = model + self.pretrained_tokenizer = pretrained_tokenizer + self.dataloader_mode = dataloader_mode + self.detach_model_after_post_compile = detach_model_after_compile + self.executable_cache_dir = executable_cache_dir + self.micro_batch_size = micro_batch_size + + if not ipu_config: + try: + logger.info("Attempting loading IPUConfig from model checkpoint:") + self.ipu_config = IPUConfig.from_pretrained( + self.model_ckpt, executable_cache_dir=self.executable_cache_dir + ) + except: + logger.warn( + "Loading default config: 'Graphcore/bert-base-uncased' - because no IPUConfig found in model folder." + ) + self.ipu_config = IPUConfig.from_pretrained( + "Graphcore/bert-base-uncased", executable_cache_dir=self.executable_cache_dir + ) + else: + self.ipu_config = ipu_config + + self.gbs = ( + self.ipu_config.inference_device_iterations + * self.ipu_config.inference_replication_factor + * self.micro_batch_size + ) + + try: + self.tokenizer = AutoTokenizer.from_pretrained(self.model_ckpt, use_fast=True) + except: + self.tokenizer = AutoTokenizer.from_pretrained(self.pretrained_tokenizer, use_fast=True) + + config = AutoConfig.from_pretrained(self.model_ckpt) + config.max_sequences_per_pack = self.max_seq_per_pack + config.problem_type = self.problem_type + + self.model = ( + PipelinedPackedBertForQuestionAnswering(config).from_pretrained(self.model_ckpt, config=config).half() + ) + + compile_data = Dataset.from_dict( + { + "id": np.array([str(i) for i in range(self.gbs)]).astype(" Date: Thu, 16 Mar 2023 16:57:12 +0000 Subject: [PATCH 06/11] changing default checkpoint for ml seq cls for tests --- packed-bert/packedBERT_multi_label_text_classification.ipynb | 4 ++-- packed-bert/pipeline/packed_bert.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packed-bert/packedBERT_multi_label_text_classification.ipynb b/packed-bert/packedBERT_multi_label_text_classification.ipynb index ef4b706..b5dcbdd 100644 --- a/packed-bert/packedBERT_multi_label_text_classification.ipynb +++ b/packed-bert/packedBERT_multi_label_text_classification.ipynb @@ -1194,11 +1194,11 @@ "\n", "from optimum.graphcore import IPUConfig\n", "\n", - "model = 'Graphcore/bert-base-uncased' # This is set as default, comment this line to run your fine-tuned model from the lines below!\n", + "model = 'arsalanu/bert-base-uncased-go_emotions' # This is set as default, comment this line to run your fine-tuned model from the lines below!\n", "# model = \"./\"+f\"{model_name}-{model_task}\" # uncomment to load your fine-tuned model from disk\n", "# model = 'your_username/{model_name}-{model_task}' # uncomment this and use your username to load from Hugging Face Hub\n", "\n", - "inference_boosted_ipu_config = IPUConfig.from_pretrained(model, \n", + "inference_boosted_ipu_config = IPUConfig.from_pretrained(model, \n", " inference_device_iterations=32,\n", " inference_replication_factor=4,\n", " ipus_per_replica=1,\n", diff --git a/packed-bert/pipeline/packed_bert.py b/packed-bert/pipeline/packed_bert.py index f1fd511..cdb2679 100644 --- a/packed-bert/pipeline/packed_bert.py +++ b/packed-bert/pipeline/packed_bert.py @@ -107,6 +107,8 @@ def __init__( ) else: self.ipu_config = ipu_config + if self.executable_cache_dir is not None: + self.ipu_config.executable_cache_dir = self.executable_cache_dir self.gbs = ( self.ipu_config.inference_device_iterations @@ -292,6 +294,8 @@ def __init__( ) else: self.ipu_config = ipu_config + if self.executable_cache_dir is not None: + self.ipu_config.executable_cache_dir = self.executable_cache_dir self.gbs = ( self.ipu_config.inference_device_iterations From a308ad1b712a4cb5fc395869d2b8ab0e4ee9cd1e Mon Sep 17 00:00:00 2001 From: Arsalan Uddin Date: Thu, 16 Mar 2023 17:07:18 +0000 Subject: [PATCH 07/11] name change for var in comment --- packed-bert/packedBERT_question_answering.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packed-bert/packedBERT_question_answering.ipynb b/packed-bert/packedBERT_question_answering.ipynb index 396b22d..9b96e36 100644 --- a/packed-bert/packedBERT_question_answering.ipynb +++ b/packed-bert/packedBERT_question_answering.ipynb @@ -1052,8 +1052,8 @@ "from pipeline.packed_bert import PackedBertQuestionAnsweringPipeline\n", "\n", "model = 'Graphcore/bert-base-uncased' # This is set as default, comment this line to run your fine-tuned model from the lines below!\n", - "# model = \"./\"+f\"{model_checkpoint}-{task}\" # uncomment to load your fine-tuned model from disk\n", - "# model = 'your_username/{model_checkpoint}-{task}' # uncomment this and use your username to load from Hugging Face Hub\n", + "# model = \"./\"+f\"{model_checkpoint}-{model_task}\" # uncomment to load your fine-tuned model from disk\n", + "# model = 'your_username/{model_checkpoint}-{model_task}' # uncomment this and use your username to load from Hugging Face Hub\n", "\n", "inference_boosted_ipu_config = IPUConfig.from_pretrained(model, \n", " inference_device_iterations=32,\n", From b1156ae57f754dd5e48dcc6110ffd3e574719e49 Mon Sep 17 00:00:00 2001 From: Arsalan Uddin Date: Thu, 16 Mar 2023 17:14:54 +0000 Subject: [PATCH 08/11] commenting push to hub --- .../packedBERT_multi_label_text_classification.ipynb | 2 +- packed-bert/packedBERT_question_answering.ipynb | 2 +- .../packedBERT_single_label_text_classification.ipynb | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packed-bert/packedBERT_multi_label_text_classification.ipynb b/packed-bert/packedBERT_multi_label_text_classification.ipynb index b5dcbdd..5e507e0 100644 --- a/packed-bert/packedBERT_multi_label_text_classification.ipynb +++ b/packed-bert/packedBERT_multi_label_text_classification.ipynb @@ -1072,7 +1072,7 @@ }, "outputs": [], "source": [ - "trainer.push_to_hub()" + "# trainer.push_to_hub()" ] }, { diff --git a/packed-bert/packedBERT_question_answering.ipynb b/packed-bert/packedBERT_question_answering.ipynb index 9b96e36..fef9b73 100644 --- a/packed-bert/packedBERT_question_answering.ipynb +++ b/packed-bert/packedBERT_question_answering.ipynb @@ -944,7 +944,7 @@ }, "outputs": [], "source": [ - "trainer.push_to_hub()" + "# trainer.push_to_hub()" ] }, { diff --git a/packed-bert/packedBERT_single_label_text_classification.ipynb b/packed-bert/packedBERT_single_label_text_classification.ipynb index 3729195..fb2e369 100644 --- a/packed-bert/packedBERT_single_label_text_classification.ipynb +++ b/packed-bert/packedBERT_single_label_text_classification.ipynb @@ -1204,7 +1204,7 @@ }, "outputs": [], "source": [ - "trainer.push_to_hub()" + "# trainer.push_to_hub()" ] }, { @@ -1337,7 +1337,7 @@ "provenance": [] }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -1355,7 +1355,7 @@ }, "vscode": { "interpreter": { - "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" } } }, From 0d76256d537425ffe4e3f25f5b9945ea35d3fc26 Mon Sep 17 00:00:00 2001 From: Arsalan Uddin Date: Fri, 17 Mar 2023 16:12:53 +0000 Subject: [PATCH 09/11] changing checkpoint directory to env var --- .../packedBERT_multi_label_text_classification.ipynb | 9 ++++++--- packed-bert/packedBERT_question_answering.ipynb | 11 +++++++---- .../packedBERT_single_label_text_classification.ipynb | 9 ++++++--- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/packed-bert/packedBERT_multi_label_text_classification.ipynb b/packed-bert/packedBERT_multi_label_text_classification.ipynb index 5e507e0..fbc3c5b 100644 --- a/packed-bert/packedBERT_multi_label_text_classification.ipynb +++ b/packed-bert/packedBERT_multi_label_text_classification.ipynb @@ -1090,7 +1090,11 @@ }, "outputs": [], "source": [ - "trainer.save_model(\"./\"+f\"{model_name}-{model_task}\")" + "from pathlib import Path\n", + "\n", + "checkpoint_directory = Path(os.getenv('CHECKPOINT_DIR', '/tmp')) / f\"{model_name}-{model_task}\"\n", + "\n", + "trainer.save_model(checkpoint_directory)" ] }, { @@ -1194,8 +1198,7 @@ "\n", "from optimum.graphcore import IPUConfig\n", "\n", - "model = 'arsalanu/bert-base-uncased-go_emotions' # This is set as default, comment this line to run your fine-tuned model from the lines below!\n", - "# model = \"./\"+f\"{model_name}-{model_task}\" # uncomment to load your fine-tuned model from disk\n", + "model = checkpoint_directory\n", "# model = 'your_username/{model_name}-{model_task}' # uncomment this and use your username to load from Hugging Face Hub\n", "\n", "inference_boosted_ipu_config = IPUConfig.from_pretrained(model, \n", diff --git a/packed-bert/packedBERT_question_answering.ipynb b/packed-bert/packedBERT_question_answering.ipynb index fef9b73..e652c74 100644 --- a/packed-bert/packedBERT_question_answering.ipynb +++ b/packed-bert/packedBERT_question_answering.ipynb @@ -964,7 +964,11 @@ }, "outputs": [], "source": [ - "trainer.save_model(f\"./{model_checkpoint}-{model_task}\")" + "from pathlib import Path\n", + "\n", + "checkpoint_directory = Path(os.getenv('CHECKPOINT_DIR', '/tmp')) / f\"{model_checkpoint}-{model_task}\"\n", + "\n", + "trainer.save_model(checkpoint_directory)" ] }, { @@ -1051,8 +1055,7 @@ "source": [ "from pipeline.packed_bert import PackedBertQuestionAnsweringPipeline\n", "\n", - "model = 'Graphcore/bert-base-uncased' # This is set as default, comment this line to run your fine-tuned model from the lines below!\n", - "# model = \"./\"+f\"{model_checkpoint}-{model_task}\" # uncomment to load your fine-tuned model from disk\n", + "model = checkpoint_directory\n", "# model = 'your_username/{model_checkpoint}-{model_task}' # uncomment this and use your username to load from Hugging Face Hub\n", "\n", "inference_boosted_ipu_config = IPUConfig.from_pretrained(model, \n", @@ -1063,7 +1066,7 @@ " )\n", "\n", "pipeline = PackedBertQuestionAnsweringPipeline(\n", - " model = f\"./{model_checkpoint}-{model_task}\",\n", + " model = model,\n", " executable_cache_dir = executable_cache_dir,\n", " max_seq_per_pack=12,\n", " max_seq_length=max_seq_length,\n", diff --git a/packed-bert/packedBERT_single_label_text_classification.ipynb b/packed-bert/packedBERT_single_label_text_classification.ipynb index fb2e369..3cda26b 100644 --- a/packed-bert/packedBERT_single_label_text_classification.ipynb +++ b/packed-bert/packedBERT_single_label_text_classification.ipynb @@ -1220,7 +1220,11 @@ "metadata": {}, "outputs": [], "source": [ - "trainer.save_model(\"./\"+f\"{model_name}-{task}\")" + "from pathlib import Path\n", + "\n", + "checkpoint_directory = Path(os.getenv('CHECKPOINT_DIR', '/tmp')) / f\"{model_name}-{task}\"\n", + "\n", + "trainer.save_model(checkpoint_directory)" ] }, { @@ -1274,8 +1278,7 @@ "from pipeline.packed_bert import PackedBertTextClassificationPipeline\n", "from optimum.graphcore import IPUConfig\n", "\n", - "model = 'Graphcore/bert-base-uncased' # This is set as default, comment this line to run your fine-tuned model from the lines below!\n", - "# model = \"./\"+f\"{model_name}-{task}\" # uncomment to load your fine-tuned model from disk\n", + "model = checkpoint_directory \n", "# model = 'your_username/{model_name}-{task}' # uncomment this and use your username to load from Hugging Face Hub\n", "\n", "inference_boosted_ipu_config = IPUConfig.from_pretrained(model, \n", From a676a13321a8a36164c7907d405c4fa0a1c62a40 Mon Sep 17 00:00:00 2001 From: Arsalan Uddin Date: Mon, 20 Mar 2023 17:18:48 +0000 Subject: [PATCH 10/11] updated .gradient files with symlinks and prepare-datasets config + readjusted paths in notebooks --- .gradient/prepare-datasets.sh | 25 ++++++++++++++++ .gradient/settings.yaml | 29 +++++++++++++++++++ ...BERT_multi_label_text_classification.ipynb | 14 ++++----- .../packedBERT_question_answering.ipynb | 13 ++++----- ...ERT_single_label_text_classification.ipynb | 15 +++++----- 5 files changed, 74 insertions(+), 22 deletions(-) diff --git a/.gradient/prepare-datasets.sh b/.gradient/prepare-datasets.sh index b398f74..4e9ac0b 100755 --- a/.gradient/prepare-datasets.sh +++ b/.gradient/prepare-datasets.sh @@ -40,6 +40,30 @@ echo "Starting preparation of datasets" # symlink exe_cache files exe_cache_source_dir="${PUBLIC_DATASET_DIR}/poplar-executables-hf-3-1" symlink-public-resources "${exe_cache_source_dir}" $POPLAR_EXECUTABLE_CACHE_DIR + +# packed bert executables +packed_sl_exe_cache_source_dir="${PUBLIC_DATASET_DIR}/packed_bert_slseqcls_exe_cache/packed_bert_slseqcls" +symlink-public-resources "${packed_exe_cache_source_dir}" "${POPLAR_EXECUTABLE_CACHE_DIR}/packed_bert_slseqcls_exe_cache" +packed_ml_exe_cache_source_dir="${PUBLIC_DATASET_DIR}/packed_bert_mlseqcls_exe_cache/packed_bert_mlseqcls" +symlink-public-resources "${packed_exe_cache_source_dir}" "${POPLAR_EXECUTABLE_CACHE_DIR}/packed_bert_mlseqcls_exe_cache" +packed_qa_exe_cache_source_dir="${PUBLIC_DATASET_DIR}/packed_bert_qa_exe_cache/packed_bert_squad" +symlink-public-resources "${packed_exe_cache_source_dir}" "${POPLAR_EXECUTABLE_CACHE_DIR}/packed_bert_qa_exe_cache" + +# packed bert datasets +packed_sl_dataset_source_dir="${PUBLIC_DATASET_DIR}/packed_bert_slseqcls_dataset_cache" +symlink-public-resources "${packed_exe_cache_source_dir}" "${HF_DATASETS}/packed_bert_slseqcls_dataset_cache" +packed_ml_dataset_source_dir="${PUBLIC_DATASET_DIR}/packed_bert_mlseqcls_dataset_cache" +symlink-public-resources "${packed_exe_cache_source_dir}" "${POPLAR_EXECUTABLE_CACHE_DIR}/packed_bert_mlseqcls_dataset_cache" +packed_qa_dataset_source_dir="${PUBLIC_DATASET_DIR}/packed_bert_qa_dataset_cache" +symlink-public-resources "${packed_exe_cache_source_dir}" "${POPLAR_EXECUTABLE_CACHE_DIR}/packed_bert_qa_dataset_cache" + +# packed bert inference checkpoints +symlink-public-resources "${PUBLIC_DATASET_DIR}/bert-base-uncased-sst2" "${CHECKPOINT_DIR}/bert-base-uncased-sst2" +symlink-public-resources "${PUBLIC_DATASET_DIR}/bert-base-uncased-go_emotions" "${CHECKPOINT_DIR}/bert-base-uncased-go_emotions" +symlink-public-resources "${PUBLIC_DATASET_DIR}/bert-base-uncased-squad" "${CHECKPOINT_DIR}/bert-base-uncased-squad" + + + # symlink HF datasets HF_DATASETS="conll2003 glue imagefolder librispeech_asr squad swag wikitext wmt16 xsum" for dataset in ${HF_DATASETS}; do @@ -48,6 +72,7 @@ for dataset in ${HF_DATASETS}; do done # Image classification dataset symlink-public-resources "${PUBLIC_DATASET_DIR}/dfki-sentinel-eurosat" "${DATASET_DIR}/dfki-sentinel-eurosat" + # pre-install the correct version of optimum for this release python -m pip install "optimum-graphcore>=0.5, <0.6" diff --git a/.gradient/settings.yaml b/.gradient/settings.yaml index 3a02448..9c93cc5 100644 --- a/.gradient/settings.yaml +++ b/.gradient/settings.yaml @@ -40,3 +40,32 @@ integrations: dfki-sentinel-eurosat: type: dataset ref: paperspace/ds8p6sv96fl1att:k5j4cob + bert-base-uncased-sst2: + type: dataset + ref: paperspace/dskrqljie6pti8y:mfqq5qk + bert-base-uncased-go_emotions: + type: dataset + ref: paperspace/dsz2f8usk60xbos:n3h8ko3 + bert-base-uncased-squad: + type: dataset + ref: paperspace/ds9ogwc0fbfh799:3mv59lg + packed_bert_slseqcls_exe_cache: + type: dataset + ref: paperspace/dsfg0gcuqbr0pfc:0pss84k + packed_bert_mlseqcls_exe_cache: + type: dataset + ref: paperspace/dsevh3ol36qzpz2:1yme9yi + packed_bert_qa_exe_cache: + type: dataset + ref: paperspace/dsson0ib8byvqpf:tcgts2v + packed_bert_slseqcls_dataset_cache: + type: dataset + ref: paperspace/dsuuz3dih9su40i:npvb833 + packed_bert_mlseqcls_dataset_cache: + type: dataset + ref: paperspace/dsuxwz4nqbbs07s:jipm3jh + packed_bert_qa_dataset_cache: + type: dataset + ref: paperspace/dssvktzrzcoaumk:5zhp5mf + + diff --git a/packed-bert/packedBERT_multi_label_text_classification.ipynb b/packed-bert/packedBERT_multi_label_text_classification.ipynb index fbc3c5b..70f611b 100644 --- a/packed-bert/packedBERT_multi_label_text_classification.ipynb +++ b/packed-bert/packedBERT_multi_label_text_classification.ipynb @@ -217,7 +217,7 @@ "import os\n", "\n", "pod_type = os.getenv(\"GRAPHCORE_POD_TYPE\", \"pod4\")\n", - "executable_cache_dir = os.getenv(\"POPLAR_EXECUTABLE_CACHE_DIR\", \"./exe_cache/\") + \"/packed_bert_mlseqcls/\"" + "executable_cache_dir = os.getenv(\"POPLAR_EXECUTABLE_CACHE_DIR\", \"/tmp/\") + \"packed_bert_mlseqcls_exe_cache/\"" ] }, { @@ -259,7 +259,7 @@ }, "outputs": [], "source": [ - "dataset = load_dataset(model_task)\n", + "dataset = load_dataset(model_task, cache_dir=os.getenv(\"POPLAR_EXECUTABLE_CACHE_DIR\", \"/tmp/\") + \"packed_bert_mlseqcls_dataset_cache/\")\n", "metric = evaluate.load(\"roc_auc\", \"multilabel\")" ] }, @@ -998,7 +998,7 @@ " lr_scheduler_type = \"cosine\",\n", " metric_for_best_model=metric_name,\n", " dataloader_drop_last=True,\n", - " dataloader_mode=\"async_rebatched\",\n", + " # dataloader_mode=\"async_rebatched\",\n", " logging_steps=1,\n", " pod_type=pod_type,\n", " gradient_accumulation_steps=gradient_accumulation_steps,\n", @@ -1092,9 +1092,9 @@ "source": [ "from pathlib import Path\n", "\n", - "checkpoint_directory = Path(os.getenv('CHECKPOINT_DIR', '/tmp')) / f\"{model_name}-{model_task}\"\n", + "saved_model_checkpoint = Path(os.getenv('CHECKPOINT_DIR', '/tmp/')) + f\"{model_name}-{model_task}\"\n", "\n", - "trainer.save_model(checkpoint_directory)" + "trainer.save_model(saved_model_checkpoint)" ] }, { @@ -1121,7 +1121,7 @@ "\n", "For the pipeline, we need to import it, and initialise a few essential parameters.\n", "\n", - "The `model` is the model checkpoint, we are going to use the locally saved checkpoint generated from training `go_emotions`. The `executable_cache_dir`, `problem_type`, `max_seq_length` must be specified. To return predictions organised by class names, the class names for your output must be passed to `label_categories`. \n", + "The `model` is the model checkpoint, we are going to use the locally saved checkpoint generated from training `go_emotions`. The `executable_cache_dir`, `problem_type`, `max_seq_length` must be specified. To return predictions organised by class names, the class names for your output must be passed to `label_categories`. If you are loading a saved model without a pre-trained tokenizer saved in the checkpoint folder, it will be loaded automatically from `bert-base-uncased`, if you wish to load a different pre-trained tokenizer, you can specify this by passing the `pretrained_tokenizer` argument with the name of your tokenizer to the `PackedBertTextClassificationPipeline`.\n", "\n", "The pipeline will automatically determine your model's IPU config, given that the checkpoint was trained using Optimum Graphcore, which will be the case for the model fine-tuned in this notebook.\n", "\n", @@ -1198,7 +1198,7 @@ "\n", "from optimum.graphcore import IPUConfig\n", "\n", - "model = checkpoint_directory\n", + "model = saved_model_checkpoint\n", "# model = 'your_username/{model_name}-{model_task}' # uncomment this and use your username to load from Hugging Face Hub\n", "\n", "inference_boosted_ipu_config = IPUConfig.from_pretrained(model, \n", diff --git a/packed-bert/packedBERT_question_answering.ipynb b/packed-bert/packedBERT_question_answering.ipynb index e652c74..db5e8b8 100644 --- a/packed-bert/packedBERT_question_answering.ipynb +++ b/packed-bert/packedBERT_question_answering.ipynb @@ -203,7 +203,7 @@ "import os\n", "\n", "pod_type = os.getenv(\"GRAPHCORE_POD_TYPE\", \"pod4\")\n", - "executable_cache_dir = os.getenv(\"POPLAR_EXECUTABLE_CACHE_DIR\", \"./exe_cache/\") + \"/packed_bert_squad/\"" + "executable_cache_dir = os.getenv(\"POPLAR_EXECUTABLE_CACHE_DIR\", \"/tmp/\") + \"packed_bert_qa_exe_cache/\"" ] }, { @@ -229,7 +229,7 @@ "from datasets import load_dataset, load_metric\n", "import evaluate\n", "\n", - "dataset = load_dataset(model_task) # Load dataset\n", + "dataset = load_dataset(model_task, cache_dir=os.getenv(\"POPLAR_EXECUTABLE_CACHE_DIR\", \"/tmp/\") + \"packed_bert_qa_dataset_cache/\" # Load dataset\n", "metric = evaluate.load(model_task) # Load metric for dataset" ] }, @@ -966,9 +966,8 @@ "source": [ "from pathlib import Path\n", "\n", - "checkpoint_directory = Path(os.getenv('CHECKPOINT_DIR', '/tmp')) / f\"{model_checkpoint}-{model_task}\"\n", - "\n", - "trainer.save_model(checkpoint_directory)" + "saved_model_checkpoint = Path(os.getenv('CHECKPOINT_DIR', '/tmp/')) + f\"{model_checkpoint}-{model_task}\"\n", + "trainer.save_model(saved_model_checkpoint)" ] }, { @@ -1033,7 +1032,7 @@ "\n", "For the pipeline, we need to import it, and initialise a few essential parameters.\n", "\n", - "The `model` is the model checkpoint, we are going to use the locally saved checkpoint generated from training SQuAD. The `executable_cache_dir` and `max_seq_length` must also be specified.\n", + "The `model` is the model checkpoint, we are going to use the locally saved checkpoint generated from training SQuAD. The `executable_cache_dir` and `max_seq_length` must also be specified. If you are loading a saved model without a pre-trained tokenizer saved in the checkpoint folder, it will be loaded automatically from `bert-base-uncased`, if you wish to load a different pre-trained tokenizer, you can specify this by passing the `pretrained_tokenizer` argument with the name of your tokenizer to the `PackedBertQuestionAnsweringPipeline`.\n", "\n", "The pipeline will automatically determine your model's IPU config, given that the checkpoint was trained using Optimum Graphcore, which will be the case for the model fine-tuned in this notebook.\n", "\n", @@ -1055,7 +1054,7 @@ "source": [ "from pipeline.packed_bert import PackedBertQuestionAnsweringPipeline\n", "\n", - "model = checkpoint_directory\n", + "model = saved_model_checkpoint\n", "# model = 'your_username/{model_checkpoint}-{model_task}' # uncomment this and use your username to load from Hugging Face Hub\n", "\n", "inference_boosted_ipu_config = IPUConfig.from_pretrained(model, \n", diff --git a/packed-bert/packedBERT_single_label_text_classification.ipynb b/packed-bert/packedBERT_single_label_text_classification.ipynb index 3cda26b..f2adfd3 100644 --- a/packed-bert/packedBERT_single_label_text_classification.ipynb +++ b/packed-bert/packedBERT_single_label_text_classification.ipynb @@ -245,7 +245,7 @@ "import os\n", "\n", "pod_type = os.getenv(\"GRAPHCORE_POD_TYPE\", \"pod4\")\n", - "executable_cache_dir = os.getenv(\"POPLAR_EXECUTABLE_CACHE_DIR\", \"./exe_cache/\") + \"/packed_bert_slseqcls/\"" + "executable_cache_dir = os.getenv(\"POPLAR_EXECUTABLE_CACHE_DIR\", \"/tmp/\") + \"packed_bert_slseqcls_exe_cache/\"" ] }, { @@ -298,7 +298,7 @@ "outputs": [], "source": [ "actual_task = \"mnli\" if task == \"mnli-mm\" else task\n", - "dataset = load_dataset(\"glue\", actual_task)\n", + "dataset = load_dataset(\"glue\", actual_task, cache_dir=os.getenv(\"POPLAR_EXECUTABLE_CACHE_DIR\", \"/tmp/\") + \"packed_bert_slseqcls_dataset_cache/\")\n", "metric = evaluate.load('glue', actual_task)" ] }, @@ -1222,9 +1222,8 @@ "source": [ "from pathlib import Path\n", "\n", - "checkpoint_directory = Path(os.getenv('CHECKPOINT_DIR', '/tmp')) / f\"{model_name}-{task}\"\n", - "\n", - "trainer.save_model(checkpoint_directory)" + "saved_model_checkpoint = Path(os.getenv('CHECKPOINT_DIR', '/tmp/')) + f\"{model_name}-{task}\"\n", + "trainer.save_model(saved_model_checkpoint)" ] }, { @@ -1249,7 +1248,7 @@ "\n", "For the pipeline, we need to import it, and initialise a few essential parameters.\n", "\n", - "The `model` is the model checkpoint, we are going to use the locally saved checkpoint generated from training `sst2`. The `executable_cache_dir`, `problem_type`, `max_seq_length` must be specified. To return predictions organised by class names, the class names for your output must be passed to `label_categories`. \n", + "The `model` is the model checkpoint, we are going to use the locally saved checkpoint generated from training `sst2`. The `executable_cache_dir`, `problem_type`, `max_seq_length` must be specified. To return predictions organised by class names, the class names for your output must be passed to `label_categories`. If you are loading a saved model without a pre-trained tokenizer saved in the checkpoint folder, it will be loaded automatically from `bert-base-uncased`, if you wish to load a different pre-trained tokenizer, you can specify this by passing the `pretrained_tokenizer` argument with the name of your tokenizer to the `PackedBertTextClassificationPipeline`.\n", "\n", "The pipeline will automatically determine your model's IPU config, given that the checkpoint was trained using Optimum Graphcore, which will be the case for the model fine-tuned in this notebook.\n", "\n", @@ -1278,7 +1277,7 @@ "from pipeline.packed_bert import PackedBertTextClassificationPipeline\n", "from optimum.graphcore import IPUConfig\n", "\n", - "model = checkpoint_directory \n", + "model = saved_model_checkpoint \n", "# model = 'your_username/{model_name}-{task}' # uncomment this and use your username to load from Hugging Face Hub\n", "\n", "inference_boosted_ipu_config = IPUConfig.from_pretrained(model, \n", @@ -1340,7 +1339,7 @@ "provenance": [] }, "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, From f96eb96162f2092e6b3a7abb568c0bdc77cb5eec Mon Sep 17 00:00:00 2001 From: Ian Hales Date: Thu, 23 Mar 2023 15:48:18 +0000 Subject: [PATCH 11/11] Update environment vars to match consistency PR. --- .gradient/prepare-datasets.sh | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.gradient/prepare-datasets.sh b/.gradient/prepare-datasets.sh index 4e9ac0b..1d2ed89 100755 --- a/.gradient/prepare-datasets.sh +++ b/.gradient/prepare-datasets.sh @@ -38,29 +38,29 @@ fi echo "Starting preparation of datasets" # symlink exe_cache files -exe_cache_source_dir="${PUBLIC_DATASET_DIR}/poplar-executables-hf-3-1" +exe_cache_source_dir="${PUBLIC_DATASETS_DIR}/poplar-executables-hf-3-1" symlink-public-resources "${exe_cache_source_dir}" $POPLAR_EXECUTABLE_CACHE_DIR # packed bert executables -packed_sl_exe_cache_source_dir="${PUBLIC_DATASET_DIR}/packed_bert_slseqcls_exe_cache/packed_bert_slseqcls" +packed_sl_exe_cache_source_dir="${PUBLIC_DATASETS_DIR}/packed_bert_slseqcls_exe_cache/packed_bert_slseqcls" symlink-public-resources "${packed_exe_cache_source_dir}" "${POPLAR_EXECUTABLE_CACHE_DIR}/packed_bert_slseqcls_exe_cache" -packed_ml_exe_cache_source_dir="${PUBLIC_DATASET_DIR}/packed_bert_mlseqcls_exe_cache/packed_bert_mlseqcls" +packed_ml_exe_cache_source_dir="${PUBLIC_DATASETS_DIR}/packed_bert_mlseqcls_exe_cache/packed_bert_mlseqcls" symlink-public-resources "${packed_exe_cache_source_dir}" "${POPLAR_EXECUTABLE_CACHE_DIR}/packed_bert_mlseqcls_exe_cache" -packed_qa_exe_cache_source_dir="${PUBLIC_DATASET_DIR}/packed_bert_qa_exe_cache/packed_bert_squad" +packed_qa_exe_cache_source_dir="${PUBLIC_DATASETS_DIR}/packed_bert_qa_exe_cache/packed_bert_squad" symlink-public-resources "${packed_exe_cache_source_dir}" "${POPLAR_EXECUTABLE_CACHE_DIR}/packed_bert_qa_exe_cache" # packed bert datasets -packed_sl_dataset_source_dir="${PUBLIC_DATASET_DIR}/packed_bert_slseqcls_dataset_cache" +packed_sl_dataset_source_dir="${PUBLIC_DATASETS_DIR}/packed_bert_slseqcls_dataset_cache" symlink-public-resources "${packed_exe_cache_source_dir}" "${HF_DATASETS}/packed_bert_slseqcls_dataset_cache" -packed_ml_dataset_source_dir="${PUBLIC_DATASET_DIR}/packed_bert_mlseqcls_dataset_cache" +packed_ml_dataset_source_dir="${PUBLIC_DATASETS_DIR}/packed_bert_mlseqcls_dataset_cache" symlink-public-resources "${packed_exe_cache_source_dir}" "${POPLAR_EXECUTABLE_CACHE_DIR}/packed_bert_mlseqcls_dataset_cache" -packed_qa_dataset_source_dir="${PUBLIC_DATASET_DIR}/packed_bert_qa_dataset_cache" +packed_qa_dataset_source_dir="${PUBLIC_DATASETS_DIR}/packed_bert_qa_dataset_cache" symlink-public-resources "${packed_exe_cache_source_dir}" "${POPLAR_EXECUTABLE_CACHE_DIR}/packed_bert_qa_dataset_cache" # packed bert inference checkpoints -symlink-public-resources "${PUBLIC_DATASET_DIR}/bert-base-uncased-sst2" "${CHECKPOINT_DIR}/bert-base-uncased-sst2" -symlink-public-resources "${PUBLIC_DATASET_DIR}/bert-base-uncased-go_emotions" "${CHECKPOINT_DIR}/bert-base-uncased-go_emotions" -symlink-public-resources "${PUBLIC_DATASET_DIR}/bert-base-uncased-squad" "${CHECKPOINT_DIR}/bert-base-uncased-squad" +symlink-public-resources "${PUBLIC_DATASETS_DIR}/bert-base-uncased-sst2" "${CHECKPOINT_DIR}/bert-base-uncased-sst2" +symlink-public-resources "${PUBLIC_DATASETS_DIR}/bert-base-uncased-go_emotions" "${CHECKPOINT_DIR}/bert-base-uncased-go_emotions" +symlink-public-resources "${PUBLIC_DATASETS_DIR}/bert-base-uncased-squad" "${CHECKPOINT_DIR}/bert-base-uncased-squad" @@ -68,10 +68,10 @@ symlink-public-resources "${PUBLIC_DATASET_DIR}/bert-base-uncased-squad" "${CHEC HF_DATASETS="conll2003 glue imagefolder librispeech_asr squad swag wikitext wmt16 xsum" for dataset in ${HF_DATASETS}; do # symlink the actual datasets - symlink-public-resources "${PUBLIC_DATASET_DIR}/${dataset}" "${HF_DATASETS_CACHE}/$(basename ${dataset})" + symlink-public-resources "${PUBLIC_DATASETS_DIR}/${dataset}" "${HF_DATASETS_CACHE}/$(basename ${dataset})" done # Image classification dataset -symlink-public-resources "${PUBLIC_DATASET_DIR}/dfki-sentinel-eurosat" "${DATASET_DIR}/dfki-sentinel-eurosat" +symlink-public-resources "${PUBLIC_DATASETS_DIR}/dfki-sentinel-eurosat" "${DATASETS_DIR}/dfki-sentinel-eurosat" # pre-install the correct version of optimum for this release python -m pip install "optimum-graphcore>=0.5, <0.6"