From cf660ace10f65c25f515ef4af0a7949f0e25be19 Mon Sep 17 00:00:00 2001 From: cuicui Date: Mon, 16 Oct 2023 10:11:56 +0800 Subject: [PATCH 1/4] Add fastvit model --- configs/fastvit/README.md | 91 ++ configs/fastvit/fastvit_t8_ascend.yaml | 60 + mindcv/models/fastvit.py | 1647 ++++++++++++++++++++++++ network_2.ckpt | Bin 0 -> 26380 bytes network_3.ckpt | Bin 0 -> 26380 bytes 5 files changed, 1798 insertions(+) create mode 100644 configs/fastvit/README.md create mode 100644 configs/fastvit/fastvit_t8_ascend.yaml create mode 100644 mindcv/models/fastvit.py create mode 100644 network_2.ckpt create mode 100644 network_3.ckpt diff --git a/configs/fastvit/README.md b/configs/fastvit/README.md new file mode 100644 index 000000000..ac7a4b123 --- /dev/null +++ b/configs/fastvit/README.md @@ -0,0 +1,91 @@ +# FastViT + + +> [A Fast Hybrid Vision Transformer using Structural Reparameterization](https://arxiv.org/abs/2303.14189) + +## Introduction + + +The recent amalgamation of transformer and convolutional designs has led to steady improvements in accuracy and efficiency of the models. In this work, we introduce FastViT, a hybrid vision transformer architecture that obtains the state-of-the-art latency-accuracy trade-off. To this end, we introduce a novel token mixing operator, RepMixer, a building block of FastViT, that uses structural reparameterization to lower the memory access cost by removing skip-connections in the network. We further apply train-time overparametrization and large kernel convolutions to boost accuracy and empirically show that these choices have minimal effect on latency. We show that - our model is 3.5x faster than CMT, a recent state-of-the-art hybrid transformer architecture, 4.9x faster than EfficientNet, and 1.9x faster than ConvNeXt on a mobile device for the same accuracy on the ImageNet dataset. At similar latency, our model obtains 4.2% better Top-1 accuracy on ImageNet than MobileOne. Our model consistently outperforms competing architectures across several tasks -- image classification, detection, segmentation and 3D mesh regression with significant improvement in latency on both a mobile device and a desktop GPU. Furthermore, our model is highly robust to out-of-distribution samples and corruptions, improving over competing robust models. + + + +## Results + + +Our reproduced model performance on ImageNet-1K is reported as follows. + +
+ +| Model | Context | Top-1 (%) | Top-5 (%) | Params (M) | Recipe | Download | +|-----------|----------|-----------|-----------|------------|-----------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------| +| FastViT-T8 | D910x8-G | 74.25 | 91.97 | 48 | [yaml](https://github.com/mindspore-lab/mindcv/blob/main/configs/fastvit/fastvit_t8_ascend.yaml) | + +
+ +#### Notes +- Context: Training context denoted as {device}x{pieces}-{MS mode}, where mindspore mode can be G - graph mode or F - pynative mode with ms function. For example, D910x8-G is for training on 8 pieces of Ascend 910 NPU using graph mode. +- Top-1 and Top-5: Accuracy reported on the validation set of ImageNet-1K. + + +## Quick Start +### Preparation + +#### Installation +Please refer to the [installation instruction](https://github.com/mindspore-lab/mindcv#installation) in MindCV. + +#### Dataset Preparation +Please download the [ImageNet-1K](https://www.image-net.org/challenges/LSVRC/2012/index.php) dataset for model training and validation. + +### Training + + +* Distributed Training + +It is easy to reproduce the reported results with the pre-defined training recipe. For distributed training on multiple Ascend 910 devices, please run + +```shell +# distributed training on multiple GPU/Ascend devices +mpirun -n 8 python train.py --config configs/fastvit/fastvit_t8_ascend.yaml --data_dir /path/to/imagenet +``` +> If the script is executed by the root user, the `--allow-run-as-root` parameter must be added to `mpirun`. + +Similarly, you can train the model on multiple GPU devices with the above `mpirun` command. + +For detailed illustration of all hyper-parameters, please refer to [config.py](https://github.com/mindspore-lab/mindcv/blob/main/config.py). + +**Note:** As the global batch size (batch_size x num_devices) is an important hyper-parameter, it is recommended to keep the global batch size unchanged for reproduction or adjust the learning rate linearly to a new global batch size. + +* Standalone Training + +If you want to train or finetune the model on a smaller dataset without distributed training, please run: + +```shell +# standalone training on a CPU/GPU/Ascend device +python train.py --config configs/fastvit/fastvit_t8_ascend.yaml --data_dir /path/to/dataset --distribute False +``` + +### Validation + +To validate the accuracy of the trained model, you can use `validate.py` and parse the checkpoint path with `--ckpt_path`. + +``` +python validate.py -c configs/fastvit/fastvit_t8_ascend.yaml --data_dir /path/to/imagenet --ckpt_path /path/to/ckpt +``` + +### Deployment + +To deploy online inference services with the trained model efficiently, please refer to the [deployment tutorial](https://mindspore-lab.github.io/mindcv/tutorials/deployment/). + +## References + + +[1] Vasu P K A, Gabriel J, Zhu J, et al. FastViT: A Fast Hybrid Vision Transformer using Structural Reparameterization[J]. arXiv preprint arXiv:2303.14189, 2023. diff --git a/configs/fastvit/fastvit_t8_ascend.yaml b/configs/fastvit/fastvit_t8_ascend.yaml new file mode 100644 index 000000000..ad0abe128 --- /dev/null +++ b/configs/fastvit/fastvit_t8_ascend.yaml @@ -0,0 +1,60 @@ +# system +mode: 0 +distribute: False +num_parallel_workers: 8 +val_while_train: True +val_interval: 1 +log_interval: 100 + +# dataset +dataset: "imagenet" +data_dir: "/path/to/imagenet" +shuffle: True +dataset_download: False +batch_size: 128 + +# augmentation +image_resize: 224 +scale: [0.08, 1.0] +ratio: [0.75, 1.333] +hflip: 0.5 +vflip: 0.0 +interpolation: "bicubic" +re_prob: 0.1 +mixup: 0.8 +cutmix: 1.0 +color_jitter: 0.4 +auto_augment: "randaug-m7-mstd0.5" + +# model +model: "fastvit_t8" +num_classes: 1000 +pretrained: False +keep_checkpoint_max: 10 +ckpt_save_policy: "latest_k" +ckpt_save_interval: 1 +ckpt_save_dir: "./ckpt" +epoch_size: 300 +dataset_sink_mode: True +ema_decay: 0.9995 +amp_level: "O2" +loss_scale_type: 'auto' + +# loss +loss: "CE" +label_smoothing: 0.1 + +# lr scheduler +scheduler: "cosine_decay" +lr: 0.001 +min_lr: 0.0 +warmup_epochs: 5 +warmup_factor: 0.01 +decay_epochs: 295 + +# optimizer +opt: "adamw" +momentum: 0.9 +weight_decay: 0.05 +filter_bias_and_bn: True +use_nesterov: False diff --git a/mindcv/models/fastvit.py b/mindcv/models/fastvit.py new file mode 100644 index 000000000..43fd1f9d4 --- /dev/null +++ b/mindcv/models/fastvit.py @@ -0,0 +1,1647 @@ +"""Reference:https://github.com/apple/ml-fastvit""" +import math +import mindspore as ms +from mindspore import ops,nn +from mindspore.numpy import ones +import mindspore.common.initializer as init +import numpy as np +from mindcv.models.registry import register_model +from mindcv.models.layers.pooling import GlobalAvgPooling + +from typing import Union,Tuple,Optional,List +import copy +from collections import OrderedDict +import os +from functools import partial + +IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD = (0.485,0.456,0.406),(0.229,0.224,0.225) + +def _cfg(url="", **kwargs): + return { + "url": url, + "num_classes": 1000, + "input_size": (3, 256, 256), + "pool_size": None, + "crop_pct": 0.95, + "interpolation": "bicubic", + "mean": IMAGENET_DEFAULT_MEAN, + "std": IMAGENET_DEFAULT_STD, + "classifier": "head", + **kwargs, + } + + +default_cfgs = { + "fastvit_t": _cfg(crop_pct=0.9), + "fastvit_s": _cfg(crop_pct=0.9), + "fastvit_m": _cfg(crop_pct=0.95), +} + + +def convolutional_stem( + in_channels: int, out_channels: int, inference_mode: bool = False +) -> nn.SequentialCell: + """Build convolutional stem with MobileOne blocks. + + Args: + in_channels: Number of input channels. + out_channels: Number of output channels. + inference_mode: Flag to instantiate model in inference mode. Default: ``False`` + + Returns: + nn.Sequential object with stem elements. + """ + return nn.SequentialCell( + MobileOneBlock( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=3, + stride=2, + padding=1, + group=1, + inference_mode=inference_mode, + use_se=False, + num_conv_branches=1, + ), + MobileOneBlock( + in_channels=out_channels, + out_channels=out_channels, + kernel_size=3, + stride=2, + padding=1, + group=out_channels, + inference_mode=inference_mode, + use_se=False, + num_conv_branches=1, + ), + MobileOneBlock( + in_channels=out_channels, + out_channels=out_channels, + kernel_size=1, + stride=1, + padding=0, + group=1, + inference_mode=inference_mode, + use_se=False, + num_conv_branches=1, + ), + ) + + +class MHSA(nn.Cell): + """Multi-headed Self Attention module. + + Source modified from: + https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py + """ + + def __init__( + self, + dim: int, + head_dim: int = 32, + qkv_bias: bool = False, + attn_drop: float = 0.0, + proj_drop: float = 0.0, + ) -> None: + """Build MHSA module that can handle 3D or 4D input tensors. + + Args: + dim: Number of embedding dimensions. + head_dim: Number of hidden dimensions per head. Default: ``32`` + qkv_bias: Use bias or not. Default: ``False`` + attn_drop: Dropout rate for attention tensor. + proj_drop: Dropout rate for projection tensor. + """ + super(MHSA,self).__init__() + assert dim % head_dim == 0, "dim should be divisible by head_dim" + self.head_dim = head_dim + self.num_heads = dim // head_dim + self.scale = head_dim**-0.5 + + self.qkv = nn.Dense(dim, dim * 3, has_bias=qkv_bias) + self.attn_drop = nn.Dropout(p=attn_drop) + self.proj = nn.Dense(dim, dim) + self.proj_drop = nn.Dropout(p=proj_drop) + self.batch_matmul = ops.BatchMatMul() + + def construct(self, x: ms.Tensor) -> ms.Tensor: + shape = x.shape + B, C, H, W = shape + N = H * W + if len(shape) == 4: + x = nn.flatten(x, start_dim=2).transpose((0,-1,-2)) # (B, N, C) + qkv = ( + self.qkv(x) + .reshape((B, N, 3, self.num_heads, self.head_dim)) + .permute(2, 0, 3, 1, 4) + ) + q, k, v = ops.Unstack(axis=0)(qkv) + + # trick here to make q@k.t more stable + attn = self.batch_matmul(q*self.scale,k.transpose(0,1,-1,-2)) + attn = nn.Softmax(axis=-1)(attn) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose((0,2,1,-1)).reshape((B, N, C)) + x = self.proj(x) + x = self.proj_drop(x) + if len(shape) == 4: + x = x.transpose((0,-1,-2)).reshape(B, C, H, W) + + return x + + +class PatchEmbed(nn.Cell): + """Convolutional patch embedding layer.""" + + def __init__( + self, + patch_size: int, + stride: int, + in_channels: int, + embed_dim: int, + inference_mode: bool = False, + ) -> None: + """Build patch embedding layer. + + Args: + patch_size: Patch size for embedding computation. + stride: Stride for convolutional embedding layer. + in_channels: Number of channels of input tensor. + embed_dim: Number of embedding dimensions. + inference_mode: Flag to instantiate model in inference mode. Default: ``False`` + """ + super().__init__() + self.layers = nn.CellList() + self.layers.append( + ReparamLargeKernelConv( + in_channels=in_channels, + out_channels=embed_dim, + kernel_size=patch_size, + stride=stride, + group=in_channels, + small_kernel=3, + inference_mode=inference_mode, + ) + ) + self.layers.append( + MobileOneBlock( + in_channels=embed_dim, + out_channels=embed_dim, + kernel_size=1, + stride=1, + padding=0, + group=1, + inference_mode=inference_mode, + use_se=False, + num_conv_branches=1, + ) + ) + + def construct(self, x: ms.Tensor) -> ms.Tensor: + for layer in self.layers: + x = layer(x) + return x + + +class RepMixer(nn.Cell): + """Reparameterizable token mixer. + + For more details, please refer to our paper: + `FastViT: A Fast Hybrid Vision Transformer using Structural Reparameterization `_ + """ + + def __init__( + self, + dim, + kernel_size=3, + use_layer_scale=True, + layer_scale_init_value=1e-5, + inference_mode: bool = False, + ): + """Build RepMixer Module. + + Args: + dim: Input feature map dimension. :math:`C_{in}` from an expected input of size :math:`(B, C_{in}, H, W)`. + kernel_size: Kernel size for spatial mixing. Default: 3 + use_layer_scale: If True, learnable layer scale is used. Default: ``True`` + layer_scale_init_value: Initial value for layer scale. Default: 1e-5 + inference_mode: If True, instantiates model in inference mode. Default: ``False`` + """ + super().__init__() + self.dim = dim + self.kernel_size = kernel_size + self.inference_mode = inference_mode + self.reparam_conv = None + + if inference_mode: + self.reparam_conv = nn.Conv2d( + in_channels=self.dim, + out_channels=self.dim, + kernel_size=self.kernel_size, + stride=1, + pad_mode='pad', + padding=self.kernel_size // 2, + group=self.dim, + has_bias=True, + ) + else: + self.norm = MobileOneBlock( + dim, + dim, + kernel_size, + padding=kernel_size // 2, + group=dim, + use_act=False, + use_scale_branch=False, + num_conv_branches=0, + ) + self.mixer = MobileOneBlock( + dim, + dim, + kernel_size, + padding=kernel_size // 2, + group=dim, + use_act=False, + ) + self.use_layer_scale = use_layer_scale + if use_layer_scale: + self.layer_scale = ms.Parameter( + layer_scale_init_value * ops.ones((dim, 1, 1),ms.float32), name = 'w',requires_grad=True + ) + + def construct(self, x: ms.Tensor) -> ms.Tensor: + if self.reparam_conv is not None: + x = self.reparam_conv(x) + return x + else: + if self.use_layer_scale: + x = x + self.layer_scale * (self.mixer(x) - self.norm(x)) + else: + x = x + self.mixer(x) - self.norm(x) + return x + + def reparameterize(self) -> None: + """Reparameterize mixer and norm into a single + convolutional layer for efficient inference. + """ + if self.inference_mode: + return + + self.mixer.reparameterize() + self.norm.reparameterize() + + if self.use_layer_scale: + w = self.mixer.id_tensor + ops.ExpandDims()(self.layer_scale,-1) * ( + self.mixer.reparam_conv.weight - self.norm.reparam_conv.weight + ) + b = ops.Squeeze()(self.layer_scale) * ( + self.mixer.reparam_conv.bias - self.norm.reparam_conv.bias + ) + else: + w = ( + self.mixer.id_tensor + + self.mixer.reparam_conv.weight + - self.norm.reparam_conv.weight + ) + b = self.mixer.reparam_conv.bias - self.norm.reparam_conv.bias + + self.reparam_conv = nn.Conv2d( + in_channels=self.dim, + out_channels=self.dim, + kernel_size=self.kernel_size, + stride=1, + pad_mode='pad', + padding=self.kernel_size // 2, + group=self.dim, + has_bias=True, + ) + self.reparam_conv.weight = w + self.reparam_conv.bias = b + + for para in self.get_parameters(): + para = ops.stop_gradient(para) + self.__delattr__("mixer") + self.__delattr__("norm") + if self.use_layer_scale: + self.__delattr__("layer_scale") + + +class ConvFFN(nn.Cell): + """Convolutional FFN Module.""" + + def __init__( + self, + in_channels: int, + hidden_channels: Optional[int] = None, + out_channels: Optional[int] = None, + act_layer: nn.Cell = nn.GELU, + drop: float = 0.0, + ) -> None: + """Build convolutional FFN module. + + Args: + in_channels: Number of input channels. + hidden_channels: Number of channels after expansion. Default: None + out_channels: Number of output channels. Default: None + act_layer: Activation layer. Default: ``GELU`` + drop: Dropout rate. Default: ``0.0``. + """ + super().__init__() + out_channels = out_channels or in_channels + hidden_channels = hidden_channels or in_channels + self.conv = nn.SequentialCell( + OrderedDict( + [("conv", + nn.Conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=7, + pad_mode = 'pad', + padding=3, + group=in_channels, + has_bias=False, + )),("bn", nn.BatchNorm2d(num_features=out_channels))] + ) + ) + self.fc1 = nn.Conv2d(in_channels, hidden_channels, kernel_size=1) + self.act = act_layer() + self.fc2 = nn.Conv2d(hidden_channels, out_channels, kernel_size=1) + self.drop = nn.Dropout(p=drop) + self._init_weights() + + def _init_weights(self) -> None: + for _,cell in self.cells_and_names(): + if isinstance(cell,nn.Conv2d): + cell.weight.set_data(init.initializer(init.TruncatedNormal(sigma=0.02),cell.weight.shape,cell.weight.dtype)) + if cell.bias is not None: + cell.bias.set_data(init.initializer(init.Zero(),cell.bias.shape,cell.bias.dtype)) + def construct(self, x: ms.Tensor) -> ms.Tensor: + x = self.conv(x) + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class RepCPE(nn.Cell): + """Implementation of conditional positional encoding. + + For more details refer to paper: + `Conditional Positional Encodings for Vision Transformers `_ + + In our implementation, we can reparameterize this module to eliminate a skip connection. + """ + + def __init__( + self, + in_channels: int, + embed_dim: int = 768, + spatial_shape: Union[int, Tuple[int, int]] = (7, 7), + inference_mode=False, + ) -> None: + """Build reparameterizable conditional positional encoding + + Args: + in_channels: Number of input channels. + embed_dim: Number of embedding dimensions. Default: 768 + spatial_shape: Spatial shape of kernel for positional encoding. Default: (7, 7) + inference_mode: Flag to instantiate block in inference mode. Default: ``False`` + """ + super(RepCPE, self).__init__() + if isinstance(spatial_shape, int): + spatial_shape = tuple([spatial_shape] * 2) + assert isinstance(spatial_shape, Tuple), ( + f'"spatial_shape" must by a sequence or int, ' + f"get {type(spatial_shape)} instead." + ) + assert len(spatial_shape) == 2, ( + f'Length of "spatial_shape" should be 2, ' + f"got {len(spatial_shape)} instead." + ) + + self.spatial_shape = spatial_shape + self.embed_dim = embed_dim + self.in_channels = in_channels + self.group = embed_dim + self.reparam_conv = None + if inference_mode: + self.reparam_conv = nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.embed_dim, + kernel_size=self.spatial_shape, + stride=1, + pad_mode='pad', + padding=int(self.spatial_shape[0] // 2), + group=self.embed_dim, + has_bias=True, + ) + else: + self.pe = nn.Conv2d( + in_channels, + embed_dim, + spatial_shape, + 1, + 'pad', + int(spatial_shape[0] // 2), + has_bias=True, + group=embed_dim, + ) + + def construct(self, x: ms.Tensor) -> ms.Tensor: + if self.reparam_conv is not None: + x = self.reparam_conv(x) + return x + else: + x = self.pe(x) + x + return x + + def reparameterize(self) -> None: + # Build equivalent Id tensor + input_dim = self.in_channels // self.group + kernel_value = ops.Zeros()( + ( + self.in_channels, + input_dim, + self.spatial_shape[0], + self.spatial_shape[1], + ),ms.float32 + ) + for i in range(self.in_channels): + kernel_value[ + i, + i % input_dim, + self.spatial_shape[0] // 2, + self.spatial_shape[1] // 2, + ] = 1 + id_tensor = kernel_value + + # Reparameterize Id tensor and conv + w_final = id_tensor + self.pe.weight + b_final = self.pe.bias + + # Introduce reparam conv + self.reparam_conv = nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.embed_dim, + kernel_size=self.spatial_shape, + stride=1, + pad_mode='pad', + padding=int(self.spatial_shape[0] // 2), + group=self.embed_dim, + has_bias=True, + ) + self.reparam_conv.weight = w_final + self.reparam_conv.bias = b_final + + for para in self.get_parameters(): + para = ops.stop_gradient(para) + self.__delattr__("pe") + + +class RepMixerBlock(nn.Cell): + """Implementation of Metaformer block with RepMixer as token mixer. + + For more details on Metaformer structure, please refer to: + `MetaFormer Is Actually What You Need for Vision `_ + """ + + def __init__( + self, + dim: int, + kernel_size: int = 3, + mlp_ratio: float = 4.0, + act_layer: nn.Cell = nn.GELU, + drop: float = 0.0, + drop_path: float = 0.0, + use_layer_scale: bool = True, + layer_scale_init_value: float = 1e-5, + inference_mode: bool = False, + ): + """Build RepMixer Block. + + Args: + dim: Number of embedding dimensions. + kernel_size: Kernel size for repmixer. Default: 3 + mlp_ratio: MLP expansion ratio. Default: 4.0 + act_layer: Activation layer. Default: ``nn.GELU`` + drop: Dropout rate. Default: 0.0 + drop_path: Drop path rate. Default: 0.0 + use_layer_scale: Flag to turn on layer scale. Default: ``True`` + layer_scale_init_value: Layer scale value at initialization. Default: 1e-5 + inference_mode: Flag to instantiate block in inference mode. Default: ``False`` + """ + + super().__init__() + + self.token_mixer = RepMixer( + dim, + kernel_size=kernel_size, + use_layer_scale=use_layer_scale, + layer_scale_init_value=layer_scale_init_value, + inference_mode=inference_mode, + ) + + assert mlp_ratio > 0, "MLP ratio should be greater than 0, found: {}".format( + mlp_ratio + ) + mlp_hidden_dim = int(dim * mlp_ratio) + self.convffn = ConvFFN( + in_channels=dim, + hidden_channels=mlp_hidden_dim, + act_layer=act_layer, + drop=drop, + ) + + # Drop Path + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + # Layer Scale + self.use_layer_scale = use_layer_scale + if use_layer_scale: + self.layer_scale = ms.Parameter( + layer_scale_init_value * ops.ones((dim, 1, 1),ms.float32), requires_grad=True + ) + + def construct(self, x): + if self.use_layer_scale: + x = self.token_mixer(x) + x = x + self.drop_path(self.layer_scale * self.convffn(x)) + else: + x = self.token_mixer(x) + x = x + self.drop_path(self.convffn(x)) + return x + + +class AttentionBlock(nn.Cell): + """Implementation of metaformer block with MHSA as token mixer. + + For more details on Metaformer structure, please refer to: + `MetaFormer Is Actually What You Need for Vision `_ + """ + + def __init__( + self, + dim: int, + mlp_ratio: float = 4.0, + act_layer: nn.Cell = nn.GELU, + norm_layer: nn.Cell = nn.BatchNorm2d, + drop: float = 0.0, + drop_path: float = 0.0, + use_layer_scale: bool = True, + layer_scale_init_value: float = 1e-5, + ): + """Build Attention Block. + + Args: + dim: Number of embedding dimensions. + mlp_ratio: MLP expansion ratio. Default: 4.0 + act_layer: Activation layer. Default: ``nn.GELU`` + norm_layer: Normalization layer. Default: ``nn.BatchNorm2d`` + drop: Dropout rate. Default: 0.0 + drop_path: Drop path rate. Default: 0.0 + use_layer_scale: Flag to turn on layer scale. Default: ``True`` + layer_scale_init_value: Layer scale value at initialization. Default: 1e-5 + """ + + super().__init__() + + self.norm = norm_layer(dim) + self.token_mixer = MHSA(dim=dim) + + assert mlp_ratio > 0, "MLP ratio should be greater than 0, found: {}".format( + mlp_ratio + ) + mlp_hidden_dim = int(dim * mlp_ratio) + self.convffn = ConvFFN( + in_channels=dim, + hidden_channels=mlp_hidden_dim, + act_layer=act_layer, + drop=drop, + ) + + # Drop path + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + # Layer Scale + self.use_layer_scale = use_layer_scale + if use_layer_scale: + self.layer_scale_1 = ms.Parameter( + layer_scale_init_value * ops.ones((dim, 1, 1),ms.float32), requires_grad=True + ) + self.layer_scale_2 = ms.Parameter( + layer_scale_init_value * ops.ones((dim, 1, 1),ms.float32), requires_grad=True + ) + + def construct(self, x): + if self.use_layer_scale: + x = x + self.drop_path(self.layer_scale_1 * self.token_mixer(self.norm(x))) + x = x + self.drop_path(self.layer_scale_2 * self.convffn(x)) + else: + x = x + self.drop_path(self.token_mixer(self.norm(x))) + x = x + self.drop_path(self.convffn(x)) + return x + + +def basic_blocks( + dim: int, + block_index: int, + num_blocks: List[int], + token_mixer_type: str, + kernel_size: int = 3, + mlp_ratio: float = 4.0, + act_layer: nn.Cell = nn.GELU, + norm_layer: nn.Cell = nn.BatchNorm2d, + drop_rate: float = 0.0, + drop_path_rate: float = 0.0, + use_layer_scale: bool = True, + layer_scale_init_value: float = 1e-5, + inference_mode=False, +) -> nn.SequentialCell: + """Build FastViT blocks within a stage. + + Args: + dim: Number of embedding dimensions. + block_index: block index. + num_blocks: List containing number of blocks per stage. + token_mixer_type: Token mixer type. + kernel_size: Kernel size for repmixer. + mlp_ratio: MLP expansion ratio. + act_layer: Activation layer. + norm_layer: Normalization layer. + drop_rate: Dropout rate. + drop_path_rate: Drop path rate. + use_layer_scale: Flag to turn on layer scale regularization. + layer_scale_init_value: Layer scale value at initialization. + inference_mode: Flag to instantiate block in inference mode. + + Returns: + nn.Sequential object of all the blocks within the stage. + """ + blocks = [] + for block_idx in range(num_blocks[block_index]): + block_dpr = ( + drop_path_rate + * (block_idx + sum(num_blocks[:block_index])) + / (sum(num_blocks) - 1) + ) + if token_mixer_type == "repmixer": + blocks.append( + RepMixerBlock( + dim, + kernel_size=kernel_size, + mlp_ratio=mlp_ratio, + act_layer=act_layer, + drop=drop_rate, + drop_path=block_dpr, + use_layer_scale=use_layer_scale, + layer_scale_init_value=layer_scale_init_value, + inference_mode=inference_mode, + ) + ) + elif token_mixer_type == "attention": + blocks.append( + AttentionBlock( + dim, + mlp_ratio=mlp_ratio, + act_layer=act_layer, + norm_layer=norm_layer, + drop=drop_rate, + drop_path=block_dpr, + use_layer_scale=use_layer_scale, + layer_scale_init_value=layer_scale_init_value, + ) + ) + else: + raise ValueError( + "Token mixer type: {} not supported".format(token_mixer_type) + ) + blocks = nn.SequentialCell(*blocks) + + return blocks + + +class FastViT(nn.Cell): + """ + This class implements `FastViT architecture `_ + """ + + def __init__( + self, + layers, + token_mixers: Tuple[str, ...], + embed_dims=None, + mlp_ratios=None, + downsamples=None, + repmixer_kernel_size=3, + norm_layer: nn.Cell = nn.BatchNorm2d, + act_layer: nn.Cell = nn.GELU, + num_classes=1000, + pos_embs=None, + down_patch_size=7, + down_stride=2, + drop_rate=0.0, + drop_path_rate=0.0, + use_layer_scale=True, + layer_scale_init_value=1e-5, + fork_feat=False, + init_cfg=None, + pretrained=None, + cls_ratio=2.0, + inference_mode=False, + **kwargs, + ) -> None: + + super().__init__() + + if not fork_feat: + self.num_classes = num_classes + self.fork_feat = fork_feat + + if pos_embs is None: + pos_embs = [None] * len(layers) + + # Convolutional stem + self.patch_embed = convolutional_stem(3, embed_dims[0], inference_mode) + + # Build the main stages of the network architecture + self.network = nn.CellList() + for i in range(len(layers)): + # Add position embeddings if requested + if pos_embs[i] is not None: + self.network.append( + pos_embs[i]( + embed_dims[i], embed_dims[i], inference_mode=inference_mode + ) + ) + stage = basic_blocks( + embed_dims[i], + i, + layers, + token_mixer_type=token_mixers[i], + kernel_size=repmixer_kernel_size, + mlp_ratio=mlp_ratios[i], + act_layer=act_layer, + norm_layer=norm_layer, + drop_rate=drop_rate, + drop_path_rate=drop_path_rate, + use_layer_scale=use_layer_scale, + layer_scale_init_value=layer_scale_init_value, + inference_mode=inference_mode, + ) + self.network.append(stage) + if i >= len(layers) - 1: + break + + # Patch merging/downsampling between stages. + if downsamples[i] or embed_dims[i] != embed_dims[i + 1]: + self.network.append( + PatchEmbed( + patch_size=down_patch_size, + stride=down_stride, + in_channels=embed_dims[i], + embed_dim=embed_dims[i + 1], + inference_mode=inference_mode, + ) + ) + + + # For segmentation and detection, extract intermediate output + if self.fork_feat: + # add a norm layer for each output + self.out_indices = [0, 2, 4, 6] + for i_emb, i_layer in enumerate(self.out_indices): + if i_emb == 0 and os.environ.get("FORK_LAST3", None): + """For RetinaNet, `start_level=1`. The first norm layer will not used. + cmd: `FORK_LAST3=1 python -m torch.distributed.launch ...` + """ + layer = nn.Identity() + else: + layer = norm_layer(embed_dims[i_emb]) + layer_name = f"norm{i_layer}" + self.insert_child_to_cell(layer_name, layer) + else: + # Classifier head + self.gap = GlobalAvgPooling() + self.conv_exp = MobileOneBlock( + in_channels=embed_dims[-1], + out_channels=int(embed_dims[-1] * cls_ratio), + kernel_size=3, + stride=1, + padding=1, + group=embed_dims[-1], + inference_mode=inference_mode, + use_se=True, + num_conv_branches=1, + ) + self.head = ( + nn.Dense(int(embed_dims[-1] * cls_ratio), num_classes) + if num_classes > 0 + else nn.Identity() + ) + + self.cls_init_weights() + self.init_cfg = copy.deepcopy(init_cfg) + + def cls_init_weights(self) -> None: + """Init. for classification""" + for _,cell in self.cells_and_names(): + if isinstance(cell,nn.Dense): + cell.weight.set_data(init.initializer(init.TruncatedNormal(sigma=0.02),cell.weight.shape,cell.weight.dtype)) + if isinstance(cell,nn.Dense) and cell.bias is not None: + cell.bias.set_data(init.initializer(init.Zero(),cell.bias.shape,cell.bias.dtype)) + + def forward_embeddings(self, x: ms.Tensor) -> ms.Tensor: + x = self.patch_embed(x) + return x + + def forward_tokens(self, x: ms.Tensor) -> ms.Tensor: + outs = [] + for idx, block in enumerate(self.network): + x = block(x) + if self.fork_feat and idx in self.out_indices: + norm_layer = getattr(self, f"norm{idx}") + x_out = norm_layer(x) + outs.append(x_out) + if self.fork_feat: + # output the features of four stages for dense prediction + return outs + # output only the features of last layer for image classification + return x + + def construct(self, x: ms.Tensor) -> ms.Tensor: + # input embedding + x = self.forward_embeddings(x) + # through backbone + x = self.forward_tokens(x) + if self.fork_feat: + # output features of four stages for dense prediction + return x + # for image classification + x = self.conv_exp(x) + x = self.gap(x) + x = x.view((x.shape[0], -1)) + cls_out = self.head(x) + return cls_out + + +@register_model +def fastvit_t8(pretrained=False, **kwargs): + """Instantiate FastViT-T8 model variant.""" + layers = [2, 2, 4, 2] + embed_dims = [48, 96, 192, 384] + mlp_ratios = [3, 3, 3, 3] + downsamples = [True, True, True, True] + token_mixers = ("repmixer", "repmixer", "repmixer", "repmixer") + model = FastViT( + layers, + token_mixers=token_mixers, + embed_dims=embed_dims, + mlp_ratios=mlp_ratios, + downsamples=downsamples, + **kwargs, + ) + model.default_cfg = default_cfgs["fastvit_t"] + if pretrained: + raise ValueError("Functionality not implemented.") + return model + + +@register_model +def fastvit_t12(pretrained=False, **kwargs): + """Instantiate FastViT-T12 model variant.""" + layers = [2, 2, 6, 2] + embed_dims = [64, 128, 256, 512] + mlp_ratios = [3, 3, 3, 3] + downsamples = [True, True, True, True] + token_mixers = ("repmixer", "repmixer", "repmixer", "repmixer") + model = FastViT( + layers, + token_mixers=token_mixers, + embed_dims=embed_dims, + mlp_ratios=mlp_ratios, + downsamples=downsamples, + **kwargs, + ) + model.default_cfg = default_cfgs["fastvit_t"] + if pretrained: + raise ValueError("Functionality not implemented.") + return model + + +@register_model +def fastvit_s12(pretrained=False, **kwargs): + """Instantiate FastViT-S12 model variant.""" + layers = [2, 2, 6, 2] + embed_dims = [64, 128, 256, 512] + mlp_ratios = [4, 4, 4, 4] + downsamples = [True, True, True, True] + token_mixers = ("repmixer", "repmixer", "repmixer", "repmixer") + model = FastViT( + layers, + token_mixers=token_mixers, + embed_dims=embed_dims, + mlp_ratios=mlp_ratios, + downsamples=downsamples, + **kwargs, + ) + model.default_cfg = default_cfgs["fastvit_s"] + if pretrained: + raise ValueError("Functionality not implemented.") + return model + + +@register_model +def fastvit_sa12(pretrained=False, **kwargs): + """Instantiate FastViT-SA12 model variant.""" + layers = [2, 2, 6, 2] + embed_dims = [64, 128, 256, 512] + mlp_ratios = [4, 4, 4, 4] + downsamples = [True, True, True, True] + pos_embs = [None, None, None, partial(RepCPE, spatial_shape=(7, 7))] + token_mixers = ("repmixer", "repmixer", "repmixer", "attention") + model = FastViT( + layers, + token_mixers=token_mixers, + embed_dims=embed_dims, + pos_embs=pos_embs, + mlp_ratios=mlp_ratios, + downsamples=downsamples, + **kwargs, + ) + model.default_cfg = default_cfgs["fastvit_s"] + if pretrained: + raise ValueError("Functionality not implemented.") + return model + + +@register_model +def fastvit_sa24(pretrained=False, **kwargs): + """Instantiate FastViT-SA24 model variant.""" + layers = [4, 4, 12, 4] + embed_dims = [64, 128, 256, 512] + mlp_ratios = [4, 4, 4, 4] + downsamples = [True, True, True, True] + pos_embs = [None, None, None, partial(RepCPE, spatial_shape=(7, 7))] + token_mixers = ("repmixer", "repmixer", "repmixer", "attention") + model = FastViT( + layers, + token_mixers=token_mixers, + embed_dims=embed_dims, + pos_embs=pos_embs, + mlp_ratios=mlp_ratios, + downsamples=downsamples, + **kwargs, + ) + model.default_cfg = default_cfgs["fastvit_s"] + if pretrained: + raise ValueError("Functionality not implemented.") + return model + + +@register_model +def fastvit_sa36(pretrained=False, **kwargs): + """Instantiate FastViT-SA36 model variant.""" + layers = [6, 6, 18, 6] + embed_dims = [64, 128, 256, 512] + mlp_ratios = [4, 4, 4, 4] + downsamples = [True, True, True, True] + pos_embs = [None, None, None, partial(RepCPE, spatial_shape=(7, 7))] + token_mixers = ("repmixer", "repmixer", "repmixer", "attention") + model = FastViT( + layers, + embed_dims=embed_dims, + token_mixers=token_mixers, + pos_embs=pos_embs, + mlp_ratios=mlp_ratios, + downsamples=downsamples, + layer_scale_init_value=1e-6, + **kwargs, + ) + model.default_cfg = default_cfgs["fastvit_m"] + if pretrained: + raise ValueError("Functionality not implemented.") + return model + + +@register_model +def fastvit_ma36(pretrained=False, **kwargs): + """Instantiate FastViT-MA36 model variant.""" + layers = [6, 6, 18, 6] + embed_dims = [76, 152, 304, 608] + mlp_ratios = [4, 4, 4, 4] + downsamples = [True, True, True, True] + pos_embs = [None, None, None, partial(RepCPE, spatial_shape=(7, 7))] + token_mixers = ("repmixer", "repmixer", "repmixer", "attention") + model = FastViT( + layers, + embed_dims=embed_dims, + token_mixers=token_mixers, + pos_embs=pos_embs, + mlp_ratios=mlp_ratios, + downsamples=downsamples, + layer_scale_init_value=1e-6, + **kwargs, + ) + model.default_cfg = default_cfgs["fastvit_m"] + if pretrained: + raise ValueError("Functionality not implemented.") + return model + +class DropPath(nn.Cell): + """DropPath (Stochastic Depth) regularization layers""" + + def __init__( + self, + drop_prob: float = 0.0, + scale_by_keep: bool = True, + ) -> None: + super().__init__() + self.keep_prob = 1.0 - drop_prob + self.scale_by_keep = scale_by_keep + self.dropout = nn.Dropout(p=drop_prob) + + def construct(self, x: ms.Tensor) -> ms.Tensor: + if self.keep_prob == 1.0 or not self.training: + return x + shape = (x.shape[0],) + (1,) * (x.ndim - 1) + random_tensor = self.dropout(ones(shape)) + if not self.scale_by_keep: + random_tensor = ops.mul(random_tensor, self.keep_prob) + return x * random_tensor + +class SEBlock(nn.Cell): + + def __init__(self, in_channels: int, rd_ratio: float = 0.0625) -> None: + """Construct a Squeeze and Excite Module. + + Args: + in_channels: Number of input channels. + rd_ratio: Input channel reduction ratio. + """ + super(SEBlock, self).__init__() + self.reduce = nn.Conv2d( + in_channels=in_channels, + out_channels=int(in_channels * rd_ratio), + kernel_size=1, + stride=1, + pad_mode='valid', + has_bias=True, + ) + self.expand = nn.Conv2d( + in_channels=int(in_channels * rd_ratio), + out_channels=in_channels, + kernel_size=1, + pad_mode='valid', + stride=1, + has_bias=True, + ) + + def construct(self, inputs: ms.Tensor) -> ms.Tensor: + """Apply forward pass.""" + b, c, h, w = inputs.shape + x = ops.AvgPool(pad_mode='valid',kernel_size=(h,w))(inputs) + x = self.reduce(x) + x = nn.ReLU()(x) + x = self.expand(x) + x = nn.Sigmoid()(x) + x = x.view((-1, c, 1, 1)) + return inputs * x + + +class MobileOneBlock(nn.Cell): + """MobileOne building block. + + This block has a multi-branched architecture at train-time + and plain-CNN style architecture at inference time + For more details, please refer to our paper: + `An Improved One millisecond Mobile Backbone` - + https://arxiv.org/pdf/2206.04040.pdf + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + stride: int = 1, + padding: int = 0, + dilation: int = 1, + group: int = 1, + inference_mode: bool = False, + use_se: bool = False, + use_act: bool = True, + use_scale_branch: bool = True, + num_conv_branches: int = 1, + activation: nn.Cell = nn.GELU, + ) -> None: + """Construct a MobileOneBlock module. + + Args: + in_channels: Number of channels in the input. + out_channels: Number of channels produced by the block. + kernel_size: Size of the convolution kernel. + stride: Stride size. + padding: Zero-padding size. + dilation: Kernel dilation factor. + group: Group number. + inference_mode: If True, instantiates model in inference mode. + use_se: Whether to use SE-ReLU activations. + use_act: Whether to use activation. Default: ``True`` + use_scale_branch: Whether to use scale branch. Default: ``True`` + num_conv_branches: Number of linear conv branches. + """ + super(MobileOneBlock, self).__init__() + self.inference_mode = inference_mode + self.group = group + self.stride = stride + self.padding = padding + self.dilation = dilation + self.kernel_size = kernel_size + self.in_channels = in_channels + self.out_channels = out_channels + self.num_conv_branches = num_conv_branches + + # Check if SE-ReLU is requested + if use_se: + self.se = SEBlock(out_channels) + else: + self.se = nn.Identity() + + if use_act: + self.activation = activation() + else: + self.activation = nn.Identity() + + if inference_mode: + self.reparam_conv = nn.Conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + pad_mode='pad', + padding=padding, + dilation=dilation, + group=group, + has_bias=True, + ) + else: + # Re-parameterizable skip connection + self.rbr_skip = ( + nn.BatchNorm2d(num_features=in_channels) + if out_channels == in_channels and stride == 1 + else None + ) + + # Re-parameterizable conv branches + if num_conv_branches > 0: + self.rbr_conv = nn.CellList() + for _ in range(self.num_conv_branches): + self.rbr_conv.append( + self._conv_bn(kernel_size=kernel_size, padding=padding) + ) + else: + self.rbr_conv = None + + # Re-parameterizable scale branch + self.rbr_scale = None + if (kernel_size > 1) and use_scale_branch: + self.rbr_scale = self._conv_bn(kernel_size=1, padding=0) + + def construct(self, x: ms.Tensor) -> ms.Tensor: + """Apply forward pass.""" + # Inference mode forward pass. + if self.inference_mode: + return self.activation(self.se(self.reparam_conv(x))) + + # Multi-branched train-time forward pass. + # Skip branch output + identity_out = 0 + if self.rbr_skip is not None: + identity_out = self.rbr_skip(x) + + # Scale branch output + scale_out = 0 + if self.rbr_scale is not None: + scale_out = self.rbr_scale(x) + + # Other branches + out = scale_out + identity_out + if self.rbr_conv is not None: + for ix in range(self.num_conv_branches): + out += self.rbr_conv[ix](x) + + return self.activation(self.se(out)) + + def reparameterize(self): + """Following works like `RepVGG: Making VGG-style ConvNets Great Again` - + https://arxiv.org/pdf/2101.03697.pdf. We re-parameterize multi-branched + architecture used at training time to obtain a plain CNN-like structure + for inference. + """ + if self.inference_mode: + return + kernel, bias = self._get_kernel_bias() + self.reparam_conv = nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.out_channels, + kernel_size=self.kernel_size, + stride=self.stride, + pad_mode='pad', + padding=self.padding, + dilation=self.dilation, + group=self.group, + has_bias=True, + ) + self.reparam_conv.weight = kernel + self.reparam_conv.bias = bias + + # Delete un-used branches + for para in self.get_parameters(): + para = ops.stop_gradient(para) + self.__delattr__("rbr_conv") + self.__delattr__("rbr_scale") + if hasattr(self, "rbr_skip"): + self.__delattr__("rbr_skip") + + self.inference_mode = True + + def _get_kernel_bias(self) -> Tuple[ms.Tensor, ms.Tensor]: + """Method to obtain re-parameterized kernel and bias. + Reference: https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py#L83 + + Returns: + Tuple of (kernel, bias) after fusing branches. + """ + # get weights and bias of scale branch + kernel_scale = 0 + bias_scale = 0 + if self.rbr_scale is not None: + kernel_scale, bias_scale = self._fuse_bn_tensor(self.rbr_scale) + # Pad scale branch kernel to match conv branch kernel size. + pad = self.kernel_size // 2 + pad_op = nn.Pad(paddings=((0,0),(0,0),(pad,pad),(pad,pad))) + kernel_scale = pad_op(kernel_scale) + + # get weights and bias of skip branch + kernel_identity = 0 + bias_identity = 0 + if self.rbr_skip is not None: + kernel_identity, bias_identity = self._fuse_bn_tensor(self.rbr_skip) + + # get weights and bias of conv branches + kernel_conv = 0 + bias_conv = 0 + if self.rbr_conv is not None: + for ix in range(self.num_conv_branches): + _kernel, _bias = self._fuse_bn_tensor(self.rbr_conv[ix]) + kernel_conv += _kernel + bias_conv += _bias + + kernel_final = kernel_conv + kernel_scale + kernel_identity + bias_final = bias_conv + bias_scale + bias_identity + return kernel_final, bias_final + + def _fuse_bn_tensor( + self, branch: Union[nn.SequentialCell, nn.BatchNorm2d] + ) -> Tuple[ms.Tensor, ms.Tensor]: + """Method to fuse batchnorm layer with preceeding conv layer. + Reference: https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py#L95 + + Args: + branch: Sequence of ops to be fused. + + Returns: + Tuple of (kernel, bias) after fusing batchnorm. + """ + if isinstance(branch, nn.SequentialCell): + kernel = branch.conv.weight + running_mean = branch.bn.moving_mean + running_var = branch.bn.moving_variance + gamma = branch.bn.gamma + beta = branch.bn.beta + eps = branch.bn.eps + else: + assert isinstance(branch, nn.BatchNorm2d) + if not hasattr(self, "id_tensor"): + input_dim = self.in_channels // self.group + kernel_value = ops.zeros( + (self.in_channels, input_dim, self.kernel_size, self.kernel_size), + ms.float32 + ) + for i in range(self.in_channels): + kernel_value[ + i, i % input_dim, self.kernel_size // 2, self.kernel_size // 2 + ] = 1 + self.id_tensor = kernel_value + kernel = self.id_tensor + running_mean = branch.moving_mean + running_var = branch.moving_variance + gamma = branch.gamma + beta = branch.beta + eps = branch.eps + std = (running_var + eps).sqrt() + t = (gamma / std).reshape(-1, 1, 1, 1) + return kernel * t, beta - running_mean * gamma / std + + def _conv_bn(self, kernel_size: int, padding: int) -> nn.SequentialCell: + """Helper method to construct conv-batchnorm layers. + + Args: + kernel_size: Size of the convolution kernel. + padding: Zero-padding size. + + Returns: + Conv-BN module. + """ + mod_list = nn.SequentialCell( + OrderedDict([("conv", + nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.out_channels, + kernel_size=kernel_size, + stride=self.stride, + pad_mode='pad', + padding=padding, + group=self.group, + has_bias=False, + )), + ("bn",nn.BatchNorm2d(num_features=self.out_channels))]) + ) + return mod_list + +class ReparamLargeKernelConv(nn.Cell): + """Building Block of RepLKNet + + This class defines overparameterized large kernel conv block + introduced in `RepLKNet `_ + + Reference: https://github.com/DingXiaoH/RepLKNet-pytorch + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + stride: int, + group: int, + small_kernel: int, + inference_mode: bool = False, + activation: nn.Cell = nn.GELU, + ) -> None: + """Construct a ReparamLargeKernelConv module. + + Args: + in_channels: Number of input channels. + out_channels: Number of output channels. + kernel_size: Kernel size of the large kernel conv branch. + stride: Stride size. Default: 1 + groups: Group number. Default: 1 + small_kernel: Kernel size of small kernel conv branch. + inference_mode: If True, instantiates model in inference mode. Default: ``False`` + activation: Activation module. Default: ``nn.GELU`` + """ + super(ReparamLargeKernelConv, self).__init__() + + self.stride = stride + self.group = group + self.in_channels = in_channels + self.out_channels = out_channels + self.activation = activation() + + self.kernel_size = kernel_size + self.small_kernel = small_kernel + self.padding = kernel_size // 2 + self.lkb_reparam = None + self.small_conv = None + if inference_mode: + self.lkb_reparam = nn.Conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + pad_mode='pad', + padding=self.padding, + dilation=1, + group=group, + has_bias=True, + ) + else: + self.lkb_origin = self._conv_bn( + kernel_size=kernel_size, padding=self.padding + ) + if small_kernel is not None: + assert ( + small_kernel <= kernel_size + ), "The kernel size for re-param cannot be larger than the large kernel!" + self.small_conv = self._conv_bn( + kernel_size=small_kernel, padding=small_kernel // 2 + ) + + def construct(self, x: ms.Tensor) -> ms.Tensor: + """Apply forward pass.""" + if self.lkb_reparam is not None: + out = self.lkb_reparam(x) + else: + out = self.lkb_origin(x) + if self.small_conv is not None: + out += self.small_conv(x) + + self.activation(out) + return out + + def get_kernel_bias(self) -> Tuple[ms.Tensor, ms.Tensor]: + """Method to obtain re-parameterized kernel and bias. + Reference: https://github.com/DingXiaoH/RepLKNet-pytorch + + Returns: + Tuple of (kernel, bias) after fusing branches. + """ + eq_k, eq_b = self._fuse_bn(self.lkb_origin.conv, self.lkb_origin.bn) + if hasattr(self, "small_conv"): + small_k, small_b = self._fuse_bn(self.small_conv.conv, self.small_conv.bn) + eq_b += small_b + pad_op = nn.Pad(paddings=((0,0),(0,0),((self.kernel_size - self.small_kernel) // 2,(self.kernel_size - self.small_kernel) // 2),((self.kernel_size - self.small_kernel) // 2,(self.kernel_size - self.small_kernel) // 2))) + eq_k += pad_op(small_k) + return eq_k, eq_b + + def reparameterize(self) -> None: + """ + Following works like `RepVGG: Making VGG-style ConvNets Great Again` - + https://arxiv.org/pdf/2101.03697.pdf. We re-parameterize multi-branched + architecture used at training time to obtain a plain CNN-like structure + for inference. + """ + eq_k, eq_b = self.get_kernel_bias() + self.lkb_reparam = nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.out_channels, + kernel_size=self.kernel_size, + stride=self.stride, + pad_mode='pad', + padding=self.padding, + dilation=self.lkb_origin.conv.dilation, + group=self.group, + has_bias=True, + ) + + self.lkb_reparam.weight = eq_k + self.lkb_reparam.bias = eq_b + self.__delattr__("lkb_origin") + if hasattr(self, "small_conv"): + self.__delattr__("small_conv") + + @staticmethod + def _fuse_bn( + conv: ms.Tensor, bn: nn.BatchNorm2d + ) -> Tuple[ms.Tensor, ms.Tensor]: + """Method to fuse batchnorm layer with conv layer. + + Args: + conv: Convolutional kernel weights. + bn: Batchnorm 2d layer. + + Returns: + Tuple of (kernel, bias) after fusing batchnorm. + """ + kernel = conv.weight + running_mean = bn.moving_mean + running_var = bn.moving_variance + gamma = bn.gamma + beta = bn.beta + eps = bn.eps + std = (running_var + eps).sqrt() + t = (gamma / std).reshape(-1, 1, 1, 1) + return kernel * t, beta - running_mean * gamma / std + + def _conv_bn(self, kernel_size: int, padding: int = 0) -> nn.SequentialCell: + """Helper method to construct conv-batchnorm layers. + + Args: + kernel_size: Size of the convolution kernel. + padding: Zero-padding size. + + Returns: + A nn.Sequential Conv-BN module. + """ + mod_list = nn.SequentialCell( + OrderedDict( + [("conv",nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.out_channels, + kernel_size=kernel_size, + stride=self.stride, + pad_mode = 'pad', + padding=padding, + group=self.group, + has_bias=False, + )), + ("bn", nn.BatchNorm2d(num_features=self.out_channels))]) + ) + return mod_list + +def reparameterize_model(model: nn.Cell) -> nn.Cell: + """Method returns a model where a multi-branched structure + used in training is re-parameterized into a single branch + for inference. + + Args: + model: MobileOne model in train mode. + + Returns: + MobileOne model in inference mode. + """ + # Avoid editing original graph + model = copy.deepcopy(model) + for _,cell in model.cells_and_names(): + if hasattr(cell, "reparameterize"): + cell.reparameterize() + return model + +class CosineWDSchedule: + def __init__(self, optimizer, t_max, eta_min=0, last_epoch=-1): + self.last_epoch = last_epoch + self.base_wds = [group["weight_decay"] for group in optimizer.param_groups] + self.t_max = t_max + self.eta_min = eta_min + + def _get_wd(self, optimizer): + if self.last_epoch == 0: + return self.base_wds + elif (self.last_epoch - 1 - self.t_max) % (2 * self.t_max) == 0: + return [ + group["weight_decay"] + + (base_lr - self.eta_min) * (1 - math.cos(math.pi / self.t_max)) / 2 + for base_lr, group in zip(self.base_wds, optimizer.param_groups) + ] + return [ + (1 + math.cos(math.pi * self.last_epoch / self.t_max)) + / (1 + math.cos(math.pi * (self.last_epoch - 1) / self.t_max)) + * (group["weight_decay"] - self.eta_min) + + self.eta_min + for group in optimizer.param_groups + ] + + def update_weight_decay(self, optimizer): + self.last_epoch += 1 + values = self._get_wd(optimizer) + for i, data in enumerate(zip(optimizer.param_groups, values)): + param_group, wd = data + # Avoid updating weight decay of param_groups that should not be decayed. + if param_group["weight_decay"] > 0.0: + param_group["weight_decay"] = wd + +class DistillationLoss(nn.Cell): + """ + This module wraps a standard criterion and adds an extra knowledge distillation loss by + taking a teacher model prediction and using it as additional supervision. + """ + + def __init__( + self, + base_criterion: nn.Cell, + teacher_model: nn.Cell, + distillation_type: str, + alpha: float, + tau: float, + ): + super(DistillationLoss,self).__init__() + self.base_criterion = base_criterion + self.teacher_model = teacher_model + assert distillation_type in ["none", "soft", "hard"] + self.distillation_type = distillation_type + self.alpha = alpha + self.tau = tau + + def construct(self, inputs, outputs, labels): + """ + Args: + inputs: The original inputs that are feed to the teacher model. + outputs: Output tensor from model being trained. + labels: the labels for the base criterion. + """ + base_loss = self.base_criterion(outputs, labels) + if self.distillation_type == "none": + return base_loss + teacher_outputs = self.teacher_model(inputs) + teacher_outputs = ops.stop_gradient(teacher_outputs) + if self.distillation_type == "soft": + T = self.tau + distillation_loss = ( + ops.KLDivLoss( + nn.LogSoftmax(outputs / T, axis=1), + nn.LogSoftmax(teacher_outputs / T, axis=1), + reduction="sum", + ) + * (T * T) + / ops.Size()(outputs) + ) + elif self.distillation_type == "hard": + distillation_loss = ops.cross_entropy(outputs, ops.Argmax(axis=1)(teacher_outputs)) + + loss = base_loss * (1 - self.alpha) + distillation_loss * self.alpha + return loss + diff --git a/network_2.ckpt b/network_2.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b1b87630a0d413928420766c5655684b572d4ae9 GIT binary patch literal 26380 zcmY(KcUX^q^#4V>q)ikNQmH5n@B5rmNXW>HLNp|!s3^12p4z27mAzBzdX|Sb5{}yRK*M1F!i!Wn`%wZTh_8)}y8DTd@YXY3uzZ?61jOUgJ9ZVqXwa}KaLX6&TBpkdwBgRQehgF<{P58Cn#pYNU2V7qa=U1_xgUei==cy48E z_n(Xmnl%3h?f(((dtuce(^sXTKy3%kwpO$A?;+c8+yv3s!K7i>KWx~cY>LUzxgce} zT6j0Y+U~rELxWsS5|2846PrWlVZez0;BKA_Ll02K$i_~!`WI^K z?8o13#&E~tes*r!YMhY#2>)eOK#ceaC7ZITufH!QjQNkJ=AUK?H*C25z|8L0;3J$I zVqjO4twe{dCp2U|n%GcrzDvkZR%>`x;Aj_lJeTV&|6=K+$#&CXEbIa5#ci{imf8kRv^qkG!Z^A%7 zs)WI|k{kaf;FH0YXuD|}PSCTU7YY~Y@4HQSy+;nO@;2kE$LB&sls1~Y(`L(mz0mjg zAoBYdk6(TWxWVZQIoY?+s;p)BDmr;5?^Xz^V(F5`?3-h?ALMPv!yu3UX@~QS7P|pI8Mp#$w8dK@0{~t&etKl({Kbj zJhjF_ZW**Xa}AXC%ZER%{v3OEHvS8-!Hc1>9FV=7N2&Cn32+Q7o8-k~S+-oZxG%T9 z5ny9$AY4j+3#U#y;IaR9VC$)JsyY7+63eEcW)BZ+n5oG9cmQs@)Iz~xAU5w0L1XP( zFm8G{ukY)}HzvhkUUn6mImMGn(KM8sT8^ii=ZW*yexbY9N8;5KS2Vlg!(xyh%&<-- zZ;xpH7gR4Cl@95yeLKm`z9xQG48)?`{dK2%sEMXC%TV6%0rZWY2zOot@!XF!9G!ez zXx<-1mSZCM{jM@RRJek_NM++w-&8z#$^b688t}#^)=;`73O5eV!wvIq)7*%);;^nX ze!Oop4!DzqWlI;aQd0`PZc`+epEJ-P#E9>wYLN919ne+(NT>hWVz0;|eBKrcwz4X0 z{X`o-mAS)|$QI%EmRK~uTgr_!CG2Rk4oCa@a9~q8X{1=c_S)V;biD z48S7|ccH?nm9mWsc+mOx)IVW9=swhC=UR3A?|B5rop-`v!Rhq_24|sdd=1B4?*nR| zG^w}y8XiAc3peY^;l;Z{Fy1c!71|2$pxN!UR=8YyEa7fquZH$vsEls5ZJ4|5hjdHBsrVTy!M!_Jx zN}RA{4NJGjp=r@j4!ksnS57K|=J&-kMR5mPE$xNxM=e9kAVbg#eG87K2VhS44mNuB z8)o}tLO`M;B%IUb!%AhaXoEQ}A83u{B{kyobz%6?IR(2`mE-6!V>xlW8kR2E%7rfB zI8=8z?M@el@C1)}n`#TS;~2e#*Z!5=^8V>E75CINxF}PcRLI>|^#QoR7rG z*KP^vCTnr|g91FAIhZ|8EyJCKCNNcNF5ak|4eLIY;FtK(5PdltYxAFz)Q-KBX}1Fn zp2>6gi)8-$a|#D{pPhA=!|_K0pwmv9b0VazKla=6t^9SX55YJ~ zLzg=v7QuDtgD`&cdU3WSj-{j5a9xro{GC^TsR!h7(9Io;v!-Bel@b-S%h2CjgHiTD zmViE%Xuhm3$2X^PykifZKWaL5+ztbOJ1d^GvKdU5sbl}-o%ln}0A!bjVLLe}`tvJq94WEr3&V_IP&?lcNj2&J=2YrorD0G5vfD-EU z4xlo6M{6&lQ1DD zp0fjQP|2YrK9uK%TKWz=a_>-H5$gjQofG*^b2E8mR6>oBJ-VAj-;n)l-L0CjeoFpO3X`zBWA7*eq7G#qOe#MZMz(M-h>7cZ{^lZ`nnmFbCI zfzj9;9mnchrh?J8V%+;do3)^rZ7d&<%Dk2MKxYLS_qIcuyOVKy#76w!lLjx{Ce(G5 zug10;hbiQG25R5;z;DJc#pS10@f^3^AUpT8aBJupIJ@s&ed_r%z!f7n?9myL$4;_5 z@mxHu)PwbpnedkMVIWu-pw!$OqUrZky!tR+bt_3aNdRs0o?b|liliaj)@r5NY# zN#(`=lK9Ie9~`kw;EJ>&7vHO~eq1PS z6l-P#qk6v%aNq00gA&7WV8diwCs~P)-3If7rd}MYzL^75tkAb<5yziM;;QA2$cjHH zy5|J+I(%JN9#Dvy3Z{H({XQ7;-ksXt%ahwsb=HZr!0WA1?7V8ca44)4Wv1B?G<)I% zBLee-XF((79yN%;Sbp)XaL+THZ?#;7g!qZvb3--jc0}`!eH z0i8RN!*(SJJbPRqo*G#%5eHSk({u%viWn&jFKiWBisEoCXO4}^%_Y-sg_!=;9KCit zgB@+gl#$b)M-ShDiefx%&$1+o!U{aIG?f+0?WvE>8Z7n<6+iCE<_{}p(x^>?*geA- z?OvzAz9lng)!r-Ow#&!Kar<0U&{>4PAE)xgtBcXeK9~DC&Ey4!?ZVuAb&l5b!J(-m zd8nry_ntq2W}UuBq02Ma`K3PxZ}&kRya(@t$MS*&;VjeoSF-%?Ku#=mCY9%RBpa55 zvuw`>_&8`FCY;aVUPe)HW%EiJAYIB<($z3ny^3#E#qhdeY3Mj*IPCsB7jsP$apzG5 zq!1_MA-hDGwpwyrnT}?G>oKThC8~}1PiXTC!S`p5iYJ5>Jn!{V9+(${YwixlhnxHJ z@sfIa~-)3Iuda&t&6wuVOzDC4J_)#BG~6^8$F>)tf^IbLxfzU!%@)E& zNfk%<@4!8=i@CsN49`3-$FK+S?ixofco4!9*B9e^i%2d(Wn5`dh~Xn`rLy@ zXpVj!U#rf;iix$5v?_!rq~9Vf+i|?-dp?c>b>5eAosQaEgxzwHFye=Mx8AMcgylw( z%H^lQEmVPL?;J;uUOQvXXd9M&_mMvPpA?6zQQ(c?>3I8DB#u9^kmnSB71x#Zz>TXb z$YrNH7d(x{n(rBS>fs(?*8pQ&Id2eu_UVfz2K(vUc|AJd;lzdZMQ|l{QMcA`V(pDv zF?W2dWa;zSoc7`bxE|dOFV}Q|*k=cYhE?(J$45y?T3X^cW0$bAcm>BV%{(!n)S2bl z`lH$`BV6Wkp01sJ2adrj@p}xSL&+R|y(=GGe0AAo{6XlfkD_a{Q_ws<7q*V@!HVa4 zJV-l*Pn}Wb*3U8Y^YT=F;+zS$KkpSwulFIfh52mR@>1}ym%>Y56)-iT1cppJES{-+ z1MgMs>3H|M9~)MR+KC#hKky$MZP#X_`yF)Nax#Cqz8B`N&t~J2VX#iz$Ok_SMumI& zTr~U|>~Wlp)+x)OwFEfh(E@HP0SI(@EzB=V5)5zrfzRQ2;u!-u7AMXJAImZvxlj)G zC+TsM{!ma{;)7G8lXsdiA8(AVkNhCvHOm3T__-uA^fXNRtw5{tw$ts#5boOhxjuVX7%2@& z;7LzT!PD9SDF46=(-)?(ak?5C%a+1FyHs|YHx+AtjTVdC=JUDD?XW7R45mG)!mU>R zxO>?$9{V)|A1E*4LZcF1HqMnT8yunkXJ6jErzba$jU{87MD%-Ih*!TA@gnON;2+sV z`OD7H3?f(NAYGmQAj~j28lb?h?jKEaK#ijg(wcN=NthW4Vq53JBdzQr5E|G{}|K=V? z(F*@m|3||H%tPI|8+qs{WehsE5s!Z=VL83abQz+!xLYrl9rwq-nXV*vzm6RHEAh3L zFT}}jlG%K$6VL9<&dmCe=YWrlTHCTZdthFdDTFl#zV$9Q40e!b`7dU7z$>@R#UaSwpe;cFliQEXj^(_nc2YA5ulf%*d zQ#L!+?EqK3_% zKGC7An{e1WcWkoSh93@mhF+>lywbmzwK84!RgkIJ=>C?pn(u*{WCoh6UnJS+be`2$ z0}Gz$^Q81(oW8<I(XZGOe|68) zk+tC@t?DlJ$y&{Cu`kMvun}L><=~CKUM%%|0?J#?KvVbCI8?p@f2bW2^U~Jxkjv{Z z>-IvtJgr(slH9tvvi3Tgv^+Zc}7cE+0rbCmOZ<21A3Ls5i@oLXE4q zdC71zx?awMWn)m|-vacmmE&)dT){OW2$!}faP?(R{5^Opk9 zt6=+(t$Kqqh>b zDRD4Xk_mey)%?u4fG6)vg^#16_;<_z^bXjKy<``6Yj{((+nCPvQv)dG@d_@EyGI*7 z?8e#U8GPVG3^&~m!tDvBVwL4gRQN9*KYMu4)&BJmWZ;KqeQm+f>>|Bw)M49uA^1fp z9FI#yqWjU^^~=`v!{-qOLjB2Rc=>oPC4U?uG&+=G;rTD(jbAD_fI28v?MiZ}1 zFeF$S*A}j#ywEthJbE;z8ARa`nU&-?_yUc{^yZp{rQI`a3jQ*)XS4OcsVz8~H~krd zOKPLXdi*^D&XNIeOrwHjPNm|R2jTpwZa>5yDaI+f@s#_nAO7&ZDlA@GfaUhyjI}w~ zP@l*V{kCxe3#*Dp0SDxpA=EuyK=0&u!DzpoPdJ`#h5-*K+TQ}?6YbS z7eAXw-{Wnu@Oy>uW0fyAuZ=^qs#0;``4!MI_8=L`GBmdbW?Uo=J1@^LIn=j@}2x<6^1b02V? zmy0He>p52QG_CK{WRo!k*mn6VnGV(9l`1pPcgsWQc)9?LOuX@WQYnAF5`vYsmuQo| zJGTmBguKd3p4!gvU|0+mOvtmLx78JP~zoIN|Yig*-2)m`<*?g=am}IQ(K6tWDXC^@;YF^0^$Rs_9{-`4jr` z!-E&ukKo4-llYY(;QWEvxaD9X{Qf=|J?;APL+2?Ht*gB``)2|TpPs|>t8F2MCH!k5 z;6+fy(SrtaQ%Di5s*ggIEIG{Y{}=X$EaUcvoALVM9sKo=3%lCopzebY)Hx{`)XN89 z+fP-RXH5%&V)gy?0wiUt`J>kOr%w-mB7=XH_lYt2zeS- z_~m0dMt(>{9dR)pv+jfyKc}~T>E z-zq$%8xgZmf7^2Yy)+;8dKcoX)qZ&S)+iyfP#Z?x&)}w3JzPX5>pe}E@GN6R98}}O zQd8n7v>x#L>mc5n5W#J0E7-2`HGEiY4@X;jf_z*VUO%ad=Yns-mg*E*)0#l1r#WE1 z*;*v2Hs+;uX*hqI6uq8%gt}r}_};q^ewnTbe%OotSZ>Gq^gbN9eGfFbZAICM+4TEL zE|*m&;^2L`yg{`H|6NIfBO{Dx`tMBqFsKa%dZ&p$XDz^yZOc)!%>k#U@8rsTnxctA zB#xE#7LDw8v*9Woq;MbByK|q`?OuX+);RN=jCj5sFc;&-dEwV8b&O4p;c1^k@a*Dk z9J2hoxM#X4Cy9Ze@pT{dFZYCt?FHEPbq!0s>7p-}-O#^s1}iM{#YICeLceZpu?feJ zvsEmO4jzGb+I-Ps%6`epij$(}`Qv0OzYa(GzJrauck|$7%P47jjW8fG8o&A%Q0k9AVEkhA#a0@d&Z+gjn7fHVg_b?|>0^H; zva8{j*SB-@FLQcdGYfwI@|IAK*Qy|F z+9BH7l*5BcyXUyibtvxfPjap$9ZxOE!vB_dU}dC)lx5^-_Pfn!-LZh(?S?0pVaN!%=N&Z84KB9j}E7gE8_xn6>j}DLWrw~sK*%`AInHusZiQw8TXg z^VsT(CMOj+px@F8%JupL!>?wDF9z&^XMaoiyPi8%VlPfK+=U5mwcvsADC%pxk;j?r zrg8eiaKoYmcDgtn+snR-O)oPzdXp-ecO>!7v~WBbSxCiWl93Nu3-`twL*s%TP*Sr3 zBcfaAZIUh?HopZ+t*5ZgeG6KDI+F$^Ex=t7mfYt@Iv%ecFRB~~;|T*-+fGg2WScwm zIeb4AgC&W}xgir#>R%x~-yXq1^EQ*->>jL>n!&M?7Vx3+5!`L{;AhQb5Yo!&)U6P7 zDoI79$nNKyZkE4T1Bd?&ht(r$pyRI;$IV~LTWhycY4!pR7mE4rC@+qYRH9y6I$LT@ zMwi1eG-2k1y=a$8^(h z?Uh<+x)2A$`_-UbbOC-D)|1!mxIkAf@2AoolX<|rIDFk{R)2rMc(#0yDE>Uu318eE z)3!VD=nd%@6%~X3wcCo5{&*p7l0pOhomljvTZhb-=8hgiVTFDWuU>JMyfwWsCF(LX zEnEs)tP;rW-f!{1^-Q$g(+9roxh_tf7DYiJsaW~Wfrs~tvb_`dm(o{jigM>K(H4Vg zc)czHFKPZHJ<*-dG*@HYj#cdC=>{7kgK_YiUHJIoKo)*x@VB!`IC#KXay}l1D;f~@ z<)>mmr9Ol>THxij0@nYwR=j)74|`Wx3xB#y*yX6e_Zywrp;{NOEt3ncRD%a3@y)#M(IH;);=xfz7gHt zY<)gt^bh39{-@eK}>^Zt?RtZG81zhey3y&u+0+U^x3CmH$}+ zo8PE%zGM)}PM5*H`Q097Wi)lH3l&#fU5eLX0NY+Hq>SEkP`B8EQ=dtr|Jnr@;xL+L zoergrWjQ?Rmlh{}*T7LL67bTA?tQposi4FCMRDv%_*zuNef?5#>1|`4eIx^?>HE`Z zn+fEBu42X8Rit9xy?dnk^000nN59<}ztoSWe;LO}_0nWaxnhix83VATYX`oW;)gpY z&gAkda!_SDnXfN2plyw#vGeb9@M&rkj=yt3tC=tAFXh@XPEN*S<{Qu>Jr#`>D)Y|u zpD4EXee#*;$)k@IBG+2u-`Kuj&|b_kPYrolpOyHpuM=LXvWBqtO6X&74aSa_6HD)! zfX4?H7WWT9Y}(0(XRf3JHL7Uo@<|xh5`y|ti`eR4Kh$4ef#)vf(wprz;+Sr4J+QDh z*3DamI{xA8QJIJqp0b$S%_6oR3B}S1CpPhL<+ZquhNWj?pCC(`c&r3hJukuA6()T1 z&Su=*mICX&1)g=ijAws*24luE4J=FH{n4FpPr;v0C(CmF#v(2|yPa~h>tV7^3Y}eF zg;GC;@buDbyi%pkMmg8&m1CxIl6eZAj2eP_TuXVVQ8*#}5EYmrUN8#G6aNKZnExP?u;$;bysHN&F>J_@6cYPZAd5`Ds2KPbv zm}C9X?Hv$g)9qhgv*DeO$FWM)YfAfX0L&Vth2A+IK)dn)^?p)-UhyYsR1aAWne&C# zY>MPgi&n7PDWIXnO~^l-N^|@-;MYx+s8(FW9aFdS$6^`d^OazGjNjXba(32ryG;POFp48iqA8gFU#y8!ZubX|P z4Y9)|-F$ZPh(g|HxRAGe)WkCNY1}ZUA150`G2KlRkNBP?y)z%d^RTkm8N5r_Uu(?k zN=i5Y#*6V;EANi9G(-dd(q>DW<^_Uhqd(_g80Yz9>p2$5r55uIq zt-MCrnYY(YMeo-$cvEu&lPsNJiOvC zc@x)S|2c-}d37Yq=XjCRgkZ_tvQcPZ=*>e85^^T1atR1h(X!q^{6DIJT%1^~S%h ze_uQSXABvMdRjeDdRmeY+4>M<)bSVGp9m5;twg}R*7O<4MvT%Gv9`F7;mi2FJ3>Rb+;c>q#7--f=Yxn#XZ(8pa z3^%_9Td5zZC4C9-PM6l=bP(QH#O0&E)#LjbIo*ezKyP!6^_ zJ%G;mJ3{ksAd6n>#VOtNcB_>lo!|Nqp5N`u`AZ^5z9b*!I(O0A6>hNQbq_#UtX)PW zI;G$ntY5!dw~~C!#=xmDHMGLY1RP{P3DNtagxat7gxSW8^^XHHDe|H|SDWQSzlVD0 z(WFB%n@YrIDcaF!>)1T6`Ugzm&Q7MewWCNE}vp{s-M<)(dLp9u@VHal77yeBF@vnrS z*-ypZ@)~$)qZwqoYH@U{Gpw1S4@H~%h-UX^hidIIOph$3rCZ(11ujwpw z?X-eF3wi@eS;C}$U9`-{nO?=Nh7!$va8d3%xs?8Z+`o$W$mc7REKC+klkd|^+ul@n zIgi4n2UBP1b6T@+IJez?Pv)MzgxG!G!1RbJfBJkw$b2iu$;Su4U0o$Q8!d<1jv{f18pbmaq^{J0_ccYah-n6S`6ilr;E?M~GrZD7d3D~^3 zK$G8SamauiQa0+r_w;|!BfHT=ht3O)jb-Hi>NU-C@sw=+ae#JyB7qI^B+q!>qEGTj6bg^N|RMGlRJ%qJX!2$QZ;>i`W;oH-MZd$J=1XpemHPk-B zw7a9=`?jy-p?rdV3Mc5x0}siz=$m3~$ZRq8SEA50pjyZ>ZGfOf{iw0U1e#h#(lX}< zw4={Wx`2Zyn9X@&e_cVv{14r5N)``xr|}l;WZD?tNBA%9F|Es9M_1qV!ez&23C9Pu z3lm*sz+Q4#=r_h4nj&|=*(D~}v;8+M{x*=j)KzKDr{feb{j@k))(WDY<%ss-*ShJC zDK9Z=gNW&O#2d#;Al~9W+15z&?K?k&+KLpCAG-xihD@jU{3z18xVS#BB_Aq^JO#)2 zg@B`uleSc`IHyB~16Fy_`th1j=WrByi$iIxV-_Xr>%rz;AH`Mr$`n&K4`hCX(puvMWECeN{eDy75Vxa%a`t$GP_y)KgIIU3$sE+&_L#gyNE3f$Jqg0xE?n)f3D zZtEWrw%M1#uz+HS=-WxRo-cx_ax0+7Glw4C+Y7!eWs>cVs$4MGyI!%w0r=f{_*`!- z^tW9H59FnB=k+)BKVHtE>l2!3fo+EIrQT?k?>eSyQm-MvMg|m@+ z;lSi#apbc8g7FeRDmYmo+G_NNv(B+}^1*JQ`a`i$)}3Sszy5*xd^#zfv(yt$FCR}| z(tc7!W+cp3UBLJK)!1r_uTUr^!?6jGFfAnkUUtvviq&|+w{3zD{x_MX<_;HkpPxa# zhGwX>FrSR)JJ7nqVoK96CzY38lJJL9;B&?~Iy&wNbcPNDmHGD|IPf)1x^6>WBeUx} z%-ZPf(rj8E7Y@%ZO@jLtL%1+YQWsaFNcM%Y;GOXg8rudy#vXGxHSYG5R3)`Wk^ErK`!Q$0)KM-tVg%7$1 z1eK(I6r?o<0^Y5ogLzjVb^1a&_;NASYdxiBxAo!Fa}}JTUqfSc-bg-l#nI}SyJ74| zTk;-~1M+L~!EKxpADWyexW#{_{Kx`odcR9}^5hp(K5zzW=efe2cm9wYzDE=$JAm18 zMbYnPBc<1^fTdsk(e+k!J^!~u{QYntU7Z>yreF?O`aG$B%3p1l>>P;scwba(lch<0 zo?7SXxsm2}8~E01sZb(+moB5MWP*_t^uae{&!&6>-Ht-KwSF^H4m85UuMOe+kH7S{ zTlG(MaHOf#mC(=iiljBXr`T8XiEv@c6g;0GE$lZM53d);ilO1#NY-tW@ZT*fR)j_> z^mi8G%YM*7v*A$f)15$ZwSrN(`K12%p6I%}kfN&1X{z!dLHq13Fd1(NhUW=p#wU~3 z)l;zf;V-cHB8A>lj|uPd&Ow!2Fb!C(4P#g5i}y}O!)$MDO4ckDYg7rg8oi|r{!L_E zRSVa?-VrXy9uSW%Nrk|!T&kNFPPT@j5IWqJr~R5wZ`YrK5Y6s7(=upr$&9+%%O&FO zcN5{q*(y<0FHxNSp_DfER^fL(^>kGFAsI&BrGapr007IENOrfYZAdcFIzPD^Nc3E)WDUSPfB#!yy(1<6rB+pVdbnOy1Pyd z%Pz}~~SWBZubKn8U9U z4{8W~Dk`df5MQ>23i}r?fe-VJ3KOmj;hFg>>(BTH(6|NR;PUnaHES({u67xoGVP8K z(vnHTC*2lr{kto6T*v{B+qv-g<1QFDzY_Kt+y;f4C&hWsWkjhGX}aY-kJA$txJh`yh_m&}d^_krr-NV{+lQ>rM-Z!w6vL_&;8*Kz zx>*+llkUsI?6=l+0flvv-8LhjkG2w&>pY`vZ(~L8bE#zLr9$V%+hXRkDnUWEroO=W zmqd4U9Ie#Xx-IKb#(O{7?*hfsd=f$-C=45Fkv1mpfADA8^woY1{VZZ(xueAN`M9WWQVa+9gQsWNQ1 zW=R7x6Tqk7C~aGzM;EvMC#DV>NyAMwdEBV=5U#Wsd;-?j2X`jJbBhz=p>DVNTB-{S zEg(Tt_oH}Q#Z>rcI35am9;W=Aui<@;5-Ym?6oP9~p=ebv;n0D>Fxsg*!Mx3mhR%)_ zjXe94R8cUP+`UOvq9*7cxn5tWJp=sv-=^m`2T<_HJz{FsZzwi25HEKY(6cCAu3Udu z9KCyqz$4aC+Q&G$|J#~e3KU?)J84Ru-!6QrsTG4h*U;yBpr3t|A#aKcc{RO+HHNy~ z$&oBUfm+0)J718ZRS~^jpF@Z5#)<3B52UL7K<}l8(!Dijgz0kObRp1BynSjNU)h}u zPs7H+=~fG9-e>~{GjiZG+z?OCtfN#l141 z3l>$BH8UO#>@g$T&n3dX3)3Zwy0aOsGf6j=H^SJ;!%hZOhRq9^X-#KH^X;YRpD*x$7Qwk+tyVI4|vTKP%+ zj3r-%0gf+(4d&{&H*5gk+qtWL;bwx~PTiWZwXYbv|2e%_>PHG&?8V%`a4|y$pssQ# z95T=&gSBe)4=hH~)WZ=Jzb{ot@m@!&$)hE?u203Armd1|E14RG4WiPh`%rH?n3TJn zUIe0w-I_v;(l7|#)t3x?^TF@$K3e9qN1W)S&6Scxv~AcI(Q4KgS~_x%P~7Z_Gj4w( zoo<~tSG+~#V-Ui=HIuTmKiK(C6g6Weu~|SAAM9$PfKCCtQ@#mS(Xu#S?u6KGe@*yx z`G=@7>lfrH`-;}%iDnJh2n%w|prB5QvWBt{*k=qqn34^f!&n$|?Xg6+Pc0k>J4j8k zA)wZ%FFdDBa6PmbI&Tz0X7nmiYlsTfT+ObJ&C(TaCLf?J-3f$rnU6xg?HTc|<`3ai z>1RkdATI=r;E72U!vrp0xQcj(C&yS zmMlvm`*(r$KX0jW{PI%Bdi@C&MyQLLR~32vxs%|2>AUc;?i_`FHb7;EWN7z^qj|P# z#F_zbAwe9DQ{G2V+9Z3J=RJb5uFQhok>!$}>KT%h>{pUE?<{EQN_S9h59Y61K1h<5 z9XX*>?LuiM)_~pbSYfh7JpC=%3laT33p2wK#g^S3uzq=oXq@ww4&}{+r=4v=P+%7{ zIp~3+gAq+L=r0oiN(=b=+R^?M-r}vdQ(@!cQqYW- zfUZUN9jcZ_4|6kMt;7WW{z)Rux^$4(rPE^l&*G)0ZKN7lA@+zrQ*RJiB#sa1PLmoB zVs9M>IHr~We!-?Bs+f{ai76O+&8Ioqs&w?9l~~!;CAbMu!sw85g3F7a^g8h~g_Yi+ zw~5as!B<+y*WnHI>S+$EwiUrfx5x0`Gc5|-ezv~2E1L{Ec0r`yPl?UHOVlJ@qbeC2 zzGgcQx>PD5_N*N0Dm&7T);ZvE=)LIQeu12`9zbM>y67}XiN}4~LS3f!sBuUmc>4}aDWgW4npM7me zKXHHk-$kE5S;if(<0H&D{{b{}jD&}`cM8M5{t_LR|Dw`5?@0QY38 z#DjiILHpVr;p6UL8n)pyot_v&R;^w@Kjhf=NG54I-y#FaA@Z^vj5F5N3!kT)mz;Tg zit2*`;k#$2*wbVsJ=5Mq{tEly(wdWEv!f+vEx!a|?<&Pr#=fHe!dmgWOq^)5p*P&J zl@N|?6%IaM#PhnzZ{30+(6C^nnCPY;TEwZ->4UbA;I1LoUtKO#PmZKxv08X8rUmYO zIU>AV;zd79)uG*VK0gVP#c%7i;P(Svs@;-Ioec-+N}nsxzFv`2dUV1P`97?=WFWsS z-Cr;5(<<2<^H{P-CRosT)*uX-xs|Fl`f=!NV|?!(MRPJ7Da~U7&7P=@M#G9kIr9l1 ze=v&l`;Uhhd%OC@of|2|l!PDFQZzY78jDs-)2b_3(D~yKh>HKnea{PF%F5Y;`X=2^h+U?{!?#Jup*#-)g42YSv-+I& zCSF`_|AbzvdI*aS*ieC`GQ{2MHUi6Ez|FqtlJ7H5z}8cTU{~m)`cIcHL$dKrA!_Lb zI{L^OmR}B?)|Lez^Cp6(yuJXkTXsSC@d(&IU;^2%|00A>U0AR2T!!q7 zc2Q+$54IguMqf|;f$96Ipt8{mOueRw7fRKk?;m@ZF@6j;r^LbdbI;-Glq+InLNk<& z*P|B~Ckp+2YvA<5nZmaj*P!ctDug#prs>ai(1-C4DMwm>eEr@yd%hQGuiQiT9+X1! ztv=v+%n62R6@t&$fne0D5PpB^P2pwQu;|)CNGa)dxIg>Alx>x2r0WTc^xq(p>D3w_UT-+$md&f`4J z5=Wj(1bmof%_hnp;T^wa$XnGwLFd9mCXhyg+ow3NyGB?a$tA?2B_Ez9M8h=4H&ie+ zkY^C;jroc_w3G@$XY)mr))*i+v*(cO4u()zln>u*qG{aM`}p$vM81-x0>Xa+RL9Q& zuQrE}5w%8C>zl>TQvSuaO>;rN>~;8pN-~eEl*zueC(&RtOGXx*w`g#YV-D(Q5|tSW zjIxX^RK{Pxs|I>}5B)1tSThvgsJNk<%tYe$V-cPXn9q#b9fI2`FYvuh1TMwn@)u(e(%XwS_ipi_8_SDko-=KM%Qk2sHzZVLvkY^?;uB^j=ZE-lo!+S zt`qb!M>YpG+OU^Or5IZwImTsz8q+%zjQhUHpkc{b>M(sSE~xTgM85bDrFa|WP%VQ! z9um~^R|3Rs&*sS1EwEli7bGL4(YY&(Kd@&bR7g%`HR^|^)g#=ve=y09G&7rKF@#gcN7V?(&4 z_8cDFr$iJLb5Zhb7{s|OT7DAUlN|>mt zfsrXw*$u9t$lKt9eb&-g(ZnTO#%92u-Al04FC9b}7jXG1g;QFOVXVhGder7APDtcv z>FXKLZ@3H}>sXSHOBLAfH_y`qv2(OcrW(hT9ML51DXm^F%L+`tLlT#$F=srcv#yLj zG=$WkikJXMx)$T(U)ebJU5-_;5n;_7qfs_<7Je%~j|V)tbL3S!HeMP-t@;qmz4xA~ zImN?VY7TWbmDx@!QAY1fAxd|Q&^4n=sJC1yo&2qy_giQ+ohPP3ckM00R#nE*XmU`UX_Lp ze!*Nj$VoUFE6e;gltw;08f7UBG?Tu8s8POUfQLX+8a7W0$fkLe}YB-D+|x=)eX zA{iKqT?dlqe95C?305{w<#FTp%iEYq1c+gaddc#BH zoXigh82d(!*@dF({e{e|hXr`tf}5!aIxs>z77IDTZ+_fPV#ho{A1?X4dOo+08D7Q& zh05}%KS7|ARfA>Rj+r4_PLrEN*@_=}xbBEQNfq1xpDWWyv+5WARL;ve6daGshL`f( z=Y+xXQ$DCRC{B$ha_?g{4eoWe;*QHDFwy8C-z(S^YSu`>S)l}6b-x4)Wrk>Z+b-a5 z-b}ltbNDG;^LX*w`-w%85k^6-a4Wq}jC-Yt=J5jJkeklW+rEX2{~6)2B|czkm;>n?u_dxc1hx%6fQXS? z7-XigkN>_Ud%RzR%d6{XWb+tj#l|3WLyxU;UrA=g4x+tc2evh*(l{eMtP+TXDC6^R zw0jx5>&78G+2al=LbEZnh7zs83Kl>U4wh-R{HdD4(6n|E}prBzred zxnwciFt;0122O&(UJIV!<@H3QG7F!rc}wyh8^Z7OYZxDDO#{#cd~OM|gU`?7bMMob zzjGrdq^^NgCA)E~&lIbL4&wBY2z*dpiB>U7@s`O{*wv~+C+$h(C(l@og=QU~c2}KM zf0B>&2aMU(!jkM!K^BMG3NZ1s3U;h$#-G*I*k5A>q~SaLbl>F#5+BQ;234duxy6kLB+6GHx^~{2gn+ya`n*=Ff<9DQ=0XohLvPh)jv z)HNK!-j0)P^QL09;w@~-wqU*$NOQE;Gg|uZEXHa^;YKc*IX-;_`t{6!YSUs66R5{& znPTkI<+A8JtAOlZcoyczorcLXrXsUn01_&BaQJL188e;$4Rtp`*t`PX4chZ=aUQ#q zu`PZ*7R=iIYUCG%Mu1In5=?50Bzv#spl#MCTHEbFOp2r7=Z@RpEg5Vv@W2yQQ31jd zjp$&r0E;t|p`=c#{K0HN@EfVbl4+g1ALD75(5eQ37jnTXViszh-oxl|?LpBtshF9s z!5(S0WSqaAN9!?8)Avv!iAUDq@;oW1=@G;_#cZUvrAW-@o%lw$4+fH+5VtKlG)b=v z)_3~DdsDDBiH|b{J`!&tr zJ8D2y_+~(2i4$u&zYndo){=R>qO7QS40T&Mi%Gej$D7dO074@nU~waww>Q}yw?)d z#f(Fwf&=MZ#k zR$+ao@4`=2>a5#B1Ms(92u|;RfVh_?6zN;wxyAiBT~`40_QmstZziMnKS6BXwhxre zb6_Mq2!Af9g|w8p%u&M#?2$M^nZsM4yEzVr{!K+eGb`xSD#7UuRwTCb8UKc71Yc8w|x_0Zf}9k4R-fCJwI!7EaOM(1YWxQaL%@iGH%i!;z?V8Ong zQou9XKNUnXC*yzT%E;7!1HApOxc|aWpOr!az9;w71aBkO=t~djjhM%L_;8;#9C>GP zY4{y>2D~QrtQ!0ij^GV-N-(uk{P1nlO*mO`8fHf6F)9G~Ot2l+Y?G%gh0Z7_Y=YyC zGCYkALtePW6{wQqW5>TU#5#9Ay!VrZ{_0|cc$Fp6{r8eEjORQoP;B=_v3E7nd%CSfmd+l=?c&tdJpSei%?5L1^(;v=eP+y zW^|=Bt6x41KLv4Au6HaXO(@3YC3;x9vkHG-wjjGA5<#D@z+8#Fh%p-)P~x3UdA^Mi zF3%IfoT_+)Vp0m$sIt zkPIn1&@`51pL@R|sl(S%A$`6#%g5rvISP*u|&53Ejx?;CG` zMcy22{C*2tXK`NO?zwcwN(0o-v7>UYB$+7rJX}?^j6S-01@}K*%4rm}Vg+o-X#HeSebh3cEVru+H?}Ai6Tf^%aenC-2Ieh0%4WSpfkq}-<&WaA|LCq`mQ;qo~WkB?8hb=Ch`246A=D;53iFxF7KXg27d1(ssHpBSdjb-%^y3nfe#+S216^d^@k!mx_>)d zsYs)dcSWgPTPS!|e}o$mKS|e=za-+hI$IOjNu%$`1H&biLyxs#MtKVQ+m+L6Ezh7J zvK9}^^C@q79EiVGV_d(hQ^OY-7-oEpTsSE~cuTeT3W@leD;$>Rk_N_F_B9HX5g?4}bG-yIW%-w|f^d z^&nytfv*qN<15Z9{c$W8HZR}A*FqCK5Lk#2vp*3j=PY2PvLJDl5*j6Zfq_^@{8@2_ z6i%GP4nMa?C!L9GnLdxVNoWgE%~u2;BQY%b{thlmEu(2NGnuC3PzX2IW%!Ys;a+w+ z-K8`e&x%YZ%g=e>ocy0KZhVi1xtBoi6YDK8J~j(& z-$?L>SIlBmBec=QON|9LF`Qvw!pyh+K{6cn;M|1WeD5R8816BFDh=0>bRjh?DBgo+ znXdFiKsE{`OMk?A_ceOL1L~0o0>1rwm4M~yI4*3a$XJ{4fTP^c6nH@FU42v-QsvWkWMjjRFGB5lZC+z?l8~AlQi2sBQ-gvG1+rB8V3C)Pt0e-fwzKe zvcyNadLh>cmKcFuwUgKumzk_dm-?O*bPFU%5naa*=Q-o zeByLew%=sB0%uzHzc(EQ>#+U*|U ze7HgU&G{qGE1y6^z&(IeF|aul0`K0XplZE5)LZE?LSD_VX0{dc<-7xY3c3aFIS$fX zcpB125Akr@WOmZe(~um`rPH6TMY{qZ1=r7Fy~-W_7qMZ$A5lORQt_pqFtd93aWu4a zV7B(GCk<2bxij6A{*@47;~YXrQMWEzmM+8O#cS|ac*?QodNLr>{V4bC7GQSPQt*uJ zv#{xlwpgEMLPMSq(Dobv@@g}_`Sl7;<{Hx}O&ZMMN6T1;0|W&^mojEw>gkp`8`83B zCY!jdk8}w2fc=%%_;mhfXmHHp<#$|w()d{HbT@>X4+MyDk_Yl*ze4q98OAYGnw@uB z5N3Eh!tj}knf|U2$T)O|>PX7aK+8e!jSGgx&o@y=ZzVg^rjx#L@}@dRZjh~U#>+oG$oDxD z1>==ma<(A=UrFAC39B1;FI+FelR2(r;?yVnmm{)_Y->G89!!EYZwuk+g3tU(liq@t z&`bE)a1B*Ge(*#86M$Fui}Bm*MwELog_&{lA%7@l1C85h#ZPUMVO#ws)9xfTj(cXQi zmubn2M(LAN-kaf<*eI?HR>lL93!%{VF6FN_ViT0+uS;wa z8iN^;yv2B4N{fwlU&=bA+%n zoi;{9!PSgCICeK4cdjc&-enO+v)6+!5Vsu1kH6;{m^Q7$hUM=HeFYmHS!N8PxT$?h^jh< zOe!UI92@-l#31R+w8Enmk#J>>9_l}8M(d{k*y6rCDzwT6o~kP0yRMxeKG;PMnmVEY zZwJV<&SafusbI5O6-bWErY=`9f!!pGwg0Zuu5@8YpDc=|;V zy&8s7J($Y6+39?|z!gg{t7TKOJO6juABcmob=gdU|phTMNE;paqk7*3JI&bvQp z=b#|#_FNjW4>F(=T?snlGFZ3Z6zMEkPczJeAwk~@hn0R%+X#KW**?w#|G@%vxr+Xm zg5Z1gBybKZ1oQPJc>D1>TKcJ;_qOg2adi0#8T@at?wi~ zOYHE5=@BA+r3yc`iQ-sqJuwUT#yhj+yoHKa3_YH83IsKg|NdYsJz6M;i$0rRo9R*V z(?ON}_3!|g9+rl0Q}1%kr33K$UKA>fZ-;GZ-gHJRH%mo-rI_i9GtUozqw`U?Ht9R$ zTDI`Jl6Ck>YlLAi714aLJUyCS4o`kZLG>FmeCh6lb$h4dspL{9yK4xBZHr-3Y!WXo z?C@QZv=)(QyMXE4@z69BYD=XU^lj zfCuyb1(fy@@v%Vg_`Vah;sB(1`|;h;^|WObiz5Sj z=`j&k8Z*TZcQ;G~?XYfaUb`3H$LmowPCNCQf-t2=8~omLTK$VUlGE|qi%1dIGqFsdmei)Md>TU-OkijJ)~qsV~qoZ3t#waBwd zL$N4!ZJaD}IRQ&H*nr9kU-Ic>A)fl8feld)(SFr?cwHdOypx;2#%oT5Kk8XjRznt^ z?g$|>)5AEUHpAf;7yH9>rcZ6N7%vh0imdkrzB! z)_s8_nf$koIE2bD={jLxF0Ft?-r;m2`9jK~O_?`MoR*G4jDli6@t!Y9=61e8nMqd6 zIa@_WQEmy&I-vpvwsNTTcs=};DyL!P59yY>TFmmP-}$Xi!r=4xMD};qPtqA-h5gIL zVbA+SjJCUgxkFd5_0dz@>LSN4(iSBf6=Kn;t%ZN>=>(>HbvjjBtWG~!^npp?XEw|C4(t!CFR|V>$>imoU7<62ti?VWa!S>Q*woz6K69R2v zO~F0$N5)n7V38Pe(aV74f6N2rm+H96W9&9?#`QvZp`Es7EXsvI- zn9nZ7`^R+g`v-BNw`VC7Bm_gin%B7SlNGa|wwr%UG@UTT7x4;D59De(ApEm6)=uok zZ=aIN-K+WV&Hfdcq~8suf%nk2!Gt$SNrG=SW=u-m+Th5=RM@mh6uVx@GB-_I;MZ+A z{I9ryR{i8ee#<8~VZTB7{@KgnyA|(zp z;l)mQY>41fCV zu(eJPUEZs)TTLZUISQKGT?1n4VpzH;nEVq2xbJEVdZrreGUX7Cld}WUQ6%+s zryw&~fW3Z8nGLDpas>h+OpK#1{rTxKnyx&>-EUv%*L~+<)ayF-cb%p4iu<5zp*99I zohGc{X}ZPO8hCF9$)KS)c_;9YRv5+*`8UHHw>g_##^sE{OsY{^#+K)v=>nxLHISL{ zktRHg;_WzNiNL>wY(@cO&!iZ;U6<~x*~KrCdc&g&Q$Wr5EvibjgU!Mbw7z~G{M<7^ zXtg$VTDc7}i;>fqTCqRyDlg&Jb)G;?8t}cHc+2JqGQpqv_zuGtsbpw9y&PGMwHfP} zemQx3vwkyL582b1na=omt3Jr?c7pP?!5~|F8O~IMU}s1<|G!&VXrafoQ?v^(L#qrR zzV-}ipV*1}Wc~5h+6G7y5&)u8fHiRf>@savIAWfU4FWav8XJs0mu1-UO?E&x&87yQ z67cYpf5b9R8m`0*p+eObj`dT-*&1HRyJZQ3HSK8G>OyNBPXKa6gr!(XyUN|)}P z&p7EEf+5)_)HZt$)OpG>q3OvWx`#{~@t@%GI z%jn&kWstXEF*0+NnGDHr%reTsu|JPs)vPkuqMODu+`N=leoW`vGi#yXyfCx(>k~@V z#KAN5FxDO8wCQ01%r#VoA9outuw@fWetDZz-na_Si`r2!>@WCzl4Gs77An^p6CtO< ziDPNjpzn>%pw#z|{5NqjD89W0Qj4U-xq$liJ#GgTO(mrMA5EVgH^h;;oeAi6V-;p+ d_>hclKHPfZiSj*mTwm7$c=3ws!oOXI{{wohO}qdA literal 0 HcmV?d00001 diff --git a/network_3.ckpt b/network_3.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e2841bd5e6764acf0f72ec2bae75e29051f38eb7 GIT binary patch literal 26380 zcmY(KX*87&)b=T4NF;iCZR-gnoFWmC@GqZ88Xj9C?#nw{|4vYo6@96 zsWca*d5~xxAMbkCv)=W7JYUW__qq4l`?{|E+sc3BlzVyytO*%mxxr_JuV1jndpSh~ zMOj68MR|=L&i(;Qf=62G-IJ@Cs$2DN{Z%A=DY!cPgq!-56uTGUgBUagjPpBF?!Mw_E zy|TlycWw@2ehUzs@_-AXtzh)IAyu6UebHrzgM-fJ$HLt62hp@ayDGr6h;$C$WTnf? zIAqIp?ET4`56&=fC|Y`mBQ@GEK7I%rq)4g^ZshR!-B2K+6i>5b4?EY<3)c}cVmGb0dtW(!;81Yk~%1hlIPrNp->i5;5>eKaMxOH7B zlw7tJ(pK0wyjw7~%JOd#f85-NN&ij5o#SNi>`G>7%insyEo?`b+*J$es=iftWK z2Y$!UBbrRFV_12)uEU1mT5KQv7=z+VAhh)zo}Zdb!J#Yhi{l@jzTi9`*gCi>_o-zAj z^EYo}-sDUydRhzD+e7jGK1GM@ecAk@M=L0oPWr#~XZ^qRmpMsMy{iKMzxoTGq<(NF z|LExtF8hkPpWa@s@r=h<&*50NWD71e8cg2H>*!5F7%Jb(<-fH|q$BQNvGdLmP<$zDyQgKu7QhL4U1C#~1p~A}r__wzb5Bt#PA$JdO zUWo#v3{fTdO)J^+oenntR>p=}L)^Y42w%M_LTg_m+;dQwPycx(zy}q6@LCbi=30=P z|9R=Vq7Jb6m5i=mjL~p-G@pIb9h2XM;ezIH-qMoJ8UJ#zXL}rFBue0RNGZCTXuy%w zL@4^a4^5V>o%aRqe<%PCk5iy^Cm)jCx`U*qZUvWTSW`K!#-6jL^07-%F#7vg zv^^A!Uq7@6EqTGX!m$X??C8&O%|2MyDuEICb5N9Z2TiNp=seX4maopmfx!=`NP3uJ z9n-LQw-QI#CGsJ;iEJPj$BQc>aqTll4DM%6Zm(-;mz^@K%FV>VCPnNKz7Ko%$zt6| zW46BE3bkiq$n}5+&2|`%8zhJ^VKywQxr+kKOR?^%8Ea}~bG)uL>d5a#`zZ&&Cwo3V z|7(xddI>lnErFMciCFSIk=Ol0UOOxWrxqlLyW}21MS&)J>fZvT`5j`jo*i61ZY#>Z zN4)T;S!~lZ;#by=ocVPzO^8}W>ZsZX6(#K)_w`e|Zewkb?*W$3iDmoq; zf|DgWd@Ff1SPwb_i{Cql^G9sw71P(SLH|`?5>kLscU3U{QZnbQamUJif5f#9wJ7tE zDPrA8;m0aFJbpxv&3`5FU`sh(-ghb*URnwAoi==^x)JVw>y2wh=i%~vGdLNv1@~>7 z%9%M=!TrE$sPXqe6Xyu7tgeD8joXlr^93e9aKRCc2VjBz1!1k~4x#_+SZ<&}sPa0D zXIDI?*wzL7-w6rN8&V}YsYu{zSpr8|+wed$HQZWg%u>&8{LODa2EHwzNz!3_$M_$Z z2dUuMZD0*haL4lgVh^3Tz(-Il14>vr}16;_YYn8adB5qGrlM; z8uyuY{f)x1&05&ad>;Dd>!9zK^<2=HjC;aX@TZMKAn#~2pR!0{n{AogH#Csj{ld`l zlpM>+YVnp&!y)zJR+{Cp9&c=!&HY~|Lzue`t~u@3^^A_V=Hy8@F(w9OUvA>!@!6Pm zH?phN!+C+@RY*47fx70(v?Zu7m*kg=b9n_DPqgEYQ4U!CM22-Qyby}o&9Q7_7^a-t zA!^0Q@x_a7m=}@2X6d&`y>~Jni}AvM$;dap4rI*>>tLF60`snV8alWft_x#u`}i2l zp67@z6aItq1D8{LRuHPhb&^^SSx#7}#vW?AWdHl7c>B5$t^4eWi~QGd!k-Wfud_z0 z(h6{Rn#F!uOR&Q?8qNE~vQj@MxKzIzYYVhkUsA;H+U}8e$4b03Z55XP9D|pqxnT5? ztthLR3oy>1(qMle{>-VMp!h6|ikpXz2GxrHKKikV;cLDz0a z6AnL2BP@4e)s_@KJ29DOsjR^9E0G5-E`?U7a^Bh+!!NFC;IwnYkgBUIHMD=j((wUe z^lWRcI`2#e=f9&JyQA=pkwDG$A7E)^6XiFAv0sfR`i1N!tr9z4)vs%YsVRt;_w}ba zt^N5!niii~m@Ha-+JM*Yb=BFfr7XB@#dLdj?2)q?rS}ZDx1S8}Pz>c>8KW?A<6KsA zPU3m1$Dv!pCsNFvh+6Zdg4Ky49MNwGZ#6y+EpZ<7&OwtBYBX4R_6W>)70DTc!-Zhy zJ=k``j_mq-BTlJ=DI?Cp?*lhT^Kl4D-^jw?Z>enkv5!N-mVkXh557{pRQR^BQ<%3s9xwDrlpL34 zQhshBsvaDPK@pF@>e(=wRj$qbeN)hUUIH!gwxVqPeVFQz%132})4QnwxMPBsm;l*) z#M6rkn)>mUdqdFgYzDXoI+ES1lVVfLadH|s8}r;2;)SuvJVR*-ZnVqg!Mmohby#;u zxzd}bKlekceU>aF0^c%lqSn*r$?W*fO)8V|HEQFQ52J9?M zrL~IcLa1*z7kDzvsy4!e$}GO~Ar6cb189X%%ym|!Fs5`b7u|_wmA9#=BsT~er_Mm> z;zS%aQw7hKI%8kW(_*I1F-p!$N2|t-xW#h~p1Ps}J5)pPl>KpWtdSq*KV8TZO1I;n zIVK3B^*GTiolA{laCv|h$&}5;+%O+r_~IIr9`okgpU(&;7AqL7l2|3NjoOyp6L)3h zu(eG)J#sDLwvk1oR6m(tYZmhJ{T`KC+1mVmg*8_UH0L;(bZXh9LD}umc&2j-yml`| z*EQMb?pO~ALwoXJ-8k^+NZ}24=Hj)ByYbw<4!BjP&cRNm7`({@3m>|2xY-)?&Pb&n z#!-CZ)B?PhCx==Zmb_ljXR%-b&pM~ddAnWc_P=tDwA;h7=^-GkZG-`~X=rC2$|`yz zFlwkGA7ARr?q2%&1=()63uzficE${T_s`Yy4YqXCVicKlVX*`e7+65yc=kYUH z7q+sCz}w}wCC5HbW2Jf3@J|v4(^hxG?WdCHpn56)lRZJVS`JF0V=f3G(SDp(?JE6t zU;=I%hTM?>cuL<9H%ytz7Sjt+W?p|jE}n)}oiSwoFa_tj6vMNb zzF66)%Sq13e5789KV`+xoxDk`+@21(qcg?gz<;84RURvxZWVf5kwX*r9vHQu1Ug>s z5`z}Cf%AU?o%G5;(X|8zDywsjd3Q|N--jJz+NnpoGe_+>1{<@oSZ<;voZqsAAMG~9 ze3}027;zhhxX(iMsq0|3IGT;KXYrW|dpKO#DV&ol6JDH^#d{uMqDN(SemZP9%#_=U zFV@Im;ZJ>5uC#_>zSHsIlI@)JC>;-#C3B0m86O{ALA}>KfU1JUZ1=hs_6oTv#E$UC zvO#8`Ycv42RWBBP_RIov)koCN{T+-8Ue8x7DnPS2gk86L^9B74oM-x6a5L4$kGIv> zwY(Dy40l8SmLC$+F(c@3`CzV2uai!W2cB#$;8X=qN-8CAbm&8KwPMNc{s#WgQ(b&8 zGJ`*G)5LXYUL4xl2IV7)!Q?|J z{+zG^6+D))X;J)wo~y=H||!b%uDh@ zAY{@A^q;29D@OdIhc-zVc7Ft)96t=VR3qA5U4ox3&cOJzja-)DLBZ$jvHfub>m=^Q zOKuzZwqzyjaX(1@2LpxB&3S^=&|>y+xcsmEo z%ZcS%C+)>oUPY9Bdm+%0k5DzSo+?fnVx0zZ&#YuVG&>7EFOpE)-+l1YHlH8b55OaB z)_6rvhO4{JK=%!sIpmr$uIcLEeKm@^d}sqr7#70?(^7FWtiWOJUevdi$$qRVFE8s9 zO+F{{GBp<-rjpNX3l>2_a2zl3j=>c_e18z``!pj-v{Q7hzwkFR9sfj7B ztjpzf%{gdnV#B8!@CHDXqvGUB_|g1 ztP4@(eL7CuI4g)(E9v66Z62ci!z{Erv4l4#kH?cQr=b6%m54j{qK3>#QDs5^uN%G& zxqUV!>6Y-xr(@9d(R7|u?@RNI2dhv<;95B(&;H0U+D3oQe>(k$2x{ofZj!WP^dOP8>M47K(l%#og z6r5g`0s*}aaQx>&{vo8mcS%&2htbDN%MRhi-g9|VlR0bTr}EqRn@Q)Y59{V#r<<*Z z(BeQEr$3J74G%Wp*wh!&cb?PmcJvOM9^p=|1~t(1eHC6O0j7R9M+I}Vd8tPzYW|48 z16QLkr}2*T#xy;=*L|U2@cJg`Bzn;J;@Lu-?H&wLY7~zIspGk*4!ZMC3maN?6-(SzboC|80rZ5gae#znGziv^<%3Pd0ItC)l z6w#%1A+2tSp<)vYUe)Eb?hNxG#S7JR{Mlk|?w3yk%G~ipq63@X`$`1q4nW#=isBv)C}0kF7K2dvE;sl|?RHd{&C9pLfkipR=&La5v_Ub;QQdEHt?3 z#qrli(}Q9GJ(i0?(Dkm_BA zotTf2Z`bmW@s*^K*PF8j7GYZ2dzx;i#^dy-!4<=gyWy7D0(N+jLqDE6!z!N?z8!D?Y61_T zj(Dc8CZ?M9=O)gk_!HZ)b%P4F#&*Mhw-@sLnc=wcYBGmvyRw#E9tuUzsrQ&HDBEs= z0s0YaI@O8;_U}N45-(0Uv>#J2hlan-C&}bUO#66%)KcRS@^xWh?|ihCb*6&lXT(eI z+_?Dq0rAbn!C2#+Nc#)CFe~#eJ!$$Qae7z@N5d9ztl@T?=q=&G$sgcf;{@y<@(tdv zFQDF+zQ9h)J-EDRCeDnOQb}t)m~KtPRsJE^601l1A9v%psDtp>HI+`c58|T}6Hx1J zf2p6e5dVAYi77w3vbjypnBV;npYG+4_xC!3$DJ`aWV{L|-%{r5q1*Y&#Z*2RX(5!n zoz5LW>-g(*6~TE+06Qh`=Z?B;c65FT`O`b-*8T^S>AxCnUS{Ib0in3)V={XEx&h~Q z4adj#bJ?Y0HC5M-=A9N(a8221F{Lw^;)HOV&I>r~q5^+2Y@?zZmq7Y^Fe@L<#ZOTY znBlF?7JIs3nCEcbvh@PV?TvUkseb_~yUn1A+;2j)uM)0|SqTecN8kg4 zG;H}1k5PT*;V*}F*!^f4uaYU}j6Tk|#Y>GfmlpEr%U@yDnNWP&SVFB+<8WS+8eUAj zO9{WHparjFv(Z_g@U8$m<(6Z}s_&Atojt&OeL4@x)x-BM3`EsuOSvIO37?1f@h+b< z^8Y;=$$SI9S-*{?-urmdv-hChG!ix(Y!?nr*o|vzRnco`9sKB(K$Q+rbTtR@cDM=6 zdS$`##;K^~p-$^RNNMp#4<20;%C9TrVS3*_T^c+J6Ib=&-^qJmY-Tw2?b^G|iR#ih z_3?PdC!Y)LcB1#YqpM_&w zW~^9y`yfAT*2Wpj{kVHrGc7nW9~Xa~#7E6{aD3M;<;0go=>JU(f4+(39Yt%gV@L$g z?4dxVMZ`u)xS=Z ztCL7c&I55nb=S@;MNU$ya9@NDktTO#AH(Lg!pWTnxktP+OweiBgcgJwsyD4lkb+%Z0BaXv5bU`sA0VZxJ;>EM_!JxWjjWIamTPvM8 zbAZd2#_+eP652Is0{FjKibqFDsqy?~_D*V~X-_wB#p|EcDkFoc*^xN)#A<#olZs|2?~2Dry$ftV zou-*8dCVLA;b_oR=$}w7Q3*)HMXxf@bf6cOT%17P3i{CaUSaq#WC82CCG(@F&%_5# z0{!=}2#2Y5&B8=O8qh4y8W$sRqy07#lMYZnjW7yHokKpfjLzjGadKQ&b~SegH`u=d zV}oH7n&m;=H-?}IXY)`syUOK{3;9{S7j_pG@JGx3ym;9jR?_ax3K!o=wr9t%=8;7h zailv_MJ6WAD8tj=+Ud!}z0&=iW6&UZA#OUGOK)NZ@y(ctxa#~v>YRBHzw0dF)jwnK z9{wTOl_YNM?u(GE!xFdccwXE~t46HA@5d*LA17;Yw46R({ZPn#X4vw&qF&tWWQP$t z(>T7V%Wqdsz|SXlQvB90ko43?tR7zhVI9RBXtM~T=l!K?*5!E5*$fumGNik`LRjy| zZjyx|XzUrs2G6G9uiLl8fCp*ZJ4X|@y-whr1(6uKEQ`VbVQ=z1zw!ey*Z3f!`^7dm10JoXIM2rd@r144OJ-LB3Z3*}e3m|ABAnetka=k{nI)uSv9lTtIhzdztBOgoY(0+}AB1zR zZ%3o@7!s3PpyB%n9v%D-Oww9~Y|C8~Vu5&lQ7peMn#N1&o1xw3i+EMekjrD0Sivj< zlVr|9G$1(Am$3_|voQD2$9oXS&jAZ`muax~@pY&(>dHP!H ziC?5qc(UUoNzc#bH#ZJoP?;Ym9QTFZvp3@KkL9?hy~`W^NaJ1SlTkssnEYoYpi=c1 zY}lKI_UTrj=wOSJ6brbtV7QoY%^%ByHVCuS26M_jJ9a-Yp8d}aK-tV>X|S9;e+}D* zDXLN2Hb;_#nm=;@_5wD@Wx7&;j)86r1ek4kmJTcm zgyR=`@$!i#_YWIlh$FGjNr33_$y^6va8WWT0CJoxq+Jl@|G&wL#Z^(VV_aO!v9(5rmu z$zuX!eskBKPeFV{oc%V`AOK@Lm7?M%;tuS&DfbWmV-k)*f?bwWd(F;T}KmoSy6%;?v&ua zC4>3Uf^hU3nhkrs?0E03U3~WbYdEyxBu(Fw$PJCZVPextZjbNA7b|x1=UW@7LAM4j zOSY4>(|&B7Z^-$(a#8+CFaFqjxb&=pE2np)U|WK}>8HA%4CDUT0*$i}35 zM`&3?501My7SC_(g^w=u!B?x(xMBAwT(Lb8i;dQ!=0HV0w5bfIdJhzLBzdCxk_`N{ z){$onehRNvc-ZG?{DD>b61dfL6d$+^TrjJZ+QWy!4oh8Jex?($@0U?pK@sWD^@|vYLL3Ze5PW1Irz7gtG(sJo!W2m&~E`JxEENBWcr) zY&fnj!N8WmctWo`F7Nst&zdAo+wKkWk&&1w<1cLe-A-=0b7|u47395h0BcX4uFT)g*c2bybg%TG>zS&-ePGjRq?Pn4XUAmneY6j!P zng^t@HJ{TbrFQL~%0#2z>!|E$Bt)$HC!VZd!#!H^!CA?Jdle~QUAZo9^*%~l&32)p z7|&aCrJ#Hvj5Vysv+_X?ocL-I&pLJqu3JR$$=~VN?iLOMrLSVIA@=yUA{-=LtjB{I z9UNP=ixWyqvDIZg?wmLjuiPBWW7KER>C7LJpwv;QI%YA~-^}8n%DYj$#SIn9gSqYM z6B=JX4Ff)C;YXwKaL;W$PN=(ZX5ii;c;lzeu5U{4KjR|$>6?lbexrDNaVRf0wq~8R zx+uD@RN7MtIa!j;N~%SJiQryD0P6=txRvEZ^0b8A#H)o6Q>}p*aA1Jj-!F3CVA|gPRm_GNo!UxFMeDv$}d?4)|Q>Z;#Y~# ze4?K8K$WTu4hd)Do`CG%n_{=J0IBf)3hcSiS6tsak2;@vN-ouXrl4<1FyxPb0oC(_ zMA<;_u-QnryL%FMb<~^FH26x-uM*$yx*T2In@Y5P!|zM)#4B^QKtuPg|3iDA7#7q> z8sk=qLB};|#hLN2sAU;=u91(l~WdrEij~b|Xoe6oPe+seB?or<@wp{F32-a?A zXm&=Gz17nWVZg(iVC>%{gtT-8^_@G!^3+%g`>IXG3*Uo$O<&&S9zkbyGa+hSH@3dy z0=9RGY08G3!n84wFtu8qdL4^mjhi0g10PvBsG`DuzU_oC(@a5siag7h*uW`k7xKE2 zE=H}Y5p5=}r6bvSkYrmYbyGS_ix-%}_*=)QcUV6d{NlavH9uN#*S{$I*XyEm?EFj$ zIx&C;=oWzg#s2uFLYI2ESBTA_+O%WWX|XCe1=>a&f`~sFGWLc7;))V?|qF3o;pv^W*jz~HNzoKoAuF;^%JQ#6q5ZFNyoU}YiW%qW$mc9?E zz;`i?nI_Bg23``*AN(s0(pJNRjRr7koGusX+ryD#>X3e5iueR$#g)lXG^8#H+FfUn z_n^Vzzo)OoV@*Fv+i*POwl>2{(;R5HyPwKir^Cm_9?&x`0xl%I6`x@wY|X8d1czt} zqg|4~uQ-SJqY3@wQ;jv9Frx^Mf49{o9>QxKXUHUI};J z*ukfHo9N0#5ys*`=q3A8uuL_EZ0+t`Ac;csi=y z1KL!kh@YZL#cp$+5lY^W?}Y2(>B)D&^k3KJ>+%z_w)dyRnb~w*zg2ReuLgZGje-9* zb<#&Frwb!mX@93FJdV(#oO$=a%`J5F<_BX*Jwpzx{{9CFCrSEqn-5LUSxt&*!{KD& zDM|I)bArkG0!XR5OjEk+@S$nNWII8Q+Yicc*wHcMb)!`H`Rf2(*7-=MGW047EY8rD z!fJ8p#AZR-IF0OHjG$q+I>_^7FOD~PAtc;a|aTK{hS6@m7rHP+o#kHHai@LQZz_DxEz_4rLO__1;IfsW z3#ZMKWl_dgUwCoovC!?|8{z7dLcycj6Ye!6z};1Yy2uzAKIm^r{b~*A@2yj0Tz5}& z-#HL^x*ijCPTi(^Im3Cw+*hD-@x8cmO9Aw&d_u$D%kb0FN^tl=9BD7#4zKn)Qecm8 z8ncQ=E2!*&akeRfSJ8YBHXkA7VcW&^3f(wKV>Y!Lo5CSof`}Sz`k9qPf8B?|mHyYn zOcxodFkT9e<_6H@p*h0v(_g?RuO1{n#(;_CX}GZBGQ^yH2Qs$jXot=iu%0l5${lhj z^-wK5Y0&`NYk$Rm58`1&)Is65-2*ylA8E%kf3VrM5o+G2kj1wD;6`s{!R&oc zHvAhe{Wsea{H|6(S)b{G&yc0i{_P*Qy|xyuXDp*@DmQ4!f+o>0+maeoU4$2|*F^;O49TINxI}6qVhE=pm1U{yOq>aN}d~aMCnkx!M}CS5Uxu6IHTsi=tC+ ze!|nMb8(ipMi=ogfj5o5NghM5h#fN+rs{;i=f*zL3r%wcGnWLo^>i>io?;9`ZyRvu z=mA2Hy^A2Z)10i+|AV&Zy+T#34P72~iu9bGgKU>d>%Apcyc!%pk#?713!%M zr)4bIdb&cNjQ`-~sR}xp7AU;PkrkHQ?M`WDM^WGc4VdqGg^s-)1sboLL^o{{A^Rmt z!}T@jn{6(sc(+JWluE%z5XI!ldcqwCKg!s@R~+V{0-uYv(){iT!g<$ig58Oautl$p zD)L&yb;%3G(2$W7y+npXs)FI(%$b}Usm={6&4eD0v*7l$V7PNQ9a@h4M_&!>q}pCC zLXSgPw6K4jH0q%vX&IU0oJ|>BJF_t~CAOIQl?^1r3*Hi^YBvx!ULYCcC-8Ww0Zg*L z3kwInqXVbxNC;CB`}S%hMfYsV4G9O;V-q1q-jqqU8lTKHZ)!Ej=ql^jAQO5E=YYwH-zb9ZO+z{n4*(EF2d?;H=_9 z$_YIyZS58cS%cjLjp^5fqgINvuG9oxd56%zB^N=a*_D<#1;F6}ugEffI6Udlz`Nac z)1UZJ0z3{UJ>wFXczY1pd`pIHqcdRe<6eBS&pP36zpf-;)Lz=PEm_zrCxZ)KkA}b( z2}04cHDIl>L;SUDG8kcwbY@CDE#0#a?a=-l*yxVH|BioXJaHo!Xn4End~v>5U-i_AJ5DeUhe zq1)r5KxSj%W90?7HZO*DeD1o0I4^@2%YSuIlug2#>Us#2MAD2SrqDxCD#}|W!81bz za#W8HJ1V5G%)EteygEncB1DK-{#oc)kS3Z*a$x7+bTTUmqQdQ^P}DBEPqPZtF=)pC;e^W! za*I}_4(rQsBQKi9hHBvqCmqnZ5hLFI6(qhsF_-cub%@{kt`=)6HE_)N>7rT77g6az z8=NTB5;A8mq|T-z!rGM0Fr7NWgN_&&hfh&T+=oc_~nlV<&Zi2ZFx535D z%J3pojy`1)2D^uuo&5Q?~{vsW^W*aqB1VZbJ zvy^W%4PJgyVy{Ifg|T;c(kYwIV*B5(qSe(DpnIv1(pC&ZW)^~yc|8oRZx+u~PY^S9 zDYD0lC9qvGkS63p$6t2tlJ?Q^m6VVsvW`9Vsy& z$&fDR*>k(lS3z5xSh-5FY)2|-4L0KEYyZ-v>#r%cScQj8auStZmOw9?%ak_JA54i4xwlow_3Ss-IqZIn(1$vX=&L9KbC-Ziqjc4Taw|;iS`)N|&}f6BvFLYA0&a%&c2N(CsGi!v5oQB!4p9 z5Be^Q%$rBsUp|80z4bA;BulVPej=dyi-4Ca9msc0gn;J5E310B3zvRn3cDT)WLB{aLj2!L zJrCszB^Gh;Q{9Q8pBy9Y^WEw1V^vs3+hJ7JT49Z&A!UR=q}u-5#NBg7f|cujxRCA( zQ`Yz3xqZF~ftABWqsSY=D&HExNvjv;WEpnNSS>NUizYu~+$mnm^%6UuzNK!B8)?hD zaIwX0w>U3vD15Nhgx3icG|8Y|DqjM0^-VmzEngtanH@oeefL$Wd;JsdHGpJ6&P7tm zv7(W48evV`2ugnOih@^Opyt|r63?@mu=D01nq!#(w{1?*bHjSEeX$;&&09_3!HRS= z<~ud4PY~uFamV%YUuj!RF4Q(Xq-Q-WA;91!MHtM3Or34w*>7%qcFuN=J={q0!z|$b z;U?kuzHS(MvPqP^a!d%htw<&BKf&*XY2u*H^A!AXCEUn21o^G%)O@~3n4C1OOP?mg zr*6f<^p&2HnYl+{!Sz#g>Ul62oobal%i9F?k^(3{mjkBuv7%G2UKCfEEe)9SMIzok zMg3Yw^3Lb)gbn>`#5-Qkg(mNJFr)CYB(bY!K3VBO8ofabKjqbRMPLm)d?S5)KNW8F z>3ToC7SX0=HAtCvkxsn7KsIMDR;pVd95-r&*H^k>W19Bp;IQqy)#uCvL5yGLI@$M_kd|Lb1dJ>@j~ z*KtcY)mBdyO@{bhB?j&lq*7-`jyUXfGyL3TjClqLwCJ%VI5mu>Vavusa9Ozh^lm># zAM+Y7v79=N&dW@K1FE52n|4o$YI?GCexC(o<-QQ4J6DQjKEp= zBZmF6rHeR(ju!@i*n1+clc}YT0WV?P_ax!TJ|h_Y`5S#UxG0UennY7>4aMQA-=$?W zTjuia4Na93D7-I1NA$U@tKa$&mFa`BC2IV|rtl*TXlLi*a%g$)nP@cAkOx>9r< zPQPh^M>dwUFV#exV>%zEZY+l+c7x2D)#AnMjT6~vj-c* zZoyxP@}tCm%6}_24`h*NsB)-a!V`U5;XkW%h*0WJ4flbtn**G@<3me=2hiij*HYI2 zHE4hDEELRmAzZoAPB$04>LQ1((QQ35!Qe{^Z8UmCr6U~SO~O71bbSE_92AJ%dq@pc zQ|P~+N8!CM2%o;5C+`=Rsm^gES5IFK@aGUnFQ{}~cvzFpO>fw`s!sgW<1DqOKZ6jD zebUY@g7Q=IYP#flo%$MFgKe1?g)urku{Lm@5M6SHOdel>z0S)d!@sHtXWqzlU7vN4 zSit;i9>R*$5$eTt`~}*Ou-j>4=G8pW#iJC&V|shbyJ;Apd9&!TaAS!HAm0 zC*yxmpF4Y+z|qL84J2{SGqElaM;vE{=ax1N;BrJCcmdAVZq;BV(NS=o?Lhn zCf+O+?Ua{`t|P9BA39>iFmF|OY9OIn-KRp>g#{cRc^CSB)Pce=!^9U&6UB4uG)O*Y zG`x21Bf1^&6%y?tshdR~yytfhj@_*g#yELXs;&myxjBR9E>S?8rCRXmb5HU)8AJD* z_EV^06X?Y#@YOeeLEBn|?G+ZFgM7C)(>3*6C- zMopvIVa~rZg!OB>lG^U{(ZWp#u3Sf17rkhR(@v-ieh+({xCve)|ksvwM+g{aes| zEidsJpJ`urXBB174-qBTcS}~R{;!L8|3eSW_R;BH?POnmQQ{t2M3VAqs_COm`3@f> zVG*kQ`s+9&xXu^MAc8aAB-$BzgWuYzh zEFAb;2_>&bNZY>Oh7y@h;q#Dc`aO9fWGc(rkDGB*ywud25}jAV`uuftvfmBRJ$V$W z+v4HTfsxd7P!Zlf8friH?OW0FOA#$P_noxA=TSmv7jfQl1mY}%!Tg+xsMx~<Eq#Sc0Lt$e|ZZLTMmgWd=omJ_9T~MQG!L@aR`BU;du02@Hfl_(Z`-b564kA^;=Z1 zW)k$0^u~Fgr%{vL9-66k03;3rK&|g=_&%W&l6H2@cAHWdp!G|9(RINy&T18u$qg4j z%?^R2W*z$7<^wAemPx#S5G@n;icMzgM5FsFVNh)?G^{=)UH2dof{dJC`jt0={=%W; zd47lRKSiARLrz~8#Z?-W29*ZVfJ90v>N)$8kf@LlLMl{b2$fPaD5Xh*p;VeBqEgax z_AL#PLM2q98TnF%1}XABzrFv#{o&km_Fn6=){;;1OqQ7tTDAIA1fx+x@TvD?JZ=Mii%5oD)v8EdnT0x0eq&Nw4 zSA*M2>q*@0Z-ib*peBhTjFQcBNXb8s5du|&?Nvs*Y8m#1Rmn`b|c^L_6;so++G)!t;8G^gt$h);E)ra$TI*#mchMA33$DA18h%n-MSE^I>E7f> z1w^1wP!c)3g~N4SBgQzS-RHn838r$H2_&-_I6JS2TU%65lgz@Av>ijk!hBAA=w{q= zP?vG|>jY2Nx8a?D7>tE2AQa<8TRDTA3;`8T@H+;L7p|e$(N_Gp$e0-qNXL&sTu}Y{ zkSfoUU^Xr>#u7Cvv>FSiKbtHIkH_~y zqVOYMn6=0YfcyJXQKRN3y)EWX?`htk!Sa`h!oyAMnj{gXa!`Uv8kc9@eF(q?7dc#! zn?~Koa~kK z#bMiTxMhVq!YgZV{V#?`cmKl^UWw?Kc$?E)kcUGBmgIxYC*W=?;*x`Y7~YtV`bnBt z@SuY}Xn#PGT86Rh=1iEM847id>+z`nIf}1(V0-r*uqgciF`BpOD<%U9bv5zdRx$S5 zCL>PD%6(Wx)!+e~!TC8^8t7i$iiVTpobJ7H?CbD{^uI@ibe_pGboOz@{VoG^ z<{@d;)GnShYtCSH2WhjvT4uoUyNzhOvy<$0dw`i5dALD%1}ot#!b)9?M_wQl6@J{n z#|_5#lm9bjeBfh8zFtIG>nXf*7YDCvhhV*v2=z9^CIv0caVmH_tp-0^92e55sFge&C#JNlZaqZ*z+_hGfnE7WlDNCD) zew!Vjf7A||wWi~^|0J3jSuxtj-hs(SIriVqU_6`~hh~qZ!L~aM-~4ePenYK9@^3!o zY5c<rY1`Pk7b3HZVo)l=C{R_rgk2}>r(s- zx7>)vUXLJNSLQSPkJ3SQqdV4R2t)bMGCHeo4eKLPPC5=1Vy~bz1|CiU(^I=>&g>NE zv(aXi-J{^flq+zG^9?7&Es2NE7%5XRfwm9JNlBeJ8(LY3(!PD%t_v5*e8+0~-yWXV z;wB9VSLV=zLIo(*F^}ijxPwxtHGE2oC6dBc^u>;LB50&TruPYOS>GZ&uT+LlL#yx# zqlY@?e=tz)4u+Q&!MB;W@x5LnCfRx8fkO

)3)ve(U0l@)s~VRg!(PLyY;2wz0%v5C_{#5zC0=@rk3&)INnzQtigM+)#=yafiSf58)5uHe%~ zAvT7mONShJj(3dI_K1}S^ zXI$hIh5Ul5Y@O|wAv zQVH5Sucw&Qd8lvNh0iZN!+@PD zfKwa_XX?XnB32M0vYUy)dn=~sW;H$$=|}lP>u9G%1}3HSLp9R~wWnnvE4mRLW#5E~ zsLR+FJWMi7O2I19hpZ6#L+^}kBnP?>n_WUk(#dF=pw^B1_bnwhbrPuD=YWbPnrL6l z&;Bd5VI2mtv9|OW5!cefxr{8+&hHP^d|jj<+z{)>xyV;-$OzX8Gc%S9(2xP%yP6S; zWBvk+t&lCYNh!hAOWClo>kTGH39)iZCGf=dE9Bt9OTalB3PaNKantT85S_zodvI?j zqmKn)dT}8{pMC%fTWvTBJdgd043A`Y4q%tOdc(c_;ymmRx&noj7s-jsS;z_cK!so_ z$xV!a;SEJlAxpS}=X_9^$7h?J(Yd8>JogI9nHP^sfQeRUWeYY7Mug^S-5k7 z2m}`K_68%sruYL4UEDzel+#fvc{a=aN*Jx?8#pMY%#>Zq;mn(}4|h);CE=H)QE|f^ zbod#=skQb%muDYAuRNFho909pe7y(vJ(Zv;NdTRW8^dL#8uF)H6io%S!PogJ-F-WS zywR$I!AvdoSJx5L{%pELEwCRS*(Jh2cRS{*o&tr-`5ebV6Ff1pn)VMp#+Gf~czdz~ zJ-v?OdLs(sdVw(TbevlDbK%6rD9o{o13lX%+#)9~8jh@nzQ?9y(a2Zmw6tQE%t*#_ zQqnl$u8or`C+Lu&Hq`wIL9=Pj&?2nJW}BZvx5QcO9wm9WwM`fHp8X8gCgNao-5f{R zailhqShxI$aooaG+;VR^{#xkD;Ax1aS{hn6pJ&^MOoJ{qHSv8g1#&JYtn&qrcm3p-Xd(&vW-v2T|&gw0B1E;bckC21CS9xfKn!0z}3>>=w# z5G;L-9u8Z|&QN?x4jj{Al!|+3g5pf>7wu2zbFYXDY!!xmK}JTZ^;4PDHNm(uEf;ZGp$6g<9mswi3U_H87CRF zqS!AWja`@5Vx!^R4ccuc{XsJk%#v>6(> z^4NBtURv5Vm(F>=8IJl*XZ@x=;V~SyQ1Y+@E1dQf--tIF2VNP&kHu9`n0*l1T27Kh|=dO$zDbU}oe?N7x10%_P}@`N6ma3|~cn(_t#X;#p!h!`v=MbG{%sQWz+ zb$^RN)LMR~;&dhM@#e84acl4x&kvEXzXZFP0)QwJ%ov~;qLPJ@s_xX%*bt`|hSH~v zg3Ke)dl;@5NI%5hN10B2=(!a_IqA(Laaj{p?K5JSyc{~EMU!<<3dOeIMa;k1H}Me6 zU@x@%q1j89pl{J8h|N-ACH4lRD=#VEpq@zBosC|$UgJ8*E{8?+ zqV&?VCL~I4@Me+&8#!7ALm5JZ{E%lCKHmaiYgpQnv4&Pwd%?t5JNy)QNup9aiALQl z_EtC_4t|;f4f5Y%G1G?+ozn2uSSu~Gc>;+{6GlC5qq-}@z*$I^aqqLG@42ZcGV>mJ zQuvr-vv?NwKaWA0zWy8x@&}U?l~Q~(u?2q0W}wlczr^$FDJ&Jwp!>3}fwr;;5u2j{ zo4gaSM{6}{dmxJ+e~-iVGCva1z7d0;#nHm)Kj~7n4#<42N+Z3`qmV%ZY8_1_!;cGz z-&SD1+o?gM$Zaq;sDE=_4l)Gk-HE(@)5SR zi?hoTt#Mh`8(LAY3q|iGKvqx8@=fan*~6+)Y*AVsQC{?#1a!;d2ODwrX01A;CMY7C zm`K%5dypj?|3g1NWo&!fh@r=i)2SPi$f&;>9KYj-;3meLapFVO>}zygMF^RumI&nC zVi>V_fmJi4sCR%f^Ivuu)i|>lf7sSz=9EHFLQS<(Pr+SP{oKqMfTgf>1f!Z$2@=fnmlN5 z$HbE^+~6Ip$Zpi5jxrBP)POu{tDV4%yi2sV=nmen`AJ55YGA%$1kpJg4wW11So>;O zcECHC#OBUnzlGLNXE{f(550#Mx72c1+Q@Uy&s%`&({^LYXD?{@xgT$F`M`SvLWO-B zEev@A$|gOiYh(tx8Lja6NeJwWDaTRGjc~nO6|V}r5O2@x_^G)HLuLrE>gUGE)Nk8B z@6iHM5tK*%wqC-u&8IQ==4%rDd=_t(6=aRS@}b1HXew48gF1a`tc14?%YS$m6W{-k zT91jtr8FB{-us&)cd;2`?#4s?iyRoP<=y+o#c@$)8})>p9RIaHKq=mcSzxdmx18C6 zCaYUXWXM`h>$g0JnW)9;ik;}{KMj-5N;4Pa2Ow^27mE12z>>@iJa&?j*;!{`hVxFy zF)io$Tz=a}?PeV@s z6p-1d!l-yQf~$ckQ=GRB?j5=fiM!W8z3y`K=zM|F{pzgJTYuP;JBcG>_9*xL7sr^F zux2+a)0f}Afd(%*Y@wBooQ_Ed;fCOT0~~nICvI$ z1&yPW;haw(C)vjfTa$W0aHk04ou zSJbvX2YoK)=v&x<7Im#Ko;ATSvm4=#$({%KMWyh?AOs((_9V<=OA#9SN+1f->>)9(~%i@ zATgv)W=Dy@NBtbk`6H9dA|6tgfR9`>p6Vy%%Npy3Y$28E(UB&2gB+PI&83% zwdXfrd$)&RlgtFZu!zL+6j4?tKak2iI>#jMw#MJz=CE5gtYd=?6mjPnf5MDN3k)wXCP&_s9G@G_oaKIkxV9UFol;Oh%h81C+0+=T<% zPJk2ZW>`VGWY_T1DIhaYnI`vFjwT| znB_{+q;@yJ+8Q~c@Ms>6)GP+)=z836Zw4Ihn@YQ#Ae>_;$Fhz>kxrE~q-lJAcVKyq2H2ILW23Vfi6?ej)*Km&oAAvIZ(I z{2z%Knt_7d2q(A}WR=(pQu9NYjWThflYi!e=IUQyBUwiu?^S~_vYnSk*-HKk*M>_S zKWGv&mre|>peiSLzEa#c$2-%N=w?0w?=#L!Xz^lDjH^aIvEL|l=L9#WbqN1PE@fYs z6q75P;^?nzUpO#L4{duKaLU;yw7;PSm2;JNuk|f5r%o4NDs`3aH*Li89>KWxy)wG( z5QWk8o7rc5`ylQ3G_0t$A_tu>V7}Hd&ZeO-yk9LydT2P7%Q&(s)(vE^Ac?$PXh1Y2 z4RO;Uz)xWp@LWP7)$%Gp`$Pj8&>KmVc+HA?w2xx2Ry)KjpG2#btDz~X3>SSGA$ASZ z=w|CZcwGQd?Nkqq`kBnzLz^Lgiv+VUBJ>ATZPO~@kjSYgg~G-*M_jVoBTvV$JoXpiyvZ$UA_ z0z-EdFhdj~&4YWG~Wu>TUi)ZYkx8ZM0Eq&VuOT_F~S z@?hs&K{zNX&vx);-(wfY zX_A0HFsjB-^-CDM(-m<}j0mnB7v!}h{UTQlb%BRSf{WU0d_UWgRCa~Y*$YmOnPDa)nE8; ztd-PlPsQJ-0;xk!I=r9p5LO?GfanX9%Px~;1Iw=xgW6e)OG-5QaPM>YOm9M-eGHhE zUO@TzGvVp!d!Xg20fy;n7#9}7h+NzNhm{4H{qqV*x!?>4EnR`#bC2TbhyyUEZ8ox# zp|J3p9AYQL9=|yS*^enNp%oUgZb>F?RUtp%7G>22`INJ8ji<)0L3*) z_}}+V?qs?nd8pn1qHf}-ed{^SsC!M+4qt~^<1%>YJ}-?rOM$6o5dcO>?fIg3)!&a3yCxTDN^H7KyEeN zgKZxN;Esbd^W(@Y+~Yn$N(w@tWziiHlo>#d@y_Gfo=5ac$~6qV)<~z@TamZf^Xb?z zGjdQuf%qPez~w;`5FVNfvU{q@)Y7$BQm~9@CsI`VEQ@Ye?~zsY`#6bfb2z)~d2NE# z;ZQC-hZ@%1r5tf-j0&BLo~M(@@Gd3x*RVf4=}`tR@p3R48iESXOFUl25!%iiqveNx zL&V`PRQQ}X9`P84Q$LSD^Pd4&C%=fA1gde%i2$_vBML5-qLmj*!F<&Pm|w9R-OP94 zmaPj=@mvYa+^Wv=G*`h2?<<_rh9F$IOcd-U#8}s=9&qUT4;A_k0iTK>-Btb+PW%p~ z(~r5aS{5Sks$T(Z^EMEJiJL%k%S(_*GJtcdY#?xL963GNjdSY#@sy7hDEX#S z2~H3>)piRume*jHkRmwg%z?ybpUL<4AHY6Z0pd^Jrp2d4n7sS%!EeKEET8uc-JQi) zNOn-`CYYzXhZ*+PLB(5x@M-gECM;kot8;b=a1X78KSfDck)nitlcDJGS`&=a z?LZ~^G7j05!+&GO5SOk^f1Q`d)Sr^fj&mU>@_c}<$Nt zPoer(OIQ(o7S|`tpxqh8=*1sT4Q@tLJz*nkEK`DUp8=eudl*yF`KaaOW|G%k0{333 z^8S-r%>JlOB&>AVJ7WWKXP6wqg#x*$U0~+wo-4caSU%!u`D`fgv-Q$W&P- zLc5eYs88pm%#*RS>;hbzFA02YiKMdkA_OevhpV?F*jvX!Xx5LTaQW0d$ov&XM*qEq z-Yj)y(&7kKUsGeAZ7Cs5r>CHD`c6XqL`-@3{8R%{T$T}9MfxL zJUxn&t@jxsW_^W<(>Zv=`Yv}luZ?a!@YtL#1;+gVA3HbtDYgH%l=^$k165UN7;6y1 z-Zd>maccpt{x=W2d^)fp;3r8ak7uHMwqWgnV;qezE^a;~fNy!!ziGP=b2}mk`5q-; zz`qd?bjqf2UtZ#=%bHAmdnTT0T!t;n_QHR60%1*jBi-@#F*@FRi^VTZXqc1($nQ`F z*7YYINJzjpMy4QDHAwOnin8wZ8_3wTK_YWSib-A<3Gz>-;GDZwRQ#|YJW4TURM)6N z@R`4`CfjLVSWK0w+6qy=L0blA$Eyg0_aX5U}dZdpStegnP*yz)!lly9nxS-pD)Hy z9~tysy9(lxl-OE+acnxc8>)_narUZEJgLpkv~Sd4Pd>W|Rj7dT79K^TCqbaT#UJZS zC(o|NWrTcv-X79aX%B_8lB5z71Mrdhv>)0#m(=i(lZcWK)gD<`y4Mb;^7J78x|!@QbrCGt*N)cXo9J}W3(%Guf!Af8lP*pe9O_Pj zcE{zAm2H5l9Twr0=+B@%c#8_07eNc%IrPD(8`VlsXHNWH042@g?1j2ivSPL+JRVvB zTS1Psm~s)ETOA-q^&MfKM!_7*->Cagie+-w!ZYzHyzFZ-?aWU^vva)Nu(FOuCES27 z%X83Ib|3Zb{RQqvmY~$$TSR$sBelA*6Y};<1HCSDvS)D?b?nX~JJY|T-VH6*nF#~y zzZC02K6AcG9|P&jjqo+An%eLD$;ma-#mhOjaeys=V{`AL-sTGKu?w9>VJj983%4{l zKKhC`i}u4x%^xUAZ-C_WTVVFNj4M)f99%wDgW=>${Mk5@#4<{RJ)8wA!@qNeEd?37 z8N0ZXVP~kxwtTw$WI1MS*vxBtn}RQeEO6q7EFGd&xII`8PJea;@fV?R?szWjP4U3v z5tiK5+metoU5wf6!_QPV=z|d_6b<;d<9W_uR9gHF^fpU^coCc}63mmP{1w!#2@Ec=Nh} z>%HCvmM@b67R_ z_&4P)R3wSA`!o)t&h!G<;k1LJ8$XMw<+b9Coi&FK9hqGJye;T$5yG{39!W-Q!{Bl{ zujJQ%Es~`>7^!tVpfh(avnas><=(gw3760G_3bra=@Sk=f9S9>?s5?FN)60=ZbDX= zCt6uP24^`QQ-5X;5ws1(@b#T!%}q1(_Jc)oTQOyxRYw9O-G;OA*<^5rT$^63$HJXwM-EfttExqiq$k%X5G8$s0J5eS<} zkK7`cIW4^>Z#{U6*G(+|P literal 0 HcmV?d00001 From 1aab1495c516bed7498ec16f6d47062f31790db8 Mon Sep 17 00:00:00 2001 From: cuicui Date: Mon, 16 Oct 2023 10:14:06 +0800 Subject: [PATCH 2/4] Add fastvit model --- network_2.ckpt | Bin 26380 -> 0 bytes network_3.ckpt | Bin 26380 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 network_2.ckpt delete mode 100644 network_3.ckpt diff --git a/network_2.ckpt b/network_2.ckpt deleted file mode 100644 index b1b87630a0d413928420766c5655684b572d4ae9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 26380 zcmY(KcUX^q^#4V>q)ikNQmH5n@B5rmNXW>HLNp|!s3^12p4z27mAzBzdX|Sb5{}yRK*M1F!i!Wn`%wZTh_8)}y8DTd@YXY3uzZ?61jOUgJ9ZVqXwa}KaLX6&TBpkdwBgRQehgF<{P58Cn#pYNU2V7qa=U1_xgUei==cy48E z_n(Xmnl%3h?f(((dtuce(^sXTKy3%kwpO$A?;+c8+yv3s!K7i>KWx~cY>LUzxgce} zT6j0Y+U~rELxWsS5|2846PrWlVZez0;BKA_Ll02K$i_~!`WI^K z?8o13#&E~tes*r!YMhY#2>)eOK#ceaC7ZITufH!QjQNkJ=AUK?H*C25z|8L0;3J$I zVqjO4twe{dCp2U|n%GcrzDvkZR%>`x;Aj_lJeTV&|6=K+$#&CXEbIa5#ci{imf8kRv^qkG!Z^A%7 zs)WI|k{kaf;FH0YXuD|}PSCTU7YY~Y@4HQSy+;nO@;2kE$LB&sls1~Y(`L(mz0mjg zAoBYdk6(TWxWVZQIoY?+s;p)BDmr;5?^Xz^V(F5`?3-h?ALMPv!yu3UX@~QS7P|pI8Mp#$w8dK@0{~t&etKl({Kbj zJhjF_ZW**Xa}AXC%ZER%{v3OEHvS8-!Hc1>9FV=7N2&Cn32+Q7o8-k~S+-oZxG%T9 z5ny9$AY4j+3#U#y;IaR9VC$)JsyY7+63eEcW)BZ+n5oG9cmQs@)Iz~xAU5w0L1XP( zFm8G{ukY)}HzvhkUUn6mImMGn(KM8sT8^ii=ZW*yexbY9N8;5KS2Vlg!(xyh%&<-- zZ;xpH7gR4Cl@95yeLKm`z9xQG48)?`{dK2%sEMXC%TV6%0rZWY2zOot@!XF!9G!ez zXx<-1mSZCM{jM@RRJek_NM++w-&8z#$^b688t}#^)=;`73O5eV!wvIq)7*%);;^nX ze!Oop4!DzqWlI;aQd0`PZc`+epEJ-P#E9>wYLN919ne+(NT>hWVz0;|eBKrcwz4X0 z{X`o-mAS)|$QI%EmRK~uTgr_!CG2Rk4oCa@a9~q8X{1=c_S)V;biD z48S7|ccH?nm9mWsc+mOx)IVW9=swhC=UR3A?|B5rop-`v!Rhq_24|sdd=1B4?*nR| zG^w}y8XiAc3peY^;l;Z{Fy1c!71|2$pxN!UR=8YyEa7fquZH$vsEls5ZJ4|5hjdHBsrVTy!M!_Jx zN}RA{4NJGjp=r@j4!ksnS57K|=J&-kMR5mPE$xNxM=e9kAVbg#eG87K2VhS44mNuB z8)o}tLO`M;B%IUb!%AhaXoEQ}A83u{B{kyobz%6?IR(2`mE-6!V>xlW8kR2E%7rfB zI8=8z?M@el@C1)}n`#TS;~2e#*Z!5=^8V>E75CINxF}PcRLI>|^#QoR7rG z*KP^vCTnr|g91FAIhZ|8EyJCKCNNcNF5ak|4eLIY;FtK(5PdltYxAFz)Q-KBX}1Fn zp2>6gi)8-$a|#D{pPhA=!|_K0pwmv9b0VazKla=6t^9SX55YJ~ zLzg=v7QuDtgD`&cdU3WSj-{j5a9xro{GC^TsR!h7(9Io;v!-Bel@b-S%h2CjgHiTD zmViE%Xuhm3$2X^PykifZKWaL5+ztbOJ1d^GvKdU5sbl}-o%ln}0A!bjVLLe}`tvJq94WEr3&V_IP&?lcNj2&J=2YrorD0G5vfD-EU z4xlo6M{6&lQ1DD zp0fjQP|2YrK9uK%TKWz=a_>-H5$gjQofG*^b2E8mR6>oBJ-VAj-;n)l-L0CjeoFpO3X`zBWA7*eq7G#qOe#MZMz(M-h>7cZ{^lZ`nnmFbCI zfzj9;9mnchrh?J8V%+;do3)^rZ7d&<%Dk2MKxYLS_qIcuyOVKy#76w!lLjx{Ce(G5 zug10;hbiQG25R5;z;DJc#pS10@f^3^AUpT8aBJupIJ@s&ed_r%z!f7n?9myL$4;_5 z@mxHu)PwbpnedkMVIWu-pw!$OqUrZky!tR+bt_3aNdRs0o?b|liliaj)@r5NY# zN#(`=lK9Ie9~`kw;EJ>&7vHO~eq1PS z6l-P#qk6v%aNq00gA&7WV8diwCs~P)-3If7rd}MYzL^75tkAb<5yziM;;QA2$cjHH zy5|J+I(%JN9#Dvy3Z{H({XQ7;-ksXt%ahwsb=HZr!0WA1?7V8ca44)4Wv1B?G<)I% zBLee-XF((79yN%;Sbp)XaL+THZ?#;7g!qZvb3--jc0}`!eH z0i8RN!*(SJJbPRqo*G#%5eHSk({u%viWn&jFKiWBisEoCXO4}^%_Y-sg_!=;9KCit zgB@+gl#$b)M-ShDiefx%&$1+o!U{aIG?f+0?WvE>8Z7n<6+iCE<_{}p(x^>?*geA- z?OvzAz9lng)!r-Ow#&!Kar<0U&{>4PAE)xgtBcXeK9~DC&Ey4!?ZVuAb&l5b!J(-m zd8nry_ntq2W}UuBq02Ma`K3PxZ}&kRya(@t$MS*&;VjeoSF-%?Ku#=mCY9%RBpa55 zvuw`>_&8`FCY;aVUPe)HW%EiJAYIB<($z3ny^3#E#qhdeY3Mj*IPCsB7jsP$apzG5 zq!1_MA-hDGwpwyrnT}?G>oKThC8~}1PiXTC!S`p5iYJ5>Jn!{V9+(${YwixlhnxHJ z@sfIa~-)3Iuda&t&6wuVOzDC4J_)#BG~6^8$F>)tf^IbLxfzU!%@)E& zNfk%<@4!8=i@CsN49`3-$FK+S?ixofco4!9*B9e^i%2d(Wn5`dh~Xn`rLy@ zXpVj!U#rf;iix$5v?_!rq~9Vf+i|?-dp?c>b>5eAosQaEgxzwHFye=Mx8AMcgylw( z%H^lQEmVPL?;J;uUOQvXXd9M&_mMvPpA?6zQQ(c?>3I8DB#u9^kmnSB71x#Zz>TXb z$YrNH7d(x{n(rBS>fs(?*8pQ&Id2eu_UVfz2K(vUc|AJd;lzdZMQ|l{QMcA`V(pDv zF?W2dWa;zSoc7`bxE|dOFV}Q|*k=cYhE?(J$45y?T3X^cW0$bAcm>BV%{(!n)S2bl z`lH$`BV6Wkp01sJ2adrj@p}xSL&+R|y(=GGe0AAo{6XlfkD_a{Q_ws<7q*V@!HVa4 zJV-l*Pn}Wb*3U8Y^YT=F;+zS$KkpSwulFIfh52mR@>1}ym%>Y56)-iT1cppJES{-+ z1MgMs>3H|M9~)MR+KC#hKky$MZP#X_`yF)Nax#Cqz8B`N&t~J2VX#iz$Ok_SMumI& zTr~U|>~Wlp)+x)OwFEfh(E@HP0SI(@EzB=V5)5zrfzRQ2;u!-u7AMXJAImZvxlj)G zC+TsM{!ma{;)7G8lXsdiA8(AVkNhCvHOm3T__-uA^fXNRtw5{tw$ts#5boOhxjuVX7%2@& z;7LzT!PD9SDF46=(-)?(ak?5C%a+1FyHs|YHx+AtjTVdC=JUDD?XW7R45mG)!mU>R zxO>?$9{V)|A1E*4LZcF1HqMnT8yunkXJ6jErzba$jU{87MD%-Ih*!TA@gnON;2+sV z`OD7H3?f(NAYGmQAj~j28lb?h?jKEaK#ijg(wcN=NthW4Vq53JBdzQr5E|G{}|K=V? z(F*@m|3||H%tPI|8+qs{WehsE5s!Z=VL83abQz+!xLYrl9rwq-nXV*vzm6RHEAh3L zFT}}jlG%K$6VL9<&dmCe=YWrlTHCTZdthFdDTFl#zV$9Q40e!b`7dU7z$>@R#UaSwpe;cFliQEXj^(_nc2YA5ulf%*d zQ#L!+?EqK3_% zKGC7An{e1WcWkoSh93@mhF+>lywbmzwK84!RgkIJ=>C?pn(u*{WCoh6UnJS+be`2$ z0}Gz$^Q81(oW8<I(XZGOe|68) zk+tC@t?DlJ$y&{Cu`kMvun}L><=~CKUM%%|0?J#?KvVbCI8?p@f2bW2^U~Jxkjv{Z z>-IvtJgr(slH9tvvi3Tgv^+Zc}7cE+0rbCmOZ<21A3Ls5i@oLXE4q zdC71zx?awMWn)m|-vacmmE&)dT){OW2$!}faP?(R{5^Opk9 zt6=+(t$Kqqh>b zDRD4Xk_mey)%?u4fG6)vg^#16_;<_z^bXjKy<``6Yj{((+nCPvQv)dG@d_@EyGI*7 z?8e#U8GPVG3^&~m!tDvBVwL4gRQN9*KYMu4)&BJmWZ;KqeQm+f>>|Bw)M49uA^1fp z9FI#yqWjU^^~=`v!{-qOLjB2Rc=>oPC4U?uG&+=G;rTD(jbAD_fI28v?MiZ}1 zFeF$S*A}j#ywEthJbE;z8ARa`nU&-?_yUc{^yZp{rQI`a3jQ*)XS4OcsVz8~H~krd zOKPLXdi*^D&XNIeOrwHjPNm|R2jTpwZa>5yDaI+f@s#_nAO7&ZDlA@GfaUhyjI}w~ zP@l*V{kCxe3#*Dp0SDxpA=EuyK=0&u!DzpoPdJ`#h5-*K+TQ}?6YbS z7eAXw-{Wnu@Oy>uW0fyAuZ=^qs#0;``4!MI_8=L`GBmdbW?Uo=J1@^LIn=j@}2x<6^1b02V? zmy0He>p52QG_CK{WRo!k*mn6VnGV(9l`1pPcgsWQc)9?LOuX@WQYnAF5`vYsmuQo| zJGTmBguKd3p4!gvU|0+mOvtmLx78JP~zoIN|Yig*-2)m`<*?g=am}IQ(K6tWDXC^@;YF^0^$Rs_9{-`4jr` z!-E&ukKo4-llYY(;QWEvxaD9X{Qf=|J?;APL+2?Ht*gB``)2|TpPs|>t8F2MCH!k5 z;6+fy(SrtaQ%Di5s*ggIEIG{Y{}=X$EaUcvoALVM9sKo=3%lCopzebY)Hx{`)XN89 z+fP-RXH5%&V)gy?0wiUt`J>kOr%w-mB7=XH_lYt2zeS- z_~m0dMt(>{9dR)pv+jfyKc}~T>E z-zq$%8xgZmf7^2Yy)+;8dKcoX)qZ&S)+iyfP#Z?x&)}w3JzPX5>pe}E@GN6R98}}O zQd8n7v>x#L>mc5n5W#J0E7-2`HGEiY4@X;jf_z*VUO%ad=Yns-mg*E*)0#l1r#WE1 z*;*v2Hs+;uX*hqI6uq8%gt}r}_};q^ewnTbe%OotSZ>Gq^gbN9eGfFbZAICM+4TEL zE|*m&;^2L`yg{`H|6NIfBO{Dx`tMBqFsKa%dZ&p$XDz^yZOc)!%>k#U@8rsTnxctA zB#xE#7LDw8v*9Woq;MbByK|q`?OuX+);RN=jCj5sFc;&-dEwV8b&O4p;c1^k@a*Dk z9J2hoxM#X4Cy9Ze@pT{dFZYCt?FHEPbq!0s>7p-}-O#^s1}iM{#YICeLceZpu?feJ zvsEmO4jzGb+I-Ps%6`epij$(}`Qv0OzYa(GzJrauck|$7%P47jjW8fG8o&A%Q0k9AVEkhA#a0@d&Z+gjn7fHVg_b?|>0^H; zva8{j*SB-@FLQcdGYfwI@|IAK*Qy|F z+9BH7l*5BcyXUyibtvxfPjap$9ZxOE!vB_dU}dC)lx5^-_Pfn!-LZh(?S?0pVaN!%=N&Z84KB9j}E7gE8_xn6>j}DLWrw~sK*%`AInHusZiQw8TXg z^VsT(CMOj+px@F8%JupL!>?wDF9z&^XMaoiyPi8%VlPfK+=U5mwcvsADC%pxk;j?r zrg8eiaKoYmcDgtn+snR-O)oPzdXp-ecO>!7v~WBbSxCiWl93Nu3-`twL*s%TP*Sr3 zBcfaAZIUh?HopZ+t*5ZgeG6KDI+F$^Ex=t7mfYt@Iv%ecFRB~~;|T*-+fGg2WScwm zIeb4AgC&W}xgir#>R%x~-yXq1^EQ*->>jL>n!&M?7Vx3+5!`L{;AhQb5Yo!&)U6P7 zDoI79$nNKyZkE4T1Bd?&ht(r$pyRI;$IV~LTWhycY4!pR7mE4rC@+qYRH9y6I$LT@ zMwi1eG-2k1y=a$8^(h z?Uh<+x)2A$`_-UbbOC-D)|1!mxIkAf@2AoolX<|rIDFk{R)2rMc(#0yDE>Uu318eE z)3!VD=nd%@6%~X3wcCo5{&*p7l0pOhomljvTZhb-=8hgiVTFDWuU>JMyfwWsCF(LX zEnEs)tP;rW-f!{1^-Q$g(+9roxh_tf7DYiJsaW~Wfrs~tvb_`dm(o{jigM>K(H4Vg zc)czHFKPZHJ<*-dG*@HYj#cdC=>{7kgK_YiUHJIoKo)*x@VB!`IC#KXay}l1D;f~@ z<)>mmr9Ol>THxij0@nYwR=j)74|`Wx3xB#y*yX6e_Zywrp;{NOEt3ncRD%a3@y)#M(IH;);=xfz7gHt zY<)gt^bh39{-@eK}>^Zt?RtZG81zhey3y&u+0+U^x3CmH$}+ zo8PE%zGM)}PM5*H`Q097Wi)lH3l&#fU5eLX0NY+Hq>SEkP`B8EQ=dtr|Jnr@;xL+L zoergrWjQ?Rmlh{}*T7LL67bTA?tQposi4FCMRDv%_*zuNef?5#>1|`4eIx^?>HE`Z zn+fEBu42X8Rit9xy?dnk^000nN59<}ztoSWe;LO}_0nWaxnhix83VATYX`oW;)gpY z&gAkda!_SDnXfN2plyw#vGeb9@M&rkj=yt3tC=tAFXh@XPEN*S<{Qu>Jr#`>D)Y|u zpD4EXee#*;$)k@IBG+2u-`Kuj&|b_kPYrolpOyHpuM=LXvWBqtO6X&74aSa_6HD)! zfX4?H7WWT9Y}(0(XRf3JHL7Uo@<|xh5`y|ti`eR4Kh$4ef#)vf(wprz;+Sr4J+QDh z*3DamI{xA8QJIJqp0b$S%_6oR3B}S1CpPhL<+ZquhNWj?pCC(`c&r3hJukuA6()T1 z&Su=*mICX&1)g=ijAws*24luE4J=FH{n4FpPr;v0C(CmF#v(2|yPa~h>tV7^3Y}eF zg;GC;@buDbyi%pkMmg8&m1CxIl6eZAj2eP_TuXVVQ8*#}5EYmrUN8#G6aNKZnExP?u;$;bysHN&F>J_@6cYPZAd5`Ds2KPbv zm}C9X?Hv$g)9qhgv*DeO$FWM)YfAfX0L&Vth2A+IK)dn)^?p)-UhyYsR1aAWne&C# zY>MPgi&n7PDWIXnO~^l-N^|@-;MYx+s8(FW9aFdS$6^`d^OazGjNjXba(32ryG;POFp48iqA8gFU#y8!ZubX|P z4Y9)|-F$ZPh(g|HxRAGe)WkCNY1}ZUA150`G2KlRkNBP?y)z%d^RTkm8N5r_Uu(?k zN=i5Y#*6V;EANi9G(-dd(q>DW<^_Uhqd(_g80Yz9>p2$5r55uIq zt-MCrnYY(YMeo-$cvEu&lPsNJiOvC zc@x)S|2c-}d37Yq=XjCRgkZ_tvQcPZ=*>e85^^T1atR1h(X!q^{6DIJT%1^~S%h ze_uQSXABvMdRjeDdRmeY+4>M<)bSVGp9m5;twg}R*7O<4MvT%Gv9`F7;mi2FJ3>Rb+;c>q#7--f=Yxn#XZ(8pa z3^%_9Td5zZC4C9-PM6l=bP(QH#O0&E)#LjbIo*ezKyP!6^_ zJ%G;mJ3{ksAd6n>#VOtNcB_>lo!|Nqp5N`u`AZ^5z9b*!I(O0A6>hNQbq_#UtX)PW zI;G$ntY5!dw~~C!#=xmDHMGLY1RP{P3DNtagxat7gxSW8^^XHHDe|H|SDWQSzlVD0 z(WFB%n@YrIDcaF!>)1T6`Ugzm&Q7MewWCNE}vp{s-M<)(dLp9u@VHal77yeBF@vnrS z*-ypZ@)~$)qZwqoYH@U{Gpw1S4@H~%h-UX^hidIIOph$3rCZ(11ujwpw z?X-eF3wi@eS;C}$U9`-{nO?=Nh7!$va8d3%xs?8Z+`o$W$mc7REKC+klkd|^+ul@n zIgi4n2UBP1b6T@+IJez?Pv)MzgxG!G!1RbJfBJkw$b2iu$;Su4U0o$Q8!d<1jv{f18pbmaq^{J0_ccYah-n6S`6ilr;E?M~GrZD7d3D~^3 zK$G8SamauiQa0+r_w;|!BfHT=ht3O)jb-Hi>NU-C@sw=+ae#JyB7qI^B+q!>qEGTj6bg^N|RMGlRJ%qJX!2$QZ;>i`W;oH-MZd$J=1XpemHPk-B zw7a9=`?jy-p?rdV3Mc5x0}siz=$m3~$ZRq8SEA50pjyZ>ZGfOf{iw0U1e#h#(lX}< zw4={Wx`2Zyn9X@&e_cVv{14r5N)``xr|}l;WZD?tNBA%9F|Es9M_1qV!ez&23C9Pu z3lm*sz+Q4#=r_h4nj&|=*(D~}v;8+M{x*=j)KzKDr{feb{j@k))(WDY<%ss-*ShJC zDK9Z=gNW&O#2d#;Al~9W+15z&?K?k&+KLpCAG-xihD@jU{3z18xVS#BB_Aq^JO#)2 zg@B`uleSc`IHyB~16Fy_`th1j=WrByi$iIxV-_Xr>%rz;AH`Mr$`n&K4`hCX(puvMWECeN{eDy75Vxa%a`t$GP_y)KgIIU3$sE+&_L#gyNE3f$Jqg0xE?n)f3D zZtEWrw%M1#uz+HS=-WxRo-cx_ax0+7Glw4C+Y7!eWs>cVs$4MGyI!%w0r=f{_*`!- z^tW9H59FnB=k+)BKVHtE>l2!3fo+EIrQT?k?>eSyQm-MvMg|m@+ z;lSi#apbc8g7FeRDmYmo+G_NNv(B+}^1*JQ`a`i$)}3Sszy5*xd^#zfv(yt$FCR}| z(tc7!W+cp3UBLJK)!1r_uTUr^!?6jGFfAnkUUtvviq&|+w{3zD{x_MX<_;HkpPxa# zhGwX>FrSR)JJ7nqVoK96CzY38lJJL9;B&?~Iy&wNbcPNDmHGD|IPf)1x^6>WBeUx} z%-ZPf(rj8E7Y@%ZO@jLtL%1+YQWsaFNcM%Y;GOXg8rudy#vXGxHSYG5R3)`Wk^ErK`!Q$0)KM-tVg%7$1 z1eK(I6r?o<0^Y5ogLzjVb^1a&_;NASYdxiBxAo!Fa}}JTUqfSc-bg-l#nI}SyJ74| zTk;-~1M+L~!EKxpADWyexW#{_{Kx`odcR9}^5hp(K5zzW=efe2cm9wYzDE=$JAm18 zMbYnPBc<1^fTdsk(e+k!J^!~u{QYntU7Z>yreF?O`aG$B%3p1l>>P;scwba(lch<0 zo?7SXxsm2}8~E01sZb(+moB5MWP*_t^uae{&!&6>-Ht-KwSF^H4m85UuMOe+kH7S{ zTlG(MaHOf#mC(=iiljBXr`T8XiEv@c6g;0GE$lZM53d);ilO1#NY-tW@ZT*fR)j_> z^mi8G%YM*7v*A$f)15$ZwSrN(`K12%p6I%}kfN&1X{z!dLHq13Fd1(NhUW=p#wU~3 z)l;zf;V-cHB8A>lj|uPd&Ow!2Fb!C(4P#g5i}y}O!)$MDO4ckDYg7rg8oi|r{!L_E zRSVa?-VrXy9uSW%Nrk|!T&kNFPPT@j5IWqJr~R5wZ`YrK5Y6s7(=upr$&9+%%O&FO zcN5{q*(y<0FHxNSp_DfER^fL(^>kGFAsI&BrGapr007IENOrfYZAdcFIzPD^Nc3E)WDUSPfB#!yy(1<6rB+pVdbnOy1Pyd z%Pz}~~SWBZubKn8U9U z4{8W~Dk`df5MQ>23i}r?fe-VJ3KOmj;hFg>>(BTH(6|NR;PUnaHES({u67xoGVP8K z(vnHTC*2lr{kto6T*v{B+qv-g<1QFDzY_Kt+y;f4C&hWsWkjhGX}aY-kJA$txJh`yh_m&}d^_krr-NV{+lQ>rM-Z!w6vL_&;8*Kz zx>*+llkUsI?6=l+0flvv-8LhjkG2w&>pY`vZ(~L8bE#zLr9$V%+hXRkDnUWEroO=W zmqd4U9Ie#Xx-IKb#(O{7?*hfsd=f$-C=45Fkv1mpfADA8^woY1{VZZ(xueAN`M9WWQVa+9gQsWNQ1 zW=R7x6Tqk7C~aGzM;EvMC#DV>NyAMwdEBV=5U#Wsd;-?j2X`jJbBhz=p>DVNTB-{S zEg(Tt_oH}Q#Z>rcI35am9;W=Aui<@;5-Ym?6oP9~p=ebv;n0D>Fxsg*!Mx3mhR%)_ zjXe94R8cUP+`UOvq9*7cxn5tWJp=sv-=^m`2T<_HJz{FsZzwi25HEKY(6cCAu3Udu z9KCyqz$4aC+Q&G$|J#~e3KU?)J84Ru-!6QrsTG4h*U;yBpr3t|A#aKcc{RO+HHNy~ z$&oBUfm+0)J718ZRS~^jpF@Z5#)<3B52UL7K<}l8(!Dijgz0kObRp1BynSjNU)h}u zPs7H+=~fG9-e>~{GjiZG+z?OCtfN#l141 z3l>$BH8UO#>@g$T&n3dX3)3Zwy0aOsGf6j=H^SJ;!%hZOhRq9^X-#KH^X;YRpD*x$7Qwk+tyVI4|vTKP%+ zj3r-%0gf+(4d&{&H*5gk+qtWL;bwx~PTiWZwXYbv|2e%_>PHG&?8V%`a4|y$pssQ# z95T=&gSBe)4=hH~)WZ=Jzb{ot@m@!&$)hE?u203Armd1|E14RG4WiPh`%rH?n3TJn zUIe0w-I_v;(l7|#)t3x?^TF@$K3e9qN1W)S&6Scxv~AcI(Q4KgS~_x%P~7Z_Gj4w( zoo<~tSG+~#V-Ui=HIuTmKiK(C6g6Weu~|SAAM9$PfKCCtQ@#mS(Xu#S?u6KGe@*yx z`G=@7>lfrH`-;}%iDnJh2n%w|prB5QvWBt{*k=qqn34^f!&n$|?Xg6+Pc0k>J4j8k zA)wZ%FFdDBa6PmbI&Tz0X7nmiYlsTfT+ObJ&C(TaCLf?J-3f$rnU6xg?HTc|<`3ai z>1RkdATI=r;E72U!vrp0xQcj(C&yS zmMlvm`*(r$KX0jW{PI%Bdi@C&MyQLLR~32vxs%|2>AUc;?i_`FHb7;EWN7z^qj|P# z#F_zbAwe9DQ{G2V+9Z3J=RJb5uFQhok>!$}>KT%h>{pUE?<{EQN_S9h59Y61K1h<5 z9XX*>?LuiM)_~pbSYfh7JpC=%3laT33p2wK#g^S3uzq=oXq@ww4&}{+r=4v=P+%7{ zIp~3+gAq+L=r0oiN(=b=+R^?M-r}vdQ(@!cQqYW- zfUZUN9jcZ_4|6kMt;7WW{z)Rux^$4(rPE^l&*G)0ZKN7lA@+zrQ*RJiB#sa1PLmoB zVs9M>IHr~We!-?Bs+f{ai76O+&8Ioqs&w?9l~~!;CAbMu!sw85g3F7a^g8h~g_Yi+ zw~5as!B<+y*WnHI>S+$EwiUrfx5x0`Gc5|-ezv~2E1L{Ec0r`yPl?UHOVlJ@qbeC2 zzGgcQx>PD5_N*N0Dm&7T);ZvE=)LIQeu12`9zbM>y67}XiN}4~LS3f!sBuUmc>4}aDWgW4npM7me zKXHHk-$kE5S;if(<0H&D{{b{}jD&}`cM8M5{t_LR|Dw`5?@0QY38 z#DjiILHpVr;p6UL8n)pyot_v&R;^w@Kjhf=NG54I-y#FaA@Z^vj5F5N3!kT)mz;Tg zit2*`;k#$2*wbVsJ=5Mq{tEly(wdWEv!f+vEx!a|?<&Pr#=fHe!dmgWOq^)5p*P&J zl@N|?6%IaM#PhnzZ{30+(6C^nnCPY;TEwZ->4UbA;I1LoUtKO#PmZKxv08X8rUmYO zIU>AV;zd79)uG*VK0gVP#c%7i;P(Svs@;-Ioec-+N}nsxzFv`2dUV1P`97?=WFWsS z-Cr;5(<<2<^H{P-CRosT)*uX-xs|Fl`f=!NV|?!(MRPJ7Da~U7&7P=@M#G9kIr9l1 ze=v&l`;Uhhd%OC@of|2|l!PDFQZzY78jDs-)2b_3(D~yKh>HKnea{PF%F5Y;`X=2^h+U?{!?#Jup*#-)g42YSv-+I& zCSF`_|AbzvdI*aS*ieC`GQ{2MHUi6Ez|FqtlJ7H5z}8cTU{~m)`cIcHL$dKrA!_Lb zI{L^OmR}B?)|Lez^Cp6(yuJXkTXsSC@d(&IU;^2%|00A>U0AR2T!!q7 zc2Q+$54IguMqf|;f$96Ipt8{mOueRw7fRKk?;m@ZF@6j;r^LbdbI;-Glq+InLNk<& z*P|B~Ckp+2YvA<5nZmaj*P!ctDug#prs>ai(1-C4DMwm>eEr@yd%hQGuiQiT9+X1! ztv=v+%n62R6@t&$fne0D5PpB^P2pwQu;|)CNGa)dxIg>Alx>x2r0WTc^xq(p>D3w_UT-+$md&f`4J z5=Wj(1bmof%_hnp;T^wa$XnGwLFd9mCXhyg+ow3NyGB?a$tA?2B_Ez9M8h=4H&ie+ zkY^C;jroc_w3G@$XY)mr))*i+v*(cO4u()zln>u*qG{aM`}p$vM81-x0>Xa+RL9Q& zuQrE}5w%8C>zl>TQvSuaO>;rN>~;8pN-~eEl*zueC(&RtOGXx*w`g#YV-D(Q5|tSW zjIxX^RK{Pxs|I>}5B)1tSThvgsJNk<%tYe$V-cPXn9q#b9fI2`FYvuh1TMwn@)u(e(%XwS_ipi_8_SDko-=KM%Qk2sHzZVLvkY^?;uB^j=ZE-lo!+S zt`qb!M>YpG+OU^Or5IZwImTsz8q+%zjQhUHpkc{b>M(sSE~xTgM85bDrFa|WP%VQ! z9um~^R|3Rs&*sS1EwEli7bGL4(YY&(Kd@&bR7g%`HR^|^)g#=ve=y09G&7rKF@#gcN7V?(&4 z_8cDFr$iJLb5Zhb7{s|OT7DAUlN|>mt zfsrXw*$u9t$lKt9eb&-g(ZnTO#%92u-Al04FC9b}7jXG1g;QFOVXVhGder7APDtcv z>FXKLZ@3H}>sXSHOBLAfH_y`qv2(OcrW(hT9ML51DXm^F%L+`tLlT#$F=srcv#yLj zG=$WkikJXMx)$T(U)ebJU5-_;5n;_7qfs_<7Je%~j|V)tbL3S!HeMP-t@;qmz4xA~ zImN?VY7TWbmDx@!QAY1fAxd|Q&^4n=sJC1yo&2qy_giQ+ohPP3ckM00R#nE*XmU`UX_Lp ze!*Nj$VoUFE6e;gltw;08f7UBG?Tu8s8POUfQLX+8a7W0$fkLe}YB-D+|x=)eX zA{iKqT?dlqe95C?305{w<#FTp%iEYq1c+gaddc#BH zoXigh82d(!*@dF({e{e|hXr`tf}5!aIxs>z77IDTZ+_fPV#ho{A1?X4dOo+08D7Q& zh05}%KS7|ARfA>Rj+r4_PLrEN*@_=}xbBEQNfq1xpDWWyv+5WARL;ve6daGshL`f( z=Y+xXQ$DCRC{B$ha_?g{4eoWe;*QHDFwy8C-z(S^YSu`>S)l}6b-x4)Wrk>Z+b-a5 z-b}ltbNDG;^LX*w`-w%85k^6-a4Wq}jC-Yt=J5jJkeklW+rEX2{~6)2B|czkm;>n?u_dxc1hx%6fQXS? z7-XigkN>_Ud%RzR%d6{XWb+tj#l|3WLyxU;UrA=g4x+tc2evh*(l{eMtP+TXDC6^R zw0jx5>&78G+2al=LbEZnh7zs83Kl>U4wh-R{HdD4(6n|E}prBzred zxnwciFt;0122O&(UJIV!<@H3QG7F!rc}wyh8^Z7OYZxDDO#{#cd~OM|gU`?7bMMob zzjGrdq^^NgCA)E~&lIbL4&wBY2z*dpiB>U7@s`O{*wv~+C+$h(C(l@og=QU~c2}KM zf0B>&2aMU(!jkM!K^BMG3NZ1s3U;h$#-G*I*k5A>q~SaLbl>F#5+BQ;234duxy6kLB+6GHx^~{2gn+ya`n*=Ff<9DQ=0XohLvPh)jv z)HNK!-j0)P^QL09;w@~-wqU*$NOQE;Gg|uZEXHa^;YKc*IX-;_`t{6!YSUs66R5{& znPTkI<+A8JtAOlZcoyczorcLXrXsUn01_&BaQJL188e;$4Rtp`*t`PX4chZ=aUQ#q zu`PZ*7R=iIYUCG%Mu1In5=?50Bzv#spl#MCTHEbFOp2r7=Z@RpEg5Vv@W2yQQ31jd zjp$&r0E;t|p`=c#{K0HN@EfVbl4+g1ALD75(5eQ37jnTXViszh-oxl|?LpBtshF9s z!5(S0WSqaAN9!?8)Avv!iAUDq@;oW1=@G;_#cZUvrAW-@o%lw$4+fH+5VtKlG)b=v z)_3~DdsDDBiH|b{J`!&tr zJ8D2y_+~(2i4$u&zYndo){=R>qO7QS40T&Mi%Gej$D7dO074@nU~waww>Q}yw?)d z#f(Fwf&=MZ#k zR$+ao@4`=2>a5#B1Ms(92u|;RfVh_?6zN;wxyAiBT~`40_QmstZziMnKS6BXwhxre zb6_Mq2!Af9g|w8p%u&M#?2$M^nZsM4yEzVr{!K+eGb`xSD#7UuRwTCb8UKc71Yc8w|x_0Zf}9k4R-fCJwI!7EaOM(1YWxQaL%@iGH%i!;z?V8Ong zQou9XKNUnXC*yzT%E;7!1HApOxc|aWpOr!az9;w71aBkO=t~djjhM%L_;8;#9C>GP zY4{y>2D~QrtQ!0ij^GV-N-(uk{P1nlO*mO`8fHf6F)9G~Ot2l+Y?G%gh0Z7_Y=YyC zGCYkALtePW6{wQqW5>TU#5#9Ay!VrZ{_0|cc$Fp6{r8eEjORQoP;B=_v3E7nd%CSfmd+l=?c&tdJpSei%?5L1^(;v=eP+y zW^|=Bt6x41KLv4Au6HaXO(@3YC3;x9vkHG-wjjGA5<#D@z+8#Fh%p-)P~x3UdA^Mi zF3%IfoT_+)Vp0m$sIt zkPIn1&@`51pL@R|sl(S%A$`6#%g5rvISP*u|&53Ejx?;CG` zMcy22{C*2tXK`NO?zwcwN(0o-v7>UYB$+7rJX}?^j6S-01@}K*%4rm}Vg+o-X#HeSebh3cEVru+H?}Ai6Tf^%aenC-2Ieh0%4WSpfkq}-<&WaA|LCq`mQ;qo~WkB?8hb=Ch`246A=D;53iFxF7KXg27d1(ssHpBSdjb-%^y3nfe#+S216^d^@k!mx_>)d zsYs)dcSWgPTPS!|e}o$mKS|e=za-+hI$IOjNu%$`1H&biLyxs#MtKVQ+m+L6Ezh7J zvK9}^^C@q79EiVGV_d(hQ^OY-7-oEpTsSE~cuTeT3W@leD;$>Rk_N_F_B9HX5g?4}bG-yIW%-w|f^d z^&nytfv*qN<15Z9{c$W8HZR}A*FqCK5Lk#2vp*3j=PY2PvLJDl5*j6Zfq_^@{8@2_ z6i%GP4nMa?C!L9GnLdxVNoWgE%~u2;BQY%b{thlmEu(2NGnuC3PzX2IW%!Ys;a+w+ z-K8`e&x%YZ%g=e>ocy0KZhVi1xtBoi6YDK8J~j(& z-$?L>SIlBmBec=QON|9LF`Qvw!pyh+K{6cn;M|1WeD5R8816BFDh=0>bRjh?DBgo+ znXdFiKsE{`OMk?A_ceOL1L~0o0>1rwm4M~yI4*3a$XJ{4fTP^c6nH@FU42v-QsvWkWMjjRFGB5lZC+z?l8~AlQi2sBQ-gvG1+rB8V3C)Pt0e-fwzKe zvcyNadLh>cmKcFuwUgKumzk_dm-?O*bPFU%5naa*=Q-o zeByLew%=sB0%uzHzc(EQ>#+U*|U ze7HgU&G{qGE1y6^z&(IeF|aul0`K0XplZE5)LZE?LSD_VX0{dc<-7xY3c3aFIS$fX zcpB125Akr@WOmZe(~um`rPH6TMY{qZ1=r7Fy~-W_7qMZ$A5lORQt_pqFtd93aWu4a zV7B(GCk<2bxij6A{*@47;~YXrQMWEzmM+8O#cS|ac*?QodNLr>{V4bC7GQSPQt*uJ zv#{xlwpgEMLPMSq(Dobv@@g}_`Sl7;<{Hx}O&ZMMN6T1;0|W&^mojEw>gkp`8`83B zCY!jdk8}w2fc=%%_;mhfXmHHp<#$|w()d{HbT@>X4+MyDk_Yl*ze4q98OAYGnw@uB z5N3Eh!tj}knf|U2$T)O|>PX7aK+8e!jSGgx&o@y=ZzVg^rjx#L@}@dRZjh~U#>+oG$oDxD z1>==ma<(A=UrFAC39B1;FI+FelR2(r;?yVnmm{)_Y->G89!!EYZwuk+g3tU(liq@t z&`bE)a1B*Ge(*#86M$Fui}Bm*MwELog_&{lA%7@l1C85h#ZPUMVO#ws)9xfTj(cXQi zmubn2M(LAN-kaf<*eI?HR>lL93!%{VF6FN_ViT0+uS;wa z8iN^;yv2B4N{fwlU&=bA+%n zoi;{9!PSgCICeK4cdjc&-enO+v)6+!5Vsu1kH6;{m^Q7$hUM=HeFYmHS!N8PxT$?h^jh< zOe!UI92@-l#31R+w8Enmk#J>>9_l}8M(d{k*y6rCDzwT6o~kP0yRMxeKG;PMnmVEY zZwJV<&SafusbI5O6-bWErY=`9f!!pGwg0Zuu5@8YpDc=|;V zy&8s7J($Y6+39?|z!gg{t7TKOJO6juABcmob=gdU|phTMNE;paqk7*3JI&bvQp z=b#|#_FNjW4>F(=T?snlGFZ3Z6zMEkPczJeAwk~@hn0R%+X#KW**?w#|G@%vxr+Xm zg5Z1gBybKZ1oQPJc>D1>TKcJ;_qOg2adi0#8T@at?wi~ zOYHE5=@BA+r3yc`iQ-sqJuwUT#yhj+yoHKa3_YH83IsKg|NdYsJz6M;i$0rRo9R*V z(?ON}_3!|g9+rl0Q}1%kr33K$UKA>fZ-;GZ-gHJRH%mo-rI_i9GtUozqw`U?Ht9R$ zTDI`Jl6Ck>YlLAi714aLJUyCS4o`kZLG>FmeCh6lb$h4dspL{9yK4xBZHr-3Y!WXo z?C@QZv=)(QyMXE4@z69BYD=XU^lj zfCuyb1(fy@@v%Vg_`Vah;sB(1`|;h;^|WObiz5Sj z=`j&k8Z*TZcQ;G~?XYfaUb`3H$LmowPCNCQf-t2=8~omLTK$VUlGE|qi%1dIGqFsdmei)Md>TU-OkijJ)~qsV~qoZ3t#waBwd zL$N4!ZJaD}IRQ&H*nr9kU-Ic>A)fl8feld)(SFr?cwHdOypx;2#%oT5Kk8XjRznt^ z?g$|>)5AEUHpAf;7yH9>rcZ6N7%vh0imdkrzB! z)_s8_nf$koIE2bD={jLxF0Ft?-r;m2`9jK~O_?`MoR*G4jDli6@t!Y9=61e8nMqd6 zIa@_WQEmy&I-vpvwsNTTcs=};DyL!P59yY>TFmmP-}$Xi!r=4xMD};qPtqA-h5gIL zVbA+SjJCUgxkFd5_0dz@>LSN4(iSBf6=Kn;t%ZN>=>(>HbvjjBtWG~!^npp?XEw|C4(t!CFR|V>$>imoU7<62ti?VWa!S>Q*woz6K69R2v zO~F0$N5)n7V38Pe(aV74f6N2rm+H96W9&9?#`QvZp`Es7EXsvI- zn9nZ7`^R+g`v-BNw`VC7Bm_gin%B7SlNGa|wwr%UG@UTT7x4;D59De(ApEm6)=uok zZ=aIN-K+WV&Hfdcq~8suf%nk2!Gt$SNrG=SW=u-m+Th5=RM@mh6uVx@GB-_I;MZ+A z{I9ryR{i8ee#<8~VZTB7{@KgnyA|(zp z;l)mQY>41fCV zu(eJPUEZs)TTLZUISQKGT?1n4VpzH;nEVq2xbJEVdZrreGUX7Cld}WUQ6%+s zryw&~fW3Z8nGLDpas>h+OpK#1{rTxKnyx&>-EUv%*L~+<)ayF-cb%p4iu<5zp*99I zohGc{X}ZPO8hCF9$)KS)c_;9YRv5+*`8UHHw>g_##^sE{OsY{^#+K)v=>nxLHISL{ zktRHg;_WzNiNL>wY(@cO&!iZ;U6<~x*~KrCdc&g&Q$Wr5EvibjgU!Mbw7z~G{M<7^ zXtg$VTDc7}i;>fqTCqRyDlg&Jb)G;?8t}cHc+2JqGQpqv_zuGtsbpw9y&PGMwHfP} zemQx3vwkyL582b1na=omt3Jr?c7pP?!5~|F8O~IMU}s1<|G!&VXrafoQ?v^(L#qrR zzV-}ipV*1}Wc~5h+6G7y5&)u8fHiRf>@savIAWfU4FWav8XJs0mu1-UO?E&x&87yQ z67cYpf5b9R8m`0*p+eObj`dT-*&1HRyJZQ3HSK8G>OyNBPXKa6gr!(XyUN|)}P z&p7EEf+5)_)HZt$)OpG>q3OvWx`#{~@t@%GI z%jn&kWstXEF*0+NnGDHr%reTsu|JPs)vPkuqMODu+`N=leoW`vGi#yXyfCx(>k~@V z#KAN5FxDO8wCQ01%r#VoA9outuw@fWetDZz-na_Si`r2!>@WCzl4Gs77An^p6CtO< ziDPNjpzn>%pw#z|{5NqjD89W0Qj4U-xq$liJ#GgTO(mrMA5EVgH^h;;oeAi6V-;p+ d_>hclKHPfZiSj*mTwm7$c=3ws!oOXI{{wohO}qdA diff --git a/network_3.ckpt b/network_3.ckpt deleted file mode 100644 index e2841bd5e6764acf0f72ec2bae75e29051f38eb7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 26380 zcmY(KX*87&)b=T4NF;iCZR-gnoFWmC@GqZ88Xj9C?#nw{|4vYo6@96 zsWca*d5~xxAMbkCv)=W7JYUW__qq4l`?{|E+sc3BlzVyytO*%mxxr_JuV1jndpSh~ zMOj68MR|=L&i(;Qf=62G-IJ@Cs$2DN{Z%A=DY!cPgq!-56uTGUgBUagjPpBF?!Mw_E zy|TlycWw@2ehUzs@_-AXtzh)IAyu6UebHrzgM-fJ$HLt62hp@ayDGr6h;$C$WTnf? zIAqIp?ET4`56&=fC|Y`mBQ@GEK7I%rq)4g^ZshR!-B2K+6i>5b4?EY<3)c}cVmGb0dtW(!;81Yk~%1hlIPrNp->i5;5>eKaMxOH7B zlw7tJ(pK0wyjw7~%JOd#f85-NN&ij5o#SNi>`G>7%insyEo?`b+*J$es=iftWK z2Y$!UBbrRFV_12)uEU1mT5KQv7=z+VAhh)zo}Zdb!J#Yhi{l@jzTi9`*gCi>_o-zAj z^EYo}-sDUydRhzD+e7jGK1GM@ecAk@M=L0oPWr#~XZ^qRmpMsMy{iKMzxoTGq<(NF z|LExtF8hkPpWa@s@r=h<&*50NWD71e8cg2H>*!5F7%Jb(<-fH|q$BQNvGdLmP<$zDyQgKu7QhL4U1C#~1p~A}r__wzb5Bt#PA$JdO zUWo#v3{fTdO)J^+oenntR>p=}L)^Y42w%M_LTg_m+;dQwPycx(zy}q6@LCbi=30=P z|9R=Vq7Jb6m5i=mjL~p-G@pIb9h2XM;ezIH-qMoJ8UJ#zXL}rFBue0RNGZCTXuy%w zL@4^a4^5V>o%aRqe<%PCk5iy^Cm)jCx`U*qZUvWTSW`K!#-6jL^07-%F#7vg zv^^A!Uq7@6EqTGX!m$X??C8&O%|2MyDuEICb5N9Z2TiNp=seX4maopmfx!=`NP3uJ z9n-LQw-QI#CGsJ;iEJPj$BQc>aqTll4DM%6Zm(-;mz^@K%FV>VCPnNKz7Ko%$zt6| zW46BE3bkiq$n}5+&2|`%8zhJ^VKywQxr+kKOR?^%8Ea}~bG)uL>d5a#`zZ&&Cwo3V z|7(xddI>lnErFMciCFSIk=Ol0UOOxWrxqlLyW}21MS&)J>fZvT`5j`jo*i61ZY#>Z zN4)T;S!~lZ;#by=ocVPzO^8}W>ZsZX6(#K)_w`e|Zewkb?*W$3iDmoq; zf|DgWd@Ff1SPwb_i{Cql^G9sw71P(SLH|`?5>kLscU3U{QZnbQamUJif5f#9wJ7tE zDPrA8;m0aFJbpxv&3`5FU`sh(-ghb*URnwAoi==^x)JVw>y2wh=i%~vGdLNv1@~>7 z%9%M=!TrE$sPXqe6Xyu7tgeD8joXlr^93e9aKRCc2VjBz1!1k~4x#_+SZ<&}sPa0D zXIDI?*wzL7-w6rN8&V}YsYu{zSpr8|+wed$HQZWg%u>&8{LODa2EHwzNz!3_$M_$Z z2dUuMZD0*haL4lgVh^3Tz(-Il14>vr}16;_YYn8adB5qGrlM; z8uyuY{f)x1&05&ad>;Dd>!9zK^<2=HjC;aX@TZMKAn#~2pR!0{n{AogH#Csj{ld`l zlpM>+YVnp&!y)zJR+{Cp9&c=!&HY~|Lzue`t~u@3^^A_V=Hy8@F(w9OUvA>!@!6Pm zH?phN!+C+@RY*47fx70(v?Zu7m*kg=b9n_DPqgEYQ4U!CM22-Qyby}o&9Q7_7^a-t zA!^0Q@x_a7m=}@2X6d&`y>~Jni}AvM$;dap4rI*>>tLF60`snV8alWft_x#u`}i2l zp67@z6aItq1D8{LRuHPhb&^^SSx#7}#vW?AWdHl7c>B5$t^4eWi~QGd!k-Wfud_z0 z(h6{Rn#F!uOR&Q?8qNE~vQj@MxKzIzYYVhkUsA;H+U}8e$4b03Z55XP9D|pqxnT5? ztthLR3oy>1(qMle{>-VMp!h6|ikpXz2GxrHKKikV;cLDz0a z6AnL2BP@4e)s_@KJ29DOsjR^9E0G5-E`?U7a^Bh+!!NFC;IwnYkgBUIHMD=j((wUe z^lWRcI`2#e=f9&JyQA=pkwDG$A7E)^6XiFAv0sfR`i1N!tr9z4)vs%YsVRt;_w}ba zt^N5!niii~m@Ha-+JM*Yb=BFfr7XB@#dLdj?2)q?rS}ZDx1S8}Pz>c>8KW?A<6KsA zPU3m1$Dv!pCsNFvh+6Zdg4Ky49MNwGZ#6y+EpZ<7&OwtBYBX4R_6W>)70DTc!-Zhy zJ=k``j_mq-BTlJ=DI?Cp?*lhT^Kl4D-^jw?Z>enkv5!N-mVkXh557{pRQR^BQ<%3s9xwDrlpL34 zQhshBsvaDPK@pF@>e(=wRj$qbeN)hUUIH!gwxVqPeVFQz%132})4QnwxMPBsm;l*) z#M6rkn)>mUdqdFgYzDXoI+ES1lVVfLadH|s8}r;2;)SuvJVR*-ZnVqg!Mmohby#;u zxzd}bKlekceU>aF0^c%lqSn*r$?W*fO)8V|HEQFQ52J9?M zrL~IcLa1*z7kDzvsy4!e$}GO~Ar6cb189X%%ym|!Fs5`b7u|_wmA9#=BsT~er_Mm> z;zS%aQw7hKI%8kW(_*I1F-p!$N2|t-xW#h~p1Ps}J5)pPl>KpWtdSq*KV8TZO1I;n zIVK3B^*GTiolA{laCv|h$&}5;+%O+r_~IIr9`okgpU(&;7AqL7l2|3NjoOyp6L)3h zu(eG)J#sDLwvk1oR6m(tYZmhJ{T`KC+1mVmg*8_UH0L;(bZXh9LD}umc&2j-yml`| z*EQMb?pO~ALwoXJ-8k^+NZ}24=Hj)ByYbw<4!BjP&cRNm7`({@3m>|2xY-)?&Pb&n z#!-CZ)B?PhCx==Zmb_ljXR%-b&pM~ddAnWc_P=tDwA;h7=^-GkZG-`~X=rC2$|`yz zFlwkGA7ARr?q2%&1=()63uzficE${T_s`Yy4YqXCVicKlVX*`e7+65yc=kYUH z7q+sCz}w}wCC5HbW2Jf3@J|v4(^hxG?WdCHpn56)lRZJVS`JF0V=f3G(SDp(?JE6t zU;=I%hTM?>cuL<9H%ytz7Sjt+W?p|jE}n)}oiSwoFa_tj6vMNb zzF66)%Sq13e5789KV`+xoxDk`+@21(qcg?gz<;84RURvxZWVf5kwX*r9vHQu1Ug>s z5`z}Cf%AU?o%G5;(X|8zDywsjd3Q|N--jJz+NnpoGe_+>1{<@oSZ<;voZqsAAMG~9 ze3}027;zhhxX(iMsq0|3IGT;KXYrW|dpKO#DV&ol6JDH^#d{uMqDN(SemZP9%#_=U zFV@Im;ZJ>5uC#_>zSHsIlI@)JC>;-#C3B0m86O{ALA}>KfU1JUZ1=hs_6oTv#E$UC zvO#8`Ycv42RWBBP_RIov)koCN{T+-8Ue8x7DnPS2gk86L^9B74oM-x6a5L4$kGIv> zwY(Dy40l8SmLC$+F(c@3`CzV2uai!W2cB#$;8X=qN-8CAbm&8KwPMNc{s#WgQ(b&8 zGJ`*G)5LXYUL4xl2IV7)!Q?|J z{+zG^6+D))X;J)wo~y=H||!b%uDh@ zAY{@A^q;29D@OdIhc-zVc7Ft)96t=VR3qA5U4ox3&cOJzja-)DLBZ$jvHfub>m=^Q zOKuzZwqzyjaX(1@2LpxB&3S^=&|>y+xcsmEo z%ZcS%C+)>oUPY9Bdm+%0k5DzSo+?fnVx0zZ&#YuVG&>7EFOpE)-+l1YHlH8b55OaB z)_6rvhO4{JK=%!sIpmr$uIcLEeKm@^d}sqr7#70?(^7FWtiWOJUevdi$$qRVFE8s9 zO+F{{GBp<-rjpNX3l>2_a2zl3j=>c_e18z``!pj-v{Q7hzwkFR9sfj7B ztjpzf%{gdnV#B8!@CHDXqvGUB_|g1 ztP4@(eL7CuI4g)(E9v66Z62ci!z{Erv4l4#kH?cQr=b6%m54j{qK3>#QDs5^uN%G& zxqUV!>6Y-xr(@9d(R7|u?@RNI2dhv<;95B(&;H0U+D3oQe>(k$2x{ofZj!WP^dOP8>M47K(l%#og z6r5g`0s*}aaQx>&{vo8mcS%&2htbDN%MRhi-g9|VlR0bTr}EqRn@Q)Y59{V#r<<*Z z(BeQEr$3J74G%Wp*wh!&cb?PmcJvOM9^p=|1~t(1eHC6O0j7R9M+I}Vd8tPzYW|48 z16QLkr}2*T#xy;=*L|U2@cJg`Bzn;J;@Lu-?H&wLY7~zIspGk*4!ZMC3maN?6-(SzboC|80rZ5gae#znGziv^<%3Pd0ItC)l z6w#%1A+2tSp<)vYUe)Eb?hNxG#S7JR{Mlk|?w3yk%G~ipq63@X`$`1q4nW#=isBv)C}0kF7K2dvE;sl|?RHd{&C9pLfkipR=&La5v_Ub;QQdEHt?3 z#qrli(}Q9GJ(i0?(Dkm_BA zotTf2Z`bmW@s*^K*PF8j7GYZ2dzx;i#^dy-!4<=gyWy7D0(N+jLqDE6!z!N?z8!D?Y61_T zj(Dc8CZ?M9=O)gk_!HZ)b%P4F#&*Mhw-@sLnc=wcYBGmvyRw#E9tuUzsrQ&HDBEs= z0s0YaI@O8;_U}N45-(0Uv>#J2hlan-C&}bUO#66%)KcRS@^xWh?|ihCb*6&lXT(eI z+_?Dq0rAbn!C2#+Nc#)CFe~#eJ!$$Qae7z@N5d9ztl@T?=q=&G$sgcf;{@y<@(tdv zFQDF+zQ9h)J-EDRCeDnOQb}t)m~KtPRsJE^601l1A9v%psDtp>HI+`c58|T}6Hx1J zf2p6e5dVAYi77w3vbjypnBV;npYG+4_xC!3$DJ`aWV{L|-%{r5q1*Y&#Z*2RX(5!n zoz5LW>-g(*6~TE+06Qh`=Z?B;c65FT`O`b-*8T^S>AxCnUS{Ib0in3)V={XEx&h~Q z4adj#bJ?Y0HC5M-=A9N(a8221F{Lw^;)HOV&I>r~q5^+2Y@?zZmq7Y^Fe@L<#ZOTY znBlF?7JIs3nCEcbvh@PV?TvUkseb_~yUn1A+;2j)uM)0|SqTecN8kg4 zG;H}1k5PT*;V*}F*!^f4uaYU}j6Tk|#Y>GfmlpEr%U@yDnNWP&SVFB+<8WS+8eUAj zO9{WHparjFv(Z_g@U8$m<(6Z}s_&Atojt&OeL4@x)x-BM3`EsuOSvIO37?1f@h+b< z^8Y;=$$SI9S-*{?-urmdv-hChG!ix(Y!?nr*o|vzRnco`9sKB(K$Q+rbTtR@cDM=6 zdS$`##;K^~p-$^RNNMp#4<20;%C9TrVS3*_T^c+J6Ib=&-^qJmY-Tw2?b^G|iR#ih z_3?PdC!Y)LcB1#YqpM_&w zW~^9y`yfAT*2Wpj{kVHrGc7nW9~Xa~#7E6{aD3M;<;0go=>JU(f4+(39Yt%gV@L$g z?4dxVMZ`u)xS=Z ztCL7c&I55nb=S@;MNU$ya9@NDktTO#AH(Lg!pWTnxktP+OweiBgcgJwsyD4lkb+%Z0BaXv5bU`sA0VZxJ;>EM_!JxWjjWIamTPvM8 zbAZd2#_+eP652Is0{FjKibqFDsqy?~_D*V~X-_wB#p|EcDkFoc*^xN)#A<#olZs|2?~2Dry$ftV zou-*8dCVLA;b_oR=$}w7Q3*)HMXxf@bf6cOT%17P3i{CaUSaq#WC82CCG(@F&%_5# z0{!=}2#2Y5&B8=O8qh4y8W$sRqy07#lMYZnjW7yHokKpfjLzjGadKQ&b~SegH`u=d zV}oH7n&m;=H-?}IXY)`syUOK{3;9{S7j_pG@JGx3ym;9jR?_ax3K!o=wr9t%=8;7h zailv_MJ6WAD8tj=+Ud!}z0&=iW6&UZA#OUGOK)NZ@y(ctxa#~v>YRBHzw0dF)jwnK z9{wTOl_YNM?u(GE!xFdccwXE~t46HA@5d*LA17;Yw46R({ZPn#X4vw&qF&tWWQP$t z(>T7V%Wqdsz|SXlQvB90ko43?tR7zhVI9RBXtM~T=l!K?*5!E5*$fumGNik`LRjy| zZjyx|XzUrs2G6G9uiLl8fCp*ZJ4X|@y-whr1(6uKEQ`VbVQ=z1zw!ey*Z3f!`^7dm10JoXIM2rd@r144OJ-LB3Z3*}e3m|ABAnetka=k{nI)uSv9lTtIhzdztBOgoY(0+}AB1zR zZ%3o@7!s3PpyB%n9v%D-Oww9~Y|C8~Vu5&lQ7peMn#N1&o1xw3i+EMekjrD0Sivj< zlVr|9G$1(Am$3_|voQD2$9oXS&jAZ`muax~@pY&(>dHP!H ziC?5qc(UUoNzc#bH#ZJoP?;Ym9QTFZvp3@KkL9?hy~`W^NaJ1SlTkssnEYoYpi=c1 zY}lKI_UTrj=wOSJ6brbtV7QoY%^%ByHVCuS26M_jJ9a-Yp8d}aK-tV>X|S9;e+}D* zDXLN2Hb;_#nm=;@_5wD@Wx7&;j)86r1ek4kmJTcm zgyR=`@$!i#_YWIlh$FGjNr33_$y^6va8WWT0CJoxq+Jl@|G&wL#Z^(VV_aO!v9(5rmu z$zuX!eskBKPeFV{oc%V`AOK@Lm7?M%;tuS&DfbWmV-k)*f?bwWd(F;T}KmoSy6%;?v&ua zC4>3Uf^hU3nhkrs?0E03U3~WbYdEyxBu(Fw$PJCZVPextZjbNA7b|x1=UW@7LAM4j zOSY4>(|&B7Z^-$(a#8+CFaFqjxb&=pE2np)U|WK}>8HA%4CDUT0*$i}35 zM`&3?501My7SC_(g^w=u!B?x(xMBAwT(Lb8i;dQ!=0HV0w5bfIdJhzLBzdCxk_`N{ z){$onehRNvc-ZG?{DD>b61dfL6d$+^TrjJZ+QWy!4oh8Jex?($@0U?pK@sWD^@|vYLL3Ze5PW1Irz7gtG(sJo!W2m&~E`JxEENBWcr) zY&fnj!N8WmctWo`F7Nst&zdAo+wKkWk&&1w<1cLe-A-=0b7|u47395h0BcX4uFT)g*c2bybg%TG>zS&-ePGjRq?Pn4XUAmneY6j!P zng^t@HJ{TbrFQL~%0#2z>!|E$Bt)$HC!VZd!#!H^!CA?Jdle~QUAZo9^*%~l&32)p z7|&aCrJ#Hvj5Vysv+_X?ocL-I&pLJqu3JR$$=~VN?iLOMrLSVIA@=yUA{-=LtjB{I z9UNP=ixWyqvDIZg?wmLjuiPBWW7KER>C7LJpwv;QI%YA~-^}8n%DYj$#SIn9gSqYM z6B=JX4Ff)C;YXwKaL;W$PN=(ZX5ii;c;lzeu5U{4KjR|$>6?lbexrDNaVRf0wq~8R zx+uD@RN7MtIa!j;N~%SJiQryD0P6=txRvEZ^0b8A#H)o6Q>}p*aA1Jj-!F3CVA|gPRm_GNo!UxFMeDv$}d?4)|Q>Z;#Y~# ze4?K8K$WTu4hd)Do`CG%n_{=J0IBf)3hcSiS6tsak2;@vN-ouXrl4<1FyxPb0oC(_ zMA<;_u-QnryL%FMb<~^FH26x-uM*$yx*T2In@Y5P!|zM)#4B^QKtuPg|3iDA7#7q> z8sk=qLB};|#hLN2sAU;=u91(l~WdrEij~b|Xoe6oPe+seB?or<@wp{F32-a?A zXm&=Gz17nWVZg(iVC>%{gtT-8^_@G!^3+%g`>IXG3*Uo$O<&&S9zkbyGa+hSH@3dy z0=9RGY08G3!n84wFtu8qdL4^mjhi0g10PvBsG`DuzU_oC(@a5siag7h*uW`k7xKE2 zE=H}Y5p5=}r6bvSkYrmYbyGS_ix-%}_*=)QcUV6d{NlavH9uN#*S{$I*XyEm?EFj$ zIx&C;=oWzg#s2uFLYI2ESBTA_+O%WWX|XCe1=>a&f`~sFGWLc7;))V?|qF3o;pv^W*jz~HNzoKoAuF;^%JQ#6q5ZFNyoU}YiW%qW$mc9?E zz;`i?nI_Bg23``*AN(s0(pJNRjRr7koGusX+ryD#>X3e5iueR$#g)lXG^8#H+FfUn z_n^Vzzo)OoV@*Fv+i*POwl>2{(;R5HyPwKir^Cm_9?&x`0xl%I6`x@wY|X8d1czt} zqg|4~uQ-SJqY3@wQ;jv9Frx^Mf49{o9>QxKXUHUI};J z*ukfHo9N0#5ys*`=q3A8uuL_EZ0+t`Ac;csi=y z1KL!kh@YZL#cp$+5lY^W?}Y2(>B)D&^k3KJ>+%z_w)dyRnb~w*zg2ReuLgZGje-9* zb<#&Frwb!mX@93FJdV(#oO$=a%`J5F<_BX*Jwpzx{{9CFCrSEqn-5LUSxt&*!{KD& zDM|I)bArkG0!XR5OjEk+@S$nNWII8Q+Yicc*wHcMb)!`H`Rf2(*7-=MGW047EY8rD z!fJ8p#AZR-IF0OHjG$q+I>_^7FOD~PAtc;a|aTK{hS6@m7rHP+o#kHHai@LQZz_DxEz_4rLO__1;IfsW z3#ZMKWl_dgUwCoovC!?|8{z7dLcycj6Ye!6z};1Yy2uzAKIm^r{b~*A@2yj0Tz5}& z-#HL^x*ijCPTi(^Im3Cw+*hD-@x8cmO9Aw&d_u$D%kb0FN^tl=9BD7#4zKn)Qecm8 z8ncQ=E2!*&akeRfSJ8YBHXkA7VcW&^3f(wKV>Y!Lo5CSof`}Sz`k9qPf8B?|mHyYn zOcxodFkT9e<_6H@p*h0v(_g?RuO1{n#(;_CX}GZBGQ^yH2Qs$jXot=iu%0l5${lhj z^-wK5Y0&`NYk$Rm58`1&)Is65-2*ylA8E%kf3VrM5o+G2kj1wD;6`s{!R&oc zHvAhe{Wsea{H|6(S)b{G&yc0i{_P*Qy|xyuXDp*@DmQ4!f+o>0+maeoU4$2|*F^;O49TINxI}6qVhE=pm1U{yOq>aN}d~aMCnkx!M}CS5Uxu6IHTsi=tC+ ze!|nMb8(ipMi=ogfj5o5NghM5h#fN+rs{;i=f*zL3r%wcGnWLo^>i>io?;9`ZyRvu z=mA2Hy^A2Z)10i+|AV&Zy+T#34P72~iu9bGgKU>d>%Apcyc!%pk#?713!%M zr)4bIdb&cNjQ`-~sR}xp7AU;PkrkHQ?M`WDM^WGc4VdqGg^s-)1sboLL^o{{A^Rmt z!}T@jn{6(sc(+JWluE%z5XI!ldcqwCKg!s@R~+V{0-uYv(){iT!g<$ig58Oautl$p zD)L&yb;%3G(2$W7y+npXs)FI(%$b}Usm={6&4eD0v*7l$V7PNQ9a@h4M_&!>q}pCC zLXSgPw6K4jH0q%vX&IU0oJ|>BJF_t~CAOIQl?^1r3*Hi^YBvx!ULYCcC-8Ww0Zg*L z3kwInqXVbxNC;CB`}S%hMfYsV4G9O;V-q1q-jqqU8lTKHZ)!Ej=ql^jAQO5E=YYwH-zb9ZO+z{n4*(EF2d?;H=_9 z$_YIyZS58cS%cjLjp^5fqgINvuG9oxd56%zB^N=a*_D<#1;F6}ugEffI6Udlz`Nac z)1UZJ0z3{UJ>wFXczY1pd`pIHqcdRe<6eBS&pP36zpf-;)Lz=PEm_zrCxZ)KkA}b( z2}04cHDIl>L;SUDG8kcwbY@CDE#0#a?a=-l*yxVH|BioXJaHo!Xn4End~v>5U-i_AJ5DeUhe zq1)r5KxSj%W90?7HZO*DeD1o0I4^@2%YSuIlug2#>Us#2MAD2SrqDxCD#}|W!81bz za#W8HJ1V5G%)EteygEncB1DK-{#oc)kS3Z*a$x7+bTTUmqQdQ^P}DBEPqPZtF=)pC;e^W! za*I}_4(rQsBQKi9hHBvqCmqnZ5hLFI6(qhsF_-cub%@{kt`=)6HE_)N>7rT77g6az z8=NTB5;A8mq|T-z!rGM0Fr7NWgN_&&hfh&T+=oc_~nlV<&Zi2ZFx535D z%J3pojy`1)2D^uuo&5Q?~{vsW^W*aqB1VZbJ zvy^W%4PJgyVy{Ifg|T;c(kYwIV*B5(qSe(DpnIv1(pC&ZW)^~yc|8oRZx+u~PY^S9 zDYD0lC9qvGkS63p$6t2tlJ?Q^m6VVsvW`9Vsy& z$&fDR*>k(lS3z5xSh-5FY)2|-4L0KEYyZ-v>#r%cScQj8auStZmOw9?%ak_JA54i4xwlow_3Ss-IqZIn(1$vX=&L9KbC-Ziqjc4Taw|;iS`)N|&}f6BvFLYA0&a%&c2N(CsGi!v5oQB!4p9 z5Be^Q%$rBsUp|80z4bA;BulVPej=dyi-4Ca9msc0gn;J5E310B3zvRn3cDT)WLB{aLj2!L zJrCszB^Gh;Q{9Q8pBy9Y^WEw1V^vs3+hJ7JT49Z&A!UR=q}u-5#NBg7f|cujxRCA( zQ`Yz3xqZF~ftABWqsSY=D&HExNvjv;WEpnNSS>NUizYu~+$mnm^%6UuzNK!B8)?hD zaIwX0w>U3vD15Nhgx3icG|8Y|DqjM0^-VmzEngtanH@oeefL$Wd;JsdHGpJ6&P7tm zv7(W48evV`2ugnOih@^Opyt|r63?@mu=D01nq!#(w{1?*bHjSEeX$;&&09_3!HRS= z<~ud4PY~uFamV%YUuj!RF4Q(Xq-Q-WA;91!MHtM3Or34w*>7%qcFuN=J={q0!z|$b z;U?kuzHS(MvPqP^a!d%htw<&BKf&*XY2u*H^A!AXCEUn21o^G%)O@~3n4C1OOP?mg zr*6f<^p&2HnYl+{!Sz#g>Ul62oobal%i9F?k^(3{mjkBuv7%G2UKCfEEe)9SMIzok zMg3Yw^3Lb)gbn>`#5-Qkg(mNJFr)CYB(bY!K3VBO8ofabKjqbRMPLm)d?S5)KNW8F z>3ToC7SX0=HAtCvkxsn7KsIMDR;pVd95-r&*H^k>W19Bp;IQqy)#uCvL5yGLI@$M_kd|Lb1dJ>@j~ z*KtcY)mBdyO@{bhB?j&lq*7-`jyUXfGyL3TjClqLwCJ%VI5mu>Vavusa9Ozh^lm># zAM+Y7v79=N&dW@K1FE52n|4o$YI?GCexC(o<-QQ4J6DQjKEp= zBZmF6rHeR(ju!@i*n1+clc}YT0WV?P_ax!TJ|h_Y`5S#UxG0UennY7>4aMQA-=$?W zTjuia4Na93D7-I1NA$U@tKa$&mFa`BC2IV|rtl*TXlLi*a%g$)nP@cAkOx>9r< zPQPh^M>dwUFV#exV>%zEZY+l+c7x2D)#AnMjT6~vj-c* zZoyxP@}tCm%6}_24`h*NsB)-a!V`U5;XkW%h*0WJ4flbtn**G@<3me=2hiij*HYI2 zHE4hDEELRmAzZoAPB$04>LQ1((QQ35!Qe{^Z8UmCr6U~SO~O71bbSE_92AJ%dq@pc zQ|P~+N8!CM2%o;5C+`=Rsm^gES5IFK@aGUnFQ{}~cvzFpO>fw`s!sgW<1DqOKZ6jD zebUY@g7Q=IYP#flo%$MFgKe1?g)urku{Lm@5M6SHOdel>z0S)d!@sHtXWqzlU7vN4 zSit;i9>R*$5$eTt`~}*Ou-j>4=G8pW#iJC&V|shbyJ;Apd9&!TaAS!HAm0 zC*yxmpF4Y+z|qL84J2{SGqElaM;vE{=ax1N;BrJCcmdAVZq;BV(NS=o?Lhn zCf+O+?Ua{`t|P9BA39>iFmF|OY9OIn-KRp>g#{cRc^CSB)Pce=!^9U&6UB4uG)O*Y zG`x21Bf1^&6%y?tshdR~yytfhj@_*g#yELXs;&myxjBR9E>S?8rCRXmb5HU)8AJD* z_EV^06X?Y#@YOeeLEBn|?G+ZFgM7C)(>3*6C- zMopvIVa~rZg!OB>lG^U{(ZWp#u3Sf17rkhR(@v-ieh+({xCve)|ksvwM+g{aes| zEidsJpJ`urXBB174-qBTcS}~R{;!L8|3eSW_R;BH?POnmQQ{t2M3VAqs_COm`3@f> zVG*kQ`s+9&xXu^MAc8aAB-$BzgWuYzh zEFAb;2_>&bNZY>Oh7y@h;q#Dc`aO9fWGc(rkDGB*ywud25}jAV`uuftvfmBRJ$V$W z+v4HTfsxd7P!Zlf8friH?OW0FOA#$P_noxA=TSmv7jfQl1mY}%!Tg+xsMx~<Eq#Sc0Lt$e|ZZLTMmgWd=omJ_9T~MQG!L@aR`BU;du02@Hfl_(Z`-b564kA^;=Z1 zW)k$0^u~Fgr%{vL9-66k03;3rK&|g=_&%W&l6H2@cAHWdp!G|9(RINy&T18u$qg4j z%?^R2W*z$7<^wAemPx#S5G@n;icMzgM5FsFVNh)?G^{=)UH2dof{dJC`jt0={=%W; zd47lRKSiARLrz~8#Z?-W29*ZVfJ90v>N)$8kf@LlLMl{b2$fPaD5Xh*p;VeBqEgax z_AL#PLM2q98TnF%1}XABzrFv#{o&km_Fn6=){;;1OqQ7tTDAIA1fx+x@TvD?JZ=Mii%5oD)v8EdnT0x0eq&Nw4 zSA*M2>q*@0Z-ib*peBhTjFQcBNXb8s5du|&?Nvs*Y8m#1Rmn`b|c^L_6;so++G)!t;8G^gt$h);E)ra$TI*#mchMA33$DA18h%n-MSE^I>E7f> z1w^1wP!c)3g~N4SBgQzS-RHn838r$H2_&-_I6JS2TU%65lgz@Av>ijk!hBAA=w{q= zP?vG|>jY2Nx8a?D7>tE2AQa<8TRDTA3;`8T@H+;L7p|e$(N_Gp$e0-qNXL&sTu}Y{ zkSfoUU^Xr>#u7Cvv>FSiKbtHIkH_~y zqVOYMn6=0YfcyJXQKRN3y)EWX?`htk!Sa`h!oyAMnj{gXa!`Uv8kc9@eF(q?7dc#! zn?~Koa~kK z#bMiTxMhVq!YgZV{V#?`cmKl^UWw?Kc$?E)kcUGBmgIxYC*W=?;*x`Y7~YtV`bnBt z@SuY}Xn#PGT86Rh=1iEM847id>+z`nIf}1(V0-r*uqgciF`BpOD<%U9bv5zdRx$S5 zCL>PD%6(Wx)!+e~!TC8^8t7i$iiVTpobJ7H?CbD{^uI@ibe_pGboOz@{VoG^ z<{@d;)GnShYtCSH2WhjvT4uoUyNzhOvy<$0dw`i5dALD%1}ot#!b)9?M_wQl6@J{n z#|_5#lm9bjeBfh8zFtIG>nXf*7YDCvhhV*v2=z9^CIv0caVmH_tp-0^92e55sFge&C#JNlZaqZ*z+_hGfnE7WlDNCD) zew!Vjf7A||wWi~^|0J3jSuxtj-hs(SIriVqU_6`~hh~qZ!L~aM-~4ePenYK9@^3!o zY5c<rY1`Pk7b3HZVo)l=C{R_rgk2}>r(s- zx7>)vUXLJNSLQSPkJ3SQqdV4R2t)bMGCHeo4eKLPPC5=1Vy~bz1|CiU(^I=>&g>NE zv(aXi-J{^flq+zG^9?7&Es2NE7%5XRfwm9JNlBeJ8(LY3(!PD%t_v5*e8+0~-yWXV z;wB9VSLV=zLIo(*F^}ijxPwxtHGE2oC6dBc^u>;LB50&TruPYOS>GZ&uT+LlL#yx# zqlY@?e=tz)4u+Q&!MB;W@x5LnCfRx8fkO

)3)ve(U0l@)s~VRg!(PLyY;2wz0%v5C_{#5zC0=@rk3&)INnzQtigM+)#=yafiSf58)5uHe%~ zAvT7mONShJj(3dI_K1}S^ zXI$hIh5Ul5Y@O|wAv zQVH5Sucw&Qd8lvNh0iZN!+@PD zfKwa_XX?XnB32M0vYUy)dn=~sW;H$$=|}lP>u9G%1}3HSLp9R~wWnnvE4mRLW#5E~ zsLR+FJWMi7O2I19hpZ6#L+^}kBnP?>n_WUk(#dF=pw^B1_bnwhbrPuD=YWbPnrL6l z&;Bd5VI2mtv9|OW5!cefxr{8+&hHP^d|jj<+z{)>xyV;-$OzX8Gc%S9(2xP%yP6S; zWBvk+t&lCYNh!hAOWClo>kTGH39)iZCGf=dE9Bt9OTalB3PaNKantT85S_zodvI?j zqmKn)dT}8{pMC%fTWvTBJdgd043A`Y4q%tOdc(c_;ymmRx&noj7s-jsS;z_cK!so_ z$xV!a;SEJlAxpS}=X_9^$7h?J(Yd8>JogI9nHP^sfQeRUWeYY7Mug^S-5k7 z2m}`K_68%sruYL4UEDzel+#fvc{a=aN*Jx?8#pMY%#>Zq;mn(}4|h);CE=H)QE|f^ zbod#=skQb%muDYAuRNFho909pe7y(vJ(Zv;NdTRW8^dL#8uF)H6io%S!PogJ-F-WS zywR$I!AvdoSJx5L{%pELEwCRS*(Jh2cRS{*o&tr-`5ebV6Ff1pn)VMp#+Gf~czdz~ zJ-v?OdLs(sdVw(TbevlDbK%6rD9o{o13lX%+#)9~8jh@nzQ?9y(a2Zmw6tQE%t*#_ zQqnl$u8or`C+Lu&Hq`wIL9=Pj&?2nJW}BZvx5QcO9wm9WwM`fHp8X8gCgNao-5f{R zailhqShxI$aooaG+;VR^{#xkD;Ax1aS{hn6pJ&^MOoJ{qHSv8g1#&JYtn&qrcm3p-Xd(&vW-v2T|&gw0B1E;bckC21CS9xfKn!0z}3>>=w# z5G;L-9u8Z|&QN?x4jj{Al!|+3g5pf>7wu2zbFYXDY!!xmK}JTZ^;4PDHNm(uEf;ZGp$6g<9mswi3U_H87CRF zqS!AWja`@5Vx!^R4ccuc{XsJk%#v>6(> z^4NBtURv5Vm(F>=8IJl*XZ@x=;V~SyQ1Y+@E1dQf--tIF2VNP&kHu9`n0*l1T27Kh|=dO$zDbU}oe?N7x10%_P}@`N6ma3|~cn(_t#X;#p!h!`v=MbG{%sQWz+ zb$^RN)LMR~;&dhM@#e84acl4x&kvEXzXZFP0)QwJ%ov~;qLPJ@s_xX%*bt`|hSH~v zg3Ke)dl;@5NI%5hN10B2=(!a_IqA(Laaj{p?K5JSyc{~EMU!<<3dOeIMa;k1H}Me6 zU@x@%q1j89pl{J8h|N-ACH4lRD=#VEpq@zBosC|$UgJ8*E{8?+ zqV&?VCL~I4@Me+&8#!7ALm5JZ{E%lCKHmaiYgpQnv4&Pwd%?t5JNy)QNup9aiALQl z_EtC_4t|;f4f5Y%G1G?+ozn2uSSu~Gc>;+{6GlC5qq-}@z*$I^aqqLG@42ZcGV>mJ zQuvr-vv?NwKaWA0zWy8x@&}U?l~Q~(u?2q0W}wlczr^$FDJ&Jwp!>3}fwr;;5u2j{ zo4gaSM{6}{dmxJ+e~-iVGCva1z7d0;#nHm)Kj~7n4#<42N+Z3`qmV%ZY8_1_!;cGz z-&SD1+o?gM$Zaq;sDE=_4l)Gk-HE(@)5SR zi?hoTt#Mh`8(LAY3q|iGKvqx8@=fan*~6+)Y*AVsQC{?#1a!;d2ODwrX01A;CMY7C zm`K%5dypj?|3g1NWo&!fh@r=i)2SPi$f&;>9KYj-;3meLapFVO>}zygMF^RumI&nC zVi>V_fmJi4sCR%f^Ivuu)i|>lf7sSz=9EHFLQS<(Pr+SP{oKqMfTgf>1f!Z$2@=fnmlN5 z$HbE^+~6Ip$Zpi5jxrBP)POu{tDV4%yi2sV=nmen`AJ55YGA%$1kpJg4wW11So>;O zcECHC#OBUnzlGLNXE{f(550#Mx72c1+Q@Uy&s%`&({^LYXD?{@xgT$F`M`SvLWO-B zEev@A$|gOiYh(tx8Lja6NeJwWDaTRGjc~nO6|V}r5O2@x_^G)HLuLrE>gUGE)Nk8B z@6iHM5tK*%wqC-u&8IQ==4%rDd=_t(6=aRS@}b1HXew48gF1a`tc14?%YS$m6W{-k zT91jtr8FB{-us&)cd;2`?#4s?iyRoP<=y+o#c@$)8})>p9RIaHKq=mcSzxdmx18C6 zCaYUXWXM`h>$g0JnW)9;ik;}{KMj-5N;4Pa2Ow^27mE12z>>@iJa&?j*;!{`hVxFy zF)io$Tz=a}?PeV@s z6p-1d!l-yQf~$ckQ=GRB?j5=fiM!W8z3y`K=zM|F{pzgJTYuP;JBcG>_9*xL7sr^F zux2+a)0f}Afd(%*Y@wBooQ_Ed;fCOT0~~nICvI$ z1&yPW;haw(C)vjfTa$W0aHk04ou zSJbvX2YoK)=v&x<7Im#Ko;ATSvm4=#$({%KMWyh?AOs((_9V<=OA#9SN+1f->>)9(~%i@ zATgv)W=Dy@NBtbk`6H9dA|6tgfR9`>p6Vy%%Npy3Y$28E(UB&2gB+PI&83% zwdXfrd$)&RlgtFZu!zL+6j4?tKak2iI>#jMw#MJz=CE5gtYd=?6mjPnf5MDN3k)wXCP&_s9G@G_oaKIkxV9UFol;Oh%h81C+0+=T<% zPJk2ZW>`VGWY_T1DIhaYnI`vFjwT| znB_{+q;@yJ+8Q~c@Ms>6)GP+)=z836Zw4Ihn@YQ#Ae>_;$Fhz>kxrE~q-lJAcVKyq2H2ILW23Vfi6?ej)*Km&oAAvIZ(I z{2z%Knt_7d2q(A}WR=(pQu9NYjWThflYi!e=IUQyBUwiu?^S~_vYnSk*-HKk*M>_S zKWGv&mre|>peiSLzEa#c$2-%N=w?0w?=#L!Xz^lDjH^aIvEL|l=L9#WbqN1PE@fYs z6q75P;^?nzUpO#L4{duKaLU;yw7;PSm2;JNuk|f5r%o4NDs`3aH*Li89>KWxy)wG( z5QWk8o7rc5`ylQ3G_0t$A_tu>V7}Hd&ZeO-yk9LydT2P7%Q&(s)(vE^Ac?$PXh1Y2 z4RO;Uz)xWp@LWP7)$%Gp`$Pj8&>KmVc+HA?w2xx2Ry)KjpG2#btDz~X3>SSGA$ASZ z=w|CZcwGQd?Nkqq`kBnzLz^Lgiv+VUBJ>ATZPO~@kjSYgg~G-*M_jVoBTvV$JoXpiyvZ$UA_ z0z-EdFhdj~&4YWG~Wu>TUi)ZYkx8ZM0Eq&VuOT_F~S z@?hs&K{zNX&vx);-(wfY zX_A0HFsjB-^-CDM(-m<}j0mnB7v!}h{UTQlb%BRSf{WU0d_UWgRCa~Y*$YmOnPDa)nE8; ztd-PlPsQJ-0;xk!I=r9p5LO?GfanX9%Px~;1Iw=xgW6e)OG-5QaPM>YOm9M-eGHhE zUO@TzGvVp!d!Xg20fy;n7#9}7h+NzNhm{4H{qqV*x!?>4EnR`#bC2TbhyyUEZ8ox# zp|J3p9AYQL9=|yS*^enNp%oUgZb>F?RUtp%7G>22`INJ8ji<)0L3*) z_}}+V?qs?nd8pn1qHf}-ed{^SsC!M+4qt~^<1%>YJ}-?rOM$6o5dcO>?fIg3)!&a3yCxTDN^H7KyEeN zgKZxN;Esbd^W(@Y+~Yn$N(w@tWziiHlo>#d@y_Gfo=5ac$~6qV)<~z@TamZf^Xb?z zGjdQuf%qPez~w;`5FVNfvU{q@)Y7$BQm~9@CsI`VEQ@Ye?~zsY`#6bfb2z)~d2NE# z;ZQC-hZ@%1r5tf-j0&BLo~M(@@Gd3x*RVf4=}`tR@p3R48iESXOFUl25!%iiqveNx zL&V`PRQQ}X9`P84Q$LSD^Pd4&C%=fA1gde%i2$_vBML5-qLmj*!F<&Pm|w9R-OP94 zmaPj=@mvYa+^Wv=G*`h2?<<_rh9F$IOcd-U#8}s=9&qUT4;A_k0iTK>-Btb+PW%p~ z(~r5aS{5Sks$T(Z^EMEJiJL%k%S(_*GJtcdY#?xL963GNjdSY#@sy7hDEX#S z2~H3>)piRume*jHkRmwg%z?ybpUL<4AHY6Z0pd^Jrp2d4n7sS%!EeKEET8uc-JQi) zNOn-`CYYzXhZ*+PLB(5x@M-gECM;kot8;b=a1X78KSfDck)nitlcDJGS`&=a z?LZ~^G7j05!+&GO5SOk^f1Q`d)Sr^fj&mU>@_c}<$Nt zPoer(OIQ(o7S|`tpxqh8=*1sT4Q@tLJz*nkEK`DUp8=eudl*yF`KaaOW|G%k0{333 z^8S-r%>JlOB&>AVJ7WWKXP6wqg#x*$U0~+wo-4caSU%!u`D`fgv-Q$W&P- zLc5eYs88pm%#*RS>;hbzFA02YiKMdkA_OevhpV?F*jvX!Xx5LTaQW0d$ov&XM*qEq z-Yj)y(&7kKUsGeAZ7Cs5r>CHD`c6XqL`-@3{8R%{T$T}9MfxL zJUxn&t@jxsW_^W<(>Zv=`Yv}luZ?a!@YtL#1;+gVA3HbtDYgH%l=^$k165UN7;6y1 z-Zd>maccpt{x=W2d^)fp;3r8ak7uHMwqWgnV;qezE^a;~fNy!!ziGP=b2}mk`5q-; zz`qd?bjqf2UtZ#=%bHAmdnTT0T!t;n_QHR60%1*jBi-@#F*@FRi^VTZXqc1($nQ`F z*7YYINJzjpMy4QDHAwOnin8wZ8_3wTK_YWSib-A<3Gz>-;GDZwRQ#|YJW4TURM)6N z@R`4`CfjLVSWK0w+6qy=L0blA$Eyg0_aX5U}dZdpStegnP*yz)!lly9nxS-pD)Hy z9~tysy9(lxl-OE+acnxc8>)_narUZEJgLpkv~Sd4Pd>W|Rj7dT79K^TCqbaT#UJZS zC(o|NWrTcv-X79aX%B_8lB5z71Mrdhv>)0#m(=i(lZcWK)gD<`y4Mb;^7J78x|!@QbrCGt*N)cXo9J}W3(%Guf!Af8lP*pe9O_Pj zcE{zAm2H5l9Twr0=+B@%c#8_07eNc%IrPD(8`VlsXHNWH042@g?1j2ivSPL+JRVvB zTS1Psm~s)ETOA-q^&MfKM!_7*->Cagie+-w!ZYzHyzFZ-?aWU^vva)Nu(FOuCES27 z%X83Ib|3Zb{RQqvmY~$$TSR$sBelA*6Y};<1HCSDvS)D?b?nX~JJY|T-VH6*nF#~y zzZC02K6AcG9|P&jjqo+An%eLD$;ma-#mhOjaeys=V{`AL-sTGKu?w9>VJj983%4{l zKKhC`i}u4x%^xUAZ-C_WTVVFNj4M)f99%wDgW=>${Mk5@#4<{RJ)8wA!@qNeEd?37 z8N0ZXVP~kxwtTw$WI1MS*vxBtn}RQeEO6q7EFGd&xII`8PJea;@fV?R?szWjP4U3v z5tiK5+metoU5wf6!_QPV=z|d_6b<;d<9W_uR9gHF^fpU^coCc}63mmP{1w!#2@Ec=Nh} z>%HCvmM@b67R_ z_&4P)R3wSA`!o)t&h!G<;k1LJ8$XMw<+b9Coi&FK9hqGJye;T$5yG{39!W-Q!{Bl{ zujJQ%Es~`>7^!tVpfh(avnas><=(gw3760G_3bra=@Sk=f9S9>?s5?FN)60=ZbDX= zCt6uP24^`QQ-5X;5ws1(@b#T!%}q1(_Jc)oTQOyxRYw9O-G;OA*<^5rT$^63$HJXwM-EfttExqiq$k%X5G8$s0J5eS<} zkK7`cIW4^>Z#{U6*G(+|P From a0b4e71d456c549a9c8fbcf1e13ffaca3636f87f Mon Sep 17 00:00:00 2001 From: cuicui Date: Mon, 16 Oct 2023 15:33:01 +0800 Subject: [PATCH 3/4] add mindspore version of fastvit model --- mindcv/models/fastvit.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/mindcv/models/fastvit.py b/mindcv/models/fastvit.py index 43fd1f9d4..77b0fddcc 100644 --- a/mindcv/models/fastvit.py +++ b/mindcv/models/fastvit.py @@ -1,18 +1,20 @@ """Reference:https://github.com/apple/ml-fastvit""" +import copy import math +import os +from collections import OrderedDict +from functools import partial +from typing import List, Optional, Tuple, Union + +import numpy as np + import mindspore as ms -from mindspore import ops,nn -from mindspore.numpy import ones import mindspore.common.initializer as init -import numpy as np -from mindcv.models.registry import register_model -from mindcv.models.layers.pooling import GlobalAvgPooling +from mindspore import nn, ops +from mindspore.numpy import ones -from typing import Union,Tuple,Optional,List -import copy -from collections import OrderedDict -import os -from functools import partial +from mindcv.models.layers.pooling import GlobalAvgPooling +from mindcv.models.registry import register_model IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD = (0.485,0.456,0.406),(0.229,0.224,0.225) @@ -135,7 +137,7 @@ def construct(self, x: ms.Tensor) -> ms.Tensor: .reshape((B, N, 3, self.num_heads, self.head_dim)) .permute(2, 0, 3, 1, 4) ) - q, k, v = ops.Unstack(axis=0)(qkv) + q, k, v = ops.Unstack(axis=0)(qkv) # trick here to make q@k.t more stable attn = self.batch_matmul(q*self.scale,k.transpose(0,1,-1,-2)) @@ -1644,4 +1646,3 @@ def construct(self, inputs, outputs, labels): loss = base_loss * (1 - self.alpha) + distillation_loss * self.alpha return loss - From 4af3e308e0e07ceea8c8a51c99896ccf77f5fe14 Mon Sep 17 00:00:00 2001 From: cuicui Date: Fri, 20 Oct 2023 17:53:48 +0800 Subject: [PATCH 4/4] Add mindspore version of fastvit model --- mindcv/models/fastvit.py | 141 ++++++++++++++++++++------------------- 1 file changed, 72 insertions(+), 69 deletions(-) diff --git a/mindcv/models/fastvit.py b/mindcv/models/fastvit.py index 77b0fddcc..9994f2c30 100644 --- a/mindcv/models/fastvit.py +++ b/mindcv/models/fastvit.py @@ -6,8 +6,6 @@ from functools import partial from typing import List, Optional, Tuple, Union -import numpy as np - import mindspore as ms import mindspore.common.initializer as init from mindspore import nn, ops @@ -16,7 +14,8 @@ from mindcv.models.layers.pooling import GlobalAvgPooling from mindcv.models.registry import register_model -IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD = (0.485,0.456,0.406),(0.229,0.224,0.225) +IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD = (0.485, 0.456, 0.406), (0.229, 0.224, 0.225) + def _cfg(url="", **kwargs): return { @@ -114,7 +113,7 @@ def __init__( attn_drop: Dropout rate for attention tensor. proj_drop: Dropout rate for projection tensor. """ - super(MHSA,self).__init__() + super(MHSA, self).__init__() assert dim % head_dim == 0, "dim should be divisible by head_dim" self.head_dim = head_dim self.num_heads = dim // head_dim @@ -131,7 +130,7 @@ def construct(self, x: ms.Tensor) -> ms.Tensor: B, C, H, W = shape N = H * W if len(shape) == 4: - x = nn.flatten(x, start_dim=2).transpose((0,-1,-2)) # (B, N, C) + x = nn.flatten(x, start_dim=2).transpose((0, -1, -2)) # (B, N, C) qkv = ( self.qkv(x) .reshape((B, N, 3, self.num_heads, self.head_dim)) @@ -140,15 +139,15 @@ def construct(self, x: ms.Tensor) -> ms.Tensor: q, k, v = ops.Unstack(axis=0)(qkv) # trick here to make q@k.t more stable - attn = self.batch_matmul(q*self.scale,k.transpose(0,1,-1,-2)) + attn = self.batch_matmul(q*self.scale, k.transpose(0, 1, -1, -2)) attn = nn.Softmax(axis=-1)(attn) attn = self.attn_drop(attn) - x = (attn @ v).transpose((0,2,1,-1)).reshape((B, N, C)) + x = (attn @ v).transpose((0, 2, 1, -1)).reshape((B, N, C)) x = self.proj(x) x = self.proj_drop(x) if len(shape) == 4: - x = x.transpose((0,-1,-2)).reshape(B, C, H, W) + x = x.transpose((0, -1, -2)).reshape(B, C, H, W) return x @@ -210,7 +209,7 @@ class RepMixer(nn.Cell): """Reparameterizable token mixer. For more details, please refer to our paper: - `FastViT: A Fast Hybrid Vision Transformer using Structural Reparameterization `_ + FastViT: A Fast Hybrid Vision Transformer using Structural Reparameterization """ def __init__( @@ -269,7 +268,7 @@ def __init__( self.use_layer_scale = use_layer_scale if use_layer_scale: self.layer_scale = ms.Parameter( - layer_scale_init_value * ops.ones((dim, 1, 1),ms.float32), name = 'w',requires_grad=True + layer_scale_init_value * ops.ones((dim, 1, 1), ms.float32), name='w', requires_grad=True ) def construct(self, x: ms.Tensor) -> ms.Tensor: @@ -294,7 +293,7 @@ def reparameterize(self) -> None: self.norm.reparameterize() if self.use_layer_scale: - w = self.mixer.id_tensor + ops.ExpandDims()(self.layer_scale,-1) * ( + w = self.mixer.id_tensor + ops.ExpandDims()(self.layer_scale, -1) * ( self.mixer.reparam_conv.weight - self.norm.reparam_conv.weight ) b = ops.Squeeze()(self.layer_scale) * ( @@ -354,18 +353,14 @@ def __init__( hidden_channels = hidden_channels or in_channels self.conv = nn.SequentialCell( OrderedDict( - [("conv", - nn.Conv2d( - in_channels=in_channels, - out_channels=out_channels, - kernel_size=7, - pad_mode = 'pad', - padding=3, - group=in_channels, - has_bias=False, - )),("bn", nn.BatchNorm2d(num_features=out_channels))] - ) - ) + [("conv", nn.Conv2d(in_channels=in_channels, + out_channels=out_channels, + kernel_size=7, + pad_mode='pad', + padding=3, + group=in_channels, + has_bias=False,)), + ("bn", nn.BatchNorm2d(num_features=out_channels))])) self.fc1 = nn.Conv2d(in_channels, hidden_channels, kernel_size=1) self.act = act_layer() self.fc2 = nn.Conv2d(hidden_channels, out_channels, kernel_size=1) @@ -373,11 +368,14 @@ def __init__( self._init_weights() def _init_weights(self) -> None: - for _,cell in self.cells_and_names(): - if isinstance(cell,nn.Conv2d): - cell.weight.set_data(init.initializer(init.TruncatedNormal(sigma=0.02),cell.weight.shape,cell.weight.dtype)) + for _, cell in self.cells_and_names(): + if isinstance(cell, nn.Conv2d): + cell.weight.set_data(init.initializer(init.TruncatedNormal(sigma=0.02), + cell.weight.shape, + cell.weight.dtype)) if cell.bias is not None: - cell.bias.set_data(init.initializer(init.Zero(),cell.bias.shape,cell.bias.dtype)) + cell.bias.set_data(init.initializer(init.Zero(), cell.bias.shape, cell.bias.dtype)) + def construct(self, x: ms.Tensor) -> ms.Tensor: x = self.conv(x) x = self.fc1(x) @@ -469,7 +467,7 @@ def reparameterize(self) -> None: input_dim, self.spatial_shape[0], self.spatial_shape[1], - ),ms.float32 + ), ms.float32 ) for i in range(self.in_channels): kernel_value[ @@ -564,7 +562,7 @@ def __init__( self.use_layer_scale = use_layer_scale if use_layer_scale: self.layer_scale = ms.Parameter( - layer_scale_init_value * ops.ones((dim, 1, 1),ms.float32), requires_grad=True + layer_scale_init_value * ops.ones((dim, 1, 1), ms.float32), requires_grad=True ) def construct(self, x): @@ -631,10 +629,10 @@ def __init__( self.use_layer_scale = use_layer_scale if use_layer_scale: self.layer_scale_1 = ms.Parameter( - layer_scale_init_value * ops.ones((dim, 1, 1),ms.float32), requires_grad=True + layer_scale_init_value * ops.ones((dim, 1, 1), ms.float32), requires_grad=True ) self.layer_scale_2 = ms.Parameter( - layer_scale_init_value * ops.ones((dim, 1, 1),ms.float32), requires_grad=True + layer_scale_init_value * ops.ones((dim, 1, 1), ms.float32), requires_grad=True ) def construct(self, x): @@ -808,8 +806,6 @@ def __init__( inference_mode=inference_mode, ) ) - - # For segmentation and detection, extract intermediate output if self.fork_feat: # add a norm layer for each output @@ -849,11 +845,13 @@ def __init__( def cls_init_weights(self) -> None: """Init. for classification""" - for _,cell in self.cells_and_names(): - if isinstance(cell,nn.Dense): - cell.weight.set_data(init.initializer(init.TruncatedNormal(sigma=0.02),cell.weight.shape,cell.weight.dtype)) - if isinstance(cell,nn.Dense) and cell.bias is not None: - cell.bias.set_data(init.initializer(init.Zero(),cell.bias.shape,cell.bias.dtype)) + for _, cell in self.cells_and_names(): + if isinstance(cell, nn.Dense): + cell.weight.set_data(init.initializer(init.TruncatedNormal(sigma=0.02), + cell.weight.shape, + cell.weight.dtype)) + if isinstance(cell, nn.Dense) and cell.bias is not None: + cell.bias.set_data(init.initializer(init.Zero(), cell.bias.shape, cell.bias.dtype)) def forward_embeddings(self, x: ms.Tensor) -> ms.Tensor: x = self.patch_embed(x) @@ -1052,6 +1050,7 @@ def fastvit_ma36(pretrained=False, **kwargs): raise ValueError("Functionality not implemented.") return model + class DropPath(nn.Cell): """DropPath (Stochastic Depth) regularization layers""" @@ -1068,12 +1067,13 @@ def __init__( def construct(self, x: ms.Tensor) -> ms.Tensor: if self.keep_prob == 1.0 or not self.training: return x - shape = (x.shape[0],) + (1,) * (x.ndim - 1) + shape = (x.shape[0], ) + (1,) * (x.ndim - 1) random_tensor = self.dropout(ones(shape)) if not self.scale_by_keep: random_tensor = ops.mul(random_tensor, self.keep_prob) return x * random_tensor + class SEBlock(nn.Cell): def __init__(self, in_channels: int, rd_ratio: float = 0.0625) -> None: @@ -1104,7 +1104,7 @@ def __init__(self, in_channels: int, rd_ratio: float = 0.0625) -> None: def construct(self, inputs: ms.Tensor) -> ms.Tensor: """Apply forward pass.""" b, c, h, w = inputs.shape - x = ops.AvgPool(pad_mode='valid',kernel_size=(h,w))(inputs) + x = ops.AvgPool(pad_mode='valid', kernel_size=(h, w))(inputs) x = self.reduce(x) x = nn.ReLU()(x) x = self.expand(x) @@ -1284,7 +1284,7 @@ def _get_kernel_bias(self) -> Tuple[ms.Tensor, ms.Tensor]: kernel_scale, bias_scale = self._fuse_bn_tensor(self.rbr_scale) # Pad scale branch kernel to match conv branch kernel size. pad = self.kernel_size // 2 - pad_op = nn.Pad(paddings=((0,0),(0,0),(pad,pad),(pad,pad))) + pad_op = nn.Pad(paddings=((0, 0), (0, 0), (pad, pad), (pad, pad))) kernel_scale = pad_op(kernel_scale) # get weights and bias of skip branch @@ -1359,21 +1359,20 @@ def _conv_bn(self, kernel_size: int, padding: int) -> nn.SequentialCell: Conv-BN module. """ mod_list = nn.SequentialCell( - OrderedDict([("conv", - nn.Conv2d( - in_channels=self.in_channels, - out_channels=self.out_channels, - kernel_size=kernel_size, - stride=self.stride, - pad_mode='pad', - padding=padding, - group=self.group, - has_bias=False, - )), - ("bn",nn.BatchNorm2d(num_features=self.out_channels))]) - ) + OrderedDict( + [("conv", nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.out_channels, + kernel_size=kernel_size, + stride=self.stride, + pad_mode='pad', + padding=padding, + group=self.group, + has_bias=False, )), + ("bn", nn.BatchNorm2d(num_features=self.out_channels))])) return mod_list + class ReparamLargeKernelConv(nn.Cell): """Building Block of RepLKNet @@ -1466,7 +1465,10 @@ def get_kernel_bias(self) -> Tuple[ms.Tensor, ms.Tensor]: if hasattr(self, "small_conv"): small_k, small_b = self._fuse_bn(self.small_conv.conv, self.small_conv.bn) eq_b += small_b - pad_op = nn.Pad(paddings=((0,0),(0,0),((self.kernel_size - self.small_kernel) // 2,(self.kernel_size - self.small_kernel) // 2),((self.kernel_size - self.small_kernel) // 2,(self.kernel_size - self.small_kernel) // 2))) + pad_op = nn.Pad(paddings=((0, 0), (0, 0), ((self.kernel_size - self.small_kernel) // 2, + (self.kernel_size - self.small_kernel) // 2), + ((self.kernel_size - self.small_kernel) // 2, + (self.kernel_size - self.small_kernel) // 2))) eq_k += pad_op(small_k) return eq_k, eq_b @@ -1531,20 +1533,19 @@ def _conv_bn(self, kernel_size: int, padding: int = 0) -> nn.SequentialCell: """ mod_list = nn.SequentialCell( OrderedDict( - [("conv",nn.Conv2d( - in_channels=self.in_channels, - out_channels=self.out_channels, - kernel_size=kernel_size, - stride=self.stride, - pad_mode = 'pad', - padding=padding, - group=self.group, - has_bias=False, - )), - ("bn", nn.BatchNorm2d(num_features=self.out_channels))]) - ) + [("conv", nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.out_channels, + kernel_size=kernel_size, + stride=self.stride, + pad_mode='pad', + padding=padding, + group=self.group, + has_bias=False,)), + ("bn", nn.BatchNorm2d(num_features=self.out_channels))])) return mod_list + def reparameterize_model(model: nn.Cell) -> nn.Cell: """Method returns a model where a multi-branched structure used in training is re-parameterized into a single branch @@ -1558,11 +1559,12 @@ def reparameterize_model(model: nn.Cell) -> nn.Cell: """ # Avoid editing original graph model = copy.deepcopy(model) - for _,cell in model.cells_and_names(): + for _, cell in model.cells_and_names(): if hasattr(cell, "reparameterize"): cell.reparameterize() return model + class CosineWDSchedule: def __init__(self, optimizer, t_max, eta_min=0, last_epoch=-1): self.last_epoch = last_epoch @@ -1596,6 +1598,7 @@ def update_weight_decay(self, optimizer): if param_group["weight_decay"] > 0.0: param_group["weight_decay"] = wd + class DistillationLoss(nn.Cell): """ This module wraps a standard criterion and adds an extra knowledge distillation loss by @@ -1610,7 +1613,7 @@ def __init__( alpha: float, tau: float, ): - super(DistillationLoss,self).__init__() + super(DistillationLoss, self).__init__() self.base_criterion = base_criterion self.teacher_model = teacher_model assert distillation_type in ["none", "soft", "hard"]