diff --git a/benchmarks/attention/README.md b/benchmarks/attention/README.md new file mode 100644 index 000000000..5341b0bd5 --- /dev/null +++ b/benchmarks/attention/README.md @@ -0,0 +1,14 @@ +## JAX Fused-Attention Benchmarking +The benchmarking process is split into two stages: *generating* the timing data, and *visualizing* the timing data. The following steps assume you are located in `TransformerEngine/benchmarks/attention` (i.e. where this README is located). First, ensure that you install requirements via `pip install -r requirements.txt`. + +### Generate Timing Data +Run the following command to generate timing data. Please use the `-h` flag for details on the available arguments. The output csv, which will later be parsed to generate the interactive visualizations, is generated in the same directory as the script, since that is where the visualization stage expects it. + +```bash +python benchmark_attention_jax.py --bench-bwd --fwd-v3 --bwd-v3 -v +``` + +Note that you can also specify a target HIP device via `HIP_VISIBLE_DEVICES=` which may be useful in isolating the benchmarks to an unused GPU on a shared machine. + +### Generating Interactive Visualization +Simply run `panel serve panel_app.py`. This will launch a web-service on your localhost which displays an interactive visualization app. If launching on a remote server, VS code users will find that their IDE automatically port-forwards the correct ports, and thus they may directly open the link that is printed after running the command. Other users must ensure that their `ssh` into the remote server includes an appropriate port-forwarding (the default port is `5006`). \ No newline at end of file diff --git a/benchmarks/attention/benchmark_attention_jax.py b/benchmarks/attention/benchmark_attention_jax.py new file mode 100644 index 000000000..93e66f526 --- /dev/null +++ b/benchmarks/attention/benchmark_attention_jax.py @@ -0,0 +1,406 @@ +# Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. +# See LICENSE for license information. + +import os, sys +from pathlib import Path +import pandas as pd +import argparse +from functools import partial +from itertools import product +import jax +from jax import numpy as jnp +import csv +from transformer_engine.jax.attention import ( + AttnBiasType, + AttnMaskType, + QKVLayout, +) +from transformer_engine.jax import fp8_autocast + +# Needed in order to dump timings properly +os.environ["XLA_FLAGS"]="--xla_gpu_graph_level=0" + +# Add test_fused_attn to the sys path +tests_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../tests/jax/") +) +sys.path.append(tests_path) + +from test_fused_attn import ( + FusedAttnRunner, + FusedAttnHelper, + SeqDescFormat, + BiasShape, + customcall_fused_dpa, +) + + +# "b, s_q, s_kv, h_q, h_kv, d_qk, d_v," +SHAPES = ((2, 2048, 2048, 12, 12, 64, 64),) + +# data type +DTYPES = [jnp.float16, jnp.bfloat16] + +ATTN_MASK_TYPES = ( + AttnMaskType.NO_MASK, + AttnMaskType.PADDING_MASK, + AttnMaskType.CAUSAL_MASK, + AttnMaskType.PADDING_CAUSAL_MASK, +) +QKV_LAYOUTS = ( + QKVLayout.BS3HD, + QKVLayout.BSHD_BS2HD, + QKVLayout.BSHD_BSHD_BSHD, + QKVLayout.T3HD, + QKVLayout.THD_T2HD, + QKVLayout.THD_THD_THD +) +SEQ_DESC_FORMATS = (SeqDescFormat.Mask, SeqDescFormat.Seqlens, SeqDescFormat.SegmentIDs) +SWA = (True, False) +IS_TRAINING = (True, False) +DROPOUT = (0.0, 0.1) +BIAS_CONFIGS = ((AttnBiasType.NO_BIAS, None), (AttnBiasType.POST_SCALE_BIAS, BiasShape._1HSS)) +CONFIGS = tuple( + product( + SHAPES, + DTYPES, + ATTN_MASK_TYPES, + QKV_LAYOUTS, + SEQ_DESC_FORMATS, + SWA, + IS_TRAINING, + DROPOUT, + BIAS_CONFIGS, + ) +) + +COLUMNS = [ + "batch_size", + "q_seq_len", + "kv_seq_len", + "q_heads", + "kv_heads", + "qk_dim", + "v_dim", + "attn_bias_type", + "attn_mask_type", + "dropout", + "dtype", + "is_training", + "qkv_layout", + "bias_shape", + "swa", + "seq_desc_format", + "mode", + "time", +] + +CWD = os.getcwd() + +class FusedAttnBenchRunner(FusedAttnRunner): + def bench_forward(self, warmup, iters, timings_dir): + """ + Run forward + """ + self._setup_inputs() + customcall_args = [ + jax.device_put(self.cp_reorder_fn(self.q), self.qkvo_sharding), + jax.device_put(self.cp_reorder_fn(self.k), self.qkvo_sharding), + jax.device_put(self.cp_reorder_fn(self.v), self.qkvo_sharding), + jax.device_put(self.bias, self.bias_sharding), + jax.device_put(self.sequence_desciptor, self.seq_desc_sharding), + jax.device_put(self.dropout_rng, self.dropout_rng_sharding), + ] + kwargs = { + "attn_bias_type": self.attn_bias_type, + "attn_mask_type": self.attn_mask_type, + "scaling_factor": self.scaling_factor, + "dropout_probability": self.dropout_prob, + "is_training": self.is_training, + "qkv_layout": self.qkv_layout, + "max_segments_per_seq": self._get_max_segments_per_sequence(), + "window_size": self.window_size, + "context_parallel_strategy": self.cp_strategy, + "context_parallel_causal_load_balanced": self.cp_load_balanced, + } + + customcall_fused_dpa_jit = jax.jit( + partial(customcall_fused_dpa, **kwargs), + static_argnames=kwargs.keys(), + in_shardings=[ + self.qkvo_sharding, + self.qkvo_sharding, + self.qkvo_sharding, + self.bias_sharding, + self.seq_desc_sharding, + self.dropout_rng_sharding, + ], + ) + with self.mesh, fp8_autocast(mesh_resource=self.mesh_resource): + for _ in range(warmup): + customcall_fused_dpa_jit(*customcall_args) + + os.environ["NVTE_DUMP_AITER_RT"] = str(timings_dir) + '/' + + for _ in range(iters): + customcall_fused_dpa_jit(*customcall_args) + + del os.environ["NVTE_DUMP_AITER_RT"] + + def bench_backward(self, warmup, iters, timings_dir): + """ + Run value_and_grad with JIT, which includes both forward and backward. + """ + self._setup_inputs() + + def grad_func(func, *args, cp_reverse_out=False, **kwargs): + # Gradient is small, use a gradient multiplier to amplify the gradient + gradient_multiplier = self.max_seqlen_q * self.num_heads_q + if self.attn_mask_type.is_causal(): + gradient_multiplier /= 10 + # Keep only valid result for the gradient + if not cp_reverse_out: + ret_valid = jnp.where( + self.pad_q[..., jnp.newaxis, jnp.newaxis], + 0, + func(*args, **kwargs), + ) + else: + ret_valid = jnp.where( + self.pad_q[..., jnp.newaxis, jnp.newaxis], + 0, + self.cp_inverse_reorder_fn(func(*args, **kwargs)), + ) + return ( + jnp.mean(ret_valid.astype(jnp.float32), dtype=jnp.float32) * gradient_multiplier + ).astype(self.dtype) + + customcall_args = [ + jax.device_put(self.cp_reorder_fn(self.q), self.qkvo_sharding), + jax.device_put(self.cp_reorder_fn(self.k), self.qkvo_sharding), + jax.device_put(self.cp_reorder_fn(self.v), self.qkvo_sharding), + jax.device_put(self.bias, self.bias_sharding), + jax.device_put(self.sequence_desciptor, self.seq_desc_sharding), + jax.device_put(self.dropout_rng, self.dropout_rng_sharding), + ] + kwargs = { + "attn_bias_type": self.attn_bias_type, + "attn_mask_type": self.attn_mask_type, + "scaling_factor": self.scaling_factor, + "dropout_probability": self.dropout_prob, + "is_training": self.is_training, + "qkv_layout": self.qkv_layout, + "max_segments_per_seq": self._get_max_segments_per_sequence(), + "window_size": self.window_size, + "context_parallel_strategy": self.cp_strategy, + "context_parallel_causal_load_balanced": self.cp_load_balanced, + } + + # We can compute dBias only for the [1, h, s, s] layout + if self.bias_shape == BiasShape._1HSS: + arg_nums = (0, 1, 2, 3) + grad_shardings = ( + self.qkvo_sharding, + self.qkvo_sharding, + self.qkvo_sharding, + self.bias_sharding, + ) + else: + arg_nums = (0, 1, 2) + grad_shardings = (self.qkvo_sharding, self.qkvo_sharding, self.qkvo_sharding) + + # Use FP16/BF16 to sum the results may cause overflow, use FP32 for the summation + jitted_primitive = jax.jit( + jax.value_and_grad( + lambda q, k, v, bias, *args: grad_func( + customcall_fused_dpa, q, k, v, bias, *args, cp_reverse_out=True, **kwargs + ), + arg_nums, + ), + in_shardings=( + self.qkvo_sharding, + self.qkvo_sharding, + self.qkvo_sharding, + self.bias_sharding, + self.seq_desc_sharding, + self.dropout_rng_sharding, + ), + out_shardings=(None, grad_shardings), + ) + with self.mesh, fp8_autocast(mesh_resource=self.mesh_resource): + for _ in range(warmup): + jitted_primitive(*customcall_args) + + os.environ["NVTE_DUMP_AITER_RT"] = str(timings_dir) + '/' + + for _ in range(iters): + jitted_primitive(*customcall_args) + + del os.environ["NVTE_DUMP_AITER_RT"] + +def _filter_configs(configs): + for config in configs: + ( + shape, + dtype, + attn_mask_type, + qkv_layout, + seq_desc_format, + swa, + is_training, + dropout_prob, + bias_config + ) = config + b, s_q, s_kv, h_q, h_kv, d_qk, d_v = shape + attn_bias_type, bias_shape = bias_config + window_size = None + if swa: + window_size = (s_kv // 10, 0) + if qkv_layout.is_thd(): + if not attn_mask_type.is_padding(): + continue + if seq_desc_format == SeqDescFormat.Mask: + continue + if qkv_layout.is_qkvpacked(): + if (s_q != s_kv) or h_q != h_kv: + continue + if s_q > s_kv and window_size is not None: + continue + if d_qk != d_v and not qkv_layout.is_separate(): + continue + + backend = FusedAttnHelper( + dtype, + dtype, + qkv_layout, + attn_bias_type, + attn_mask_type, + dropout_prob, + h_q, h_kv, + s_q, s_kv, + d_qk, d_v, + (-1, -1) if window_size is None else window_size, + ).get_fused_attn_backend() + if backend == -1: + continue + if ( + attn_bias_type == AttnBiasType.POST_SCALE_BIAS + and bias_shape != BiasShape._1HSS + ): + if attn_mask_type.is_padding(): + continue + yield config + +def read_timings(timings_dir, rows, output, mode): + timings_path = timings_dir / f'aiter-{mode}-timings.txt' + times = pd.read_csv(timings_path, header=None, dtype=float) + os.remove(timings_path) + rows.extend([output | {"mode": mode, "time": t} for t in times[0].to_list()]) + +# Runs profiler and records timing information +def benchmark_dot_product_attention_profiler(args): + rows = [] + src_dir = Path(__file__).parent + timings_dir = src_dir / "timings" + os.makedirs(timings_dir, exist_ok=True) + for n, config in enumerate(_filter_configs(CONFIGS)): + ( + shape, + dtype, + attn_mask_type, + qkv_layout, + seq_desc_format, + swa, + is_training, + dropout_prob, + bias_config + ) = config + b, s_q, s_kv, h_q, h_kv, d_qk, d_v = shape + attn_bias_type, bias_shape = bias_config + window_size = None + if swa: + window_size = (s_kv // 10, 0) + output = { + "batch_size":b, + "q_seq_len":s_q, + "kv_seq_len":s_kv, + "q_heads":h_q, + "kv_heads":h_kv, + "qk_dim":d_qk, + "v_dim":d_v, + "attn_bias_type":attn_bias_type, + "attn_mask_type":attn_mask_type, + "dropout":dropout_prob, + "dtype":dtype, + "is_training":is_training, + "qkv_layout":qkv_layout, + "bias_shape":bias_shape, + "swa":swa, + "seq_desc_format":seq_desc_format, + } + if args.v: + print(f"Progress: {n+1}") + if args.v > 1: + print(output) + runner = FusedAttnBenchRunner( + b, s_q, s_kv, + h_q, h_kv, + d_qk, d_v, + attn_bias_type, + attn_mask_type, + dropout_prob, + True, + dtype, + is_training, + qkv_layout, + bias_shape, + window_size, + seq_desc_format, + ) + bench_fn = runner.bench_backward if args.bench_bwd else runner.bench_forward + bench_fn(args.warmup, args.iters, timings_dir) + + read_timings(timings_dir, rows, output, mode="fwd") + if args.bench_bwd: + read_timings(timings_dir, rows, output, mode="bwd") + + os.rmdir(timings_dir) + output_path = Path(__file__).parent + os.makedirs(output_path, exist_ok=True) + with open(output_path / "times.csv", "w", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=COLUMNS) + writer.writeheader() + writer.writerows(rows) + +class env_manager: + def __init__(self, fwd, bwd): + self.vals = {} + self.config = ((fwd, "FWD"), (bwd, "BWD")) + + def __enter__(self): + for flag, mode in self.config: + if flag: + self.vals[mode] = os.environ.get(f"NVTE_CK_USES_{mode}_V3") + os.environ[f"NVTE_CK_USES_{mode}_V3"] = "1" + + def __exit__(self, exc_type, exc_value, traceback): + for flag, mode in self.config: + if flag: + del os.environ[f"NVTE_CK_USES_{mode}_V3"] + if self.vals[mode]: + os.environ[f"NVTE_CK_USES_{mode}_V3"] = self.vals[mode] + +def main(args): + with env_manager(args.fwd_v3, args.bwd_v3): + benchmark_dot_product_attention_profiler(args) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--bench-bwd", action="store_true", help="Whether to bench the backwards pass as well.") + parser.add_argument("--fwd-v3", action="store_true", help="Use NVTE_CK_USES_FWD_V3=1 for AITER fwd kernels.") + parser.add_argument("--bwd-v3", action="store_true", help="Use NVTE_CK_USES_BWD_V3=1 for AITER bwd kernels.") + parser.add_argument("-v", action='count', default=0, help="Whether to include verbose debug outputs.") + parser.add_argument("--warmup", type=int, default=10, help="The number of iterations to run the kernel before logging run time. (default 10)") + parser.add_argument("--iters", type=int, default=50, help="The number of iterations to run the kernel while logging run time. (default 50)") + args = parser.parse_args() + main(args) diff --git a/benchmarks/attention/panel_app.py b/benchmarks/attention/panel_app.py new file mode 100644 index 000000000..df9b5c486 --- /dev/null +++ b/benchmarks/attention/panel_app.py @@ -0,0 +1,127 @@ +# Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. +# See LICENSE for license information. + +import pandas as pd +import panel as pn +import seaborn as sns +from matplotlib.figure import Figure +from jax import numpy as jnp + +pn.extension(design="material", sizing_mode="stretch_width") + +ATTRIBUTES = [ + "bias_config", + "attn_mask_type", + "qkv_layout", + "is_training", + "swa", + "dropout", + "mode", + "dtype", + "seq_desc_format", +] +CONVERTERS = { + "attn_mask_type": { + "AttnMaskType.NO_MASK": "None", + "AttnMaskType.CAUSAL_MASK": "Causal", + "AttnMaskType.PADDING_MASK": "Padding", + "AttnMaskType.PADDING_CAUSAL_MASK": "Padding Causal", + "AttnMaskType.CAUSAL_BOTTOM_RIGHT_MASK": "Causal Bottom-Right", + "AttnMaskType.PADDING_CAUSAL_BOTTOM_RIGHT_MASK": "Padding Causal Bottom-Right", + }, + "attn_bias_type": { + "AttnBiasType.NO_BIAS":"None", + "AttnBiasType.POST_SCALE_BIAS":"Post-Scale Bias", + }, + "dropout": { + 0: False, + 0.1: True, + }, + "dtype": { + str(jnp.float16):"FP16", + str(jnp.bfloat16):"BF16", + }, + "qkv_layout": { + "QKVLayout.BS3HD":"BSHD-Packed", + "QKVLayout.BSHD_BS2HD":"BSHD-KV-Packed", + "QKVLayout.BSHD_BSHD_BSHD":"BSHD-Separate", + "QKVLayout.T3HD":"THD-Packed", + "QKVLayout.THD_T2HD":"THD-KV-Packed", + "QKVLayout.THD_THD_THD":"THD-Separate", + }, + "bias_shape":{ + "BiasShape._1HSS":"_1HSS", + "NaN":"None", + }, + "seq_desc_format":{ + "SeqDescFormat.Mask": "Mask", + "SeqDescFormat.SegmentIDs": "SegmentIDs", + "SeqDescFormat.Seqlens": "Seqlens", + } +} +BIAS_CONFIGS = { + ("None", "None"): "None", + ("Post-Scale Bias", "_1HSS"): "Post-Scale _1HSS" +} + +@pn.cache +def get_data(): + df = pd.read_csv("times.csv").fillna("NaN") + df["time"] *= 1000 + for key in CONVERTERS: + df[key] = df[key].map(lambda x: CONVERTERS[key][x]) + df["bias_config"] = df.apply( + lambda row: BIAS_CONFIGS[(row["attn_bias_type"], row["bias_shape"])], + axis=1 + ) + df = df.drop(columns=["attn_bias_type", "bias_shape"]) + return df + +def _selector_widgets(): + df = get_data() + return { + cat: pn.widgets.Select(name=cat, options=list(df[cat].unique())) + for cat in ATTRIBUTES + } + + +selector_widgets = _selector_widgets() + +def make_plot(hue, indep, percentile, **kwargs): + fig = Figure(figsize=(8, 8)) + ax = fig.add_subplot(111) + df = get_data() + for attr in ATTRIBUTES: + if attr not in {hue, indep}: + df = df[(df[attr]==kwargs[attr])] + + for idx in df[indep].unique(): + for jdx in df[hue].unique(): + subset = df[(df[indep]==idx) & (df[hue]==jdx)] + df[(df[indep]==idx) & (df[hue]==jdx)] = subset[subset.time < subset.time.quantile(percentile)] + + if not df.empty: + ax.set(xlabel=indep, ylabel='Time (ms)') + sns.swarmplot(ax=ax, data=df, x=indep, y="time", hue=hue, dodge=True) + return fig + +hue_selector = pn.widgets.Select(name="Hue", options=ATTRIBUTES, value="dtype") +indep_selector = pn.widgets.Select(name="Independent Variable", options=ATTRIBUTES, value="attn_mask_type") +percentile_trim = pn.widgets.FloatSlider(value=.95, start=0, end=1, step=.01, name="Percentile Trim") +bound_make_plot = pn.bind( + make_plot, + hue=hue_selector, + indep=indep_selector, + percentile=percentile_trim, + **selector_widgets, +) + +template = pn.template.BootstrapTemplate( + title='JAX Fused Attention Benchmarks', + sidebar=pn.Row( + pn.Column(hue_selector, indep_selector, percentile_trim), + pn.Column(*[selector_widgets[k] for k in selector_widgets]), + ) +) +template.main.append(pn.pane.Matplotlib(bound_make_plot, dpi=144, height=600)) +template.servable(); \ No newline at end of file diff --git a/benchmarks/attention/plotting.ipynb b/benchmarks/attention/plotting.ipynb new file mode 100644 index 000000000..cc9fd5086 --- /dev/null +++ b/benchmarks/attention/plotting.ipynb @@ -0,0 +1,898 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 50, + "id": "729dd513", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "(function(root) {\n", + " function now() {\n", + " return new Date();\n", + " }\n", + "\n", + " const force = true;\n", + " const py_version = '3.8.0'.replace('rc', '-rc.').replace('.dev', '-dev.');\n", + " const reloading = false;\n", + " const Bokeh = root.Bokeh;\n", + "\n", + " // Set a timeout for this load but only if we are not already initializing\n", + " if (typeof (root._bokeh_timeout) === \"undefined\" || (force || !root._bokeh_is_initializing)) {\n", + " root._bokeh_timeout = Date.now() + 5000;\n", + " root._bokeh_failed_load = false;\n", + " }\n", + "\n", + " function run_callbacks() {\n", + " try {\n", + " root._bokeh_onload_callbacks.forEach(function(callback) {\n", + " if (callback != null)\n", + " callback();\n", + " });\n", + " } finally {\n", + " delete root._bokeh_onload_callbacks;\n", + " }\n", + " console.debug(\"Bokeh: all callbacks have finished\");\n", + " }\n", + "\n", + " function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n", + " if (css_urls == null) css_urls = [];\n", + " if (js_urls == null) js_urls = [];\n", + " if (js_modules == null) js_modules = [];\n", + " if (js_exports == null) js_exports = {};\n", + "\n", + " root._bokeh_onload_callbacks.push(callback);\n", + "\n", + " if (root._bokeh_is_loading > 0) {\n", + " // Don't load bokeh if it is still initializing\n", + " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", + " return null;\n", + " } else if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n", + " // There is nothing to load\n", + " run_callbacks();\n", + " return null;\n", + " }\n", + "\n", + " function on_load() {\n", + " root._bokeh_is_loading--;\n", + " if (root._bokeh_is_loading === 0) {\n", + " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", + " run_callbacks()\n", + " }\n", + " }\n", + " window._bokeh_on_load = on_load\n", + "\n", + " function on_error(e) {\n", + " const src_el = e.srcElement\n", + " console.error(\"failed to load \" + (src_el.href || src_el.src));\n", + " }\n", + "\n", + " const skip = [];\n", + " if (window.requirejs) {\n", + " window.requirejs.config({'packages': {}, 'paths': {}, 'shim': {}});\n", + " root._bokeh_is_loading = css_urls.length + 0;\n", + " } else {\n", + " root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n", + " }\n", + "\n", + " const existing_stylesheets = []\n", + " const links = document.getElementsByTagName('link')\n", + " for (let i = 0; i < links.length; i++) {\n", + " const link = links[i]\n", + " if (link.href != null) {\n", + " existing_stylesheets.push(link.href)\n", + " }\n", + " }\n", + " for (let i = 0; i < css_urls.length; i++) {\n", + " const url = css_urls[i];\n", + " const escaped = encodeURI(url)\n", + " if (existing_stylesheets.indexOf(escaped) !== -1) {\n", + " on_load()\n", + " continue;\n", + " }\n", + " const element = document.createElement(\"link\");\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.rel = \"stylesheet\";\n", + " element.type = \"text/css\";\n", + " element.href = url;\n", + " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", + " document.body.appendChild(element);\n", + " } var existing_scripts = []\n", + " const scripts = document.getElementsByTagName('script')\n", + " for (let i = 0; i < scripts.length; i++) {\n", + " var script = scripts[i]\n", + " if (script.src != null) {\n", + " existing_scripts.push(script.src)\n", + " }\n", + " }\n", + " for (let i = 0; i < js_urls.length; i++) {\n", + " const url = js_urls[i];\n", + " const escaped = encodeURI(url)\n", + " if (skip.indexOf(escaped) !== -1 || existing_scripts.indexOf(escaped) !== -1) {\n", + " if (!window.requirejs) {\n", + " on_load();\n", + " }\n", + " continue;\n", + " }\n", + " const element = document.createElement('script');\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.async = false;\n", + " element.src = url;\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.head.appendChild(element);\n", + " }\n", + " for (let i = 0; i < js_modules.length; i++) {\n", + " const url = js_modules[i];\n", + " const escaped = encodeURI(url)\n", + " if (skip.indexOf(escaped) !== -1 || existing_scripts.indexOf(escaped) !== -1) {\n", + " if (!window.requirejs) {\n", + " on_load();\n", + " }\n", + " continue;\n", + " }\n", + " var element = document.createElement('script');\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.async = false;\n", + " element.src = url;\n", + " element.type = \"module\";\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.head.appendChild(element);\n", + " }\n", + " for (const name in js_exports) {\n", + " const url = js_exports[name];\n", + " const escaped = encodeURI(url)\n", + " if (skip.indexOf(escaped) >= 0 || root[name] != null) {\n", + " if (!window.requirejs) {\n", + " on_load();\n", + " }\n", + " continue;\n", + " }\n", + " var element = document.createElement('script');\n", + " element.onerror = on_error;\n", + " element.async = false;\n", + " element.type = \"module\";\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " element.textContent = `\n", + " import ${name} from \"${url}\"\n", + " window.${name} = ${name}\n", + " window._bokeh_on_load()\n", + " `\n", + " document.head.appendChild(element);\n", + " }\n", + " if (!js_urls.length && !js_modules.length) {\n", + " on_load()\n", + " }\n", + " };\n", + "\n", + " function inject_raw_css(css) {\n", + " const element = document.createElement(\"style\");\n", + " element.appendChild(document.createTextNode(css));\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " const js_urls = [\"https://cdn.holoviz.org/panel/1.8.1/dist/bundled/reactiveesm/es-module-shims@^1.10.0/dist/es-module-shims.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-3.8.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.8.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.8.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.8.0.min.js\", \"https://cdn.holoviz.org/panel/1.8.1/dist/panel.min.js\"];\n", + " const js_modules = [];\n", + " const js_exports = {};\n", + " const css_urls = [];\n", + " const inline_js = [ function(Bokeh) {\n", + " Bokeh.set_log_level(\"info\");\n", + " },\n", + "function(Bokeh) {} // ensure no trailing comma for IE\n", + " ];\n", + "\n", + " function run_inline_js() {\n", + " if ((root.Bokeh !== undefined) || (force === true)) {\n", + " for (let i = 0; i < inline_js.length; i++) {\n", + " try {\n", + " inline_js[i].call(root, root.Bokeh);\n", + " } catch(e) {\n", + " if (!reloading) {\n", + " throw e;\n", + " }\n", + " }\n", + " }\n", + " // Cache old bokeh versions\n", + " if (Bokeh != undefined && !reloading) {\n", + " var NewBokeh = root.Bokeh;\n", + " if (Bokeh.versions === undefined) {\n", + " Bokeh.versions = new Map();\n", + " }\n", + " if (NewBokeh.version !== Bokeh.version) {\n", + " Bokeh.versions.set(NewBokeh.version, NewBokeh)\n", + " }\n", + " root.Bokeh = Bokeh;\n", + " }\n", + " } else if (Date.now() < root._bokeh_timeout) {\n", + " setTimeout(run_inline_js, 100);\n", + " } else if (!root._bokeh_failed_load) {\n", + " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", + " root._bokeh_failed_load = true;\n", + " }\n", + " root._bokeh_is_initializing = false\n", + " }\n", + "\n", + " function load_or_wait() {\n", + " // Implement a backoff loop that tries to ensure we do not load multiple\n", + " // versions of Bokeh and its dependencies at the same time.\n", + " // In recent versions we use the root._bokeh_is_initializing flag\n", + " // to determine whether there is an ongoing attempt to initialize\n", + " // bokeh, however for backward compatibility we also try to ensure\n", + " // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n", + " // before older versions are fully initialized.\n", + " if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n", + " // If the timeout and bokeh was not successfully loaded we reset\n", + " // everything and try loading again\n", + " root._bokeh_timeout = Date.now() + 5000;\n", + " root._bokeh_is_initializing = false;\n", + " root._bokeh_onload_callbacks = undefined;\n", + " root._bokeh_is_loading = 0\n", + " console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n", + " load_or_wait();\n", + " } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n", + " setTimeout(load_or_wait, 100);\n", + " } else {\n", + " root._bokeh_is_initializing = true\n", + " root._bokeh_onload_callbacks = []\n", + " const bokeh_loaded = root.Bokeh != null && (root.Bokeh.version === py_version || (root.Bokeh.versions !== undefined && root.Bokeh.versions.has(py_version)));\n", + " if (!reloading && !bokeh_loaded) {\n", + " if (root.Bokeh) {\n", + " root.Bokeh = undefined;\n", + " }\n", + " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", + " }\n", + " load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n", + " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", + " run_inline_js();\n", + " });\n", + " }\n", + " }\n", + " // Give older versions of the autoload script a head-start to ensure\n", + " // they initialize before we start loading newer version.\n", + " setTimeout(load_or_wait, 100)\n", + "}(window));" + ], + "application/vnd.holoviews_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n const py_version = '3.8.0'.replace('rc', '-rc.').replace('.dev', '-dev.');\n const reloading = false;\n const Bokeh = root.Bokeh;\n\n // Set a timeout for this load but only if we are not already initializing\n if (typeof (root._bokeh_timeout) === \"undefined\" || (force || !root._bokeh_is_initializing)) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks;\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n if (js_modules == null) js_modules = [];\n if (js_exports == null) js_exports = {};\n\n root._bokeh_onload_callbacks.push(callback);\n\n if (root._bokeh_is_loading > 0) {\n // Don't load bokeh if it is still initializing\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n } else if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n // There is nothing to load\n run_callbacks();\n return null;\n }\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n window._bokeh_on_load = on_load\n\n function on_error(e) {\n const src_el = e.srcElement\n console.error(\"failed to load \" + (src_el.href || src_el.src));\n }\n\n const skip = [];\n if (window.requirejs) {\n window.requirejs.config({'packages': {}, 'paths': {}, 'shim': {}});\n root._bokeh_is_loading = css_urls.length + 0;\n } else {\n root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n }\n\n const existing_stylesheets = []\n const links = document.getElementsByTagName('link')\n for (let i = 0; i < links.length; i++) {\n const link = links[i]\n if (link.href != null) {\n existing_stylesheets.push(link.href)\n }\n }\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const escaped = encodeURI(url)\n if (existing_stylesheets.indexOf(escaped) !== -1) {\n on_load()\n continue;\n }\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n } var existing_scripts = []\n const scripts = document.getElementsByTagName('script')\n for (let i = 0; i < scripts.length; i++) {\n var script = scripts[i]\n if (script.src != null) {\n existing_scripts.push(script.src)\n }\n }\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const escaped = encodeURI(url)\n if (skip.indexOf(escaped) !== -1 || existing_scripts.indexOf(escaped) !== -1) {\n if (!window.requirejs) {\n on_load();\n }\n continue;\n }\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (let i = 0; i < js_modules.length; i++) {\n const url = js_modules[i];\n const escaped = encodeURI(url)\n if (skip.indexOf(escaped) !== -1 || existing_scripts.indexOf(escaped) !== -1) {\n if (!window.requirejs) {\n on_load();\n }\n continue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (const name in js_exports) {\n const url = js_exports[name];\n const escaped = encodeURI(url)\n if (skip.indexOf(escaped) >= 0 || root[name] != null) {\n if (!window.requirejs) {\n on_load();\n }\n continue;\n }\n var element = document.createElement('script');\n element.onerror = on_error;\n element.async = false;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n element.textContent = `\n import ${name} from \"${url}\"\n window.${name} = ${name}\n window._bokeh_on_load()\n `\n document.head.appendChild(element);\n }\n if (!js_urls.length && !js_modules.length) {\n on_load()\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n const js_urls = [\"https://cdn.holoviz.org/panel/1.8.1/dist/bundled/reactiveesm/es-module-shims@^1.10.0/dist/es-module-shims.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-3.8.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.8.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.8.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.8.0.min.js\", \"https://cdn.holoviz.org/panel/1.8.1/dist/panel.min.js\"];\n const js_modules = [];\n const js_exports = {};\n const css_urls = [];\n const inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n ];\n\n function run_inline_js() {\n if ((root.Bokeh !== undefined) || (force === true)) {\n for (let i = 0; i < inline_js.length; i++) {\n try {\n inline_js[i].call(root, root.Bokeh);\n } catch(e) {\n if (!reloading) {\n throw e;\n }\n }\n }\n // Cache old bokeh versions\n if (Bokeh != undefined && !reloading) {\n var NewBokeh = root.Bokeh;\n if (Bokeh.versions === undefined) {\n Bokeh.versions = new Map();\n }\n if (NewBokeh.version !== Bokeh.version) {\n Bokeh.versions.set(NewBokeh.version, NewBokeh)\n }\n root.Bokeh = Bokeh;\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n }\n root._bokeh_is_initializing = false\n }\n\n function load_or_wait() {\n // Implement a backoff loop that tries to ensure we do not load multiple\n // versions of Bokeh and its dependencies at the same time.\n // In recent versions we use the root._bokeh_is_initializing flag\n // to determine whether there is an ongoing attempt to initialize\n // bokeh, however for backward compatibility we also try to ensure\n // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n // before older versions are fully initialized.\n if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n // If the timeout and bokeh was not successfully loaded we reset\n // everything and try loading again\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_is_initializing = false;\n root._bokeh_onload_callbacks = undefined;\n root._bokeh_is_loading = 0\n console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n load_or_wait();\n } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n setTimeout(load_or_wait, 100);\n } else {\n root._bokeh_is_initializing = true\n root._bokeh_onload_callbacks = []\n const bokeh_loaded = root.Bokeh != null && (root.Bokeh.version === py_version || (root.Bokeh.versions !== undefined && root.Bokeh.versions.has(py_version)));\n if (!reloading && !bokeh_loaded) {\n if (root.Bokeh) {\n root.Bokeh = undefined;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n }\n load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n }\n // Give older versions of the autoload script a head-start to ensure\n // they initialize before we start loading newer version.\n setTimeout(load_or_wait, 100)\n}(window));" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "\n", + "if ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n", + " window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n", + "}\n", + "\n", + "\n", + " function JupyterCommManager() {\n", + " }\n", + "\n", + " JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n", + " if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n", + " var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n", + " comm_manager.register_target(comm_id, function(comm) {\n", + " comm.on_msg(msg_handler);\n", + " });\n", + " } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n", + " window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n", + " comm.onMsg = msg_handler;\n", + " });\n", + " } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n", + " google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n", + " var messages = comm.messages[Symbol.asyncIterator]();\n", + " function processIteratorResult(result) {\n", + " var message = result.value;\n", + " var content = {data: message.data, comm_id};\n", + " var buffers = []\n", + " for (var buffer of message.buffers || []) {\n", + " buffers.push(new DataView(buffer))\n", + " }\n", + " var metadata = message.metadata || {};\n", + " var msg = {content, buffers, metadata}\n", + " msg_handler(msg);\n", + " return messages.next().then(processIteratorResult);\n", + " }\n", + " return messages.next().then(processIteratorResult);\n", + " })\n", + " }\n", + " }\n", + "\n", + " JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n", + " if (comm_id in window.PyViz.comms) {\n", + " return window.PyViz.comms[comm_id];\n", + " } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n", + " var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n", + " var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n", + " if (msg_handler) {\n", + " comm.on_msg(msg_handler);\n", + " }\n", + " } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n", + " var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n", + " let retries = 0;\n", + " const open = () => {\n", + " if (comm.active) {\n", + " comm.open();\n", + " } else if (retries > 3) {\n", + " console.warn('Comm target never activated')\n", + " } else {\n", + " retries += 1\n", + " setTimeout(open, 500)\n", + " }\n", + " }\n", + " if (comm.active) {\n", + " comm.open();\n", + " } else {\n", + " setTimeout(open, 500)\n", + " }\n", + " if (msg_handler) {\n", + " comm.onMsg = msg_handler;\n", + " }\n", + " } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n", + " var comm_promise = google.colab.kernel.comms.open(comm_id)\n", + " comm_promise.then((comm) => {\n", + " window.PyViz.comms[comm_id] = comm;\n", + " if (msg_handler) {\n", + " var messages = comm.messages[Symbol.asyncIterator]();\n", + " function processIteratorResult(result) {\n", + " var message = result.value;\n", + " var content = {data: message.data};\n", + " var metadata = message.metadata || {comm_id};\n", + " var msg = {content, metadata}\n", + " msg_handler(msg);\n", + " return messages.next().then(processIteratorResult);\n", + " }\n", + " return messages.next().then(processIteratorResult);\n", + " }\n", + " })\n", + " var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n", + " return comm_promise.then((comm) => {\n", + " comm.send(data, metadata, buffers, disposeOnDone);\n", + " });\n", + " };\n", + " var comm = {\n", + " send: sendClosure\n", + " };\n", + " }\n", + " window.PyViz.comms[comm_id] = comm;\n", + " return comm;\n", + " }\n", + " window.PyViz.comm_manager = new JupyterCommManager();\n", + " \n", + "\n", + "\n", + "var JS_MIME_TYPE = 'application/javascript';\n", + "var HTML_MIME_TYPE = 'text/html';\n", + "var EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\n", + "var CLASS_NAME = 'output';\n", + "\n", + "/**\n", + " * Render data to the DOM node\n", + " */\n", + "function render(props, node) {\n", + " var div = document.createElement(\"div\");\n", + " var script = document.createElement(\"script\");\n", + " node.appendChild(div);\n", + " node.appendChild(script);\n", + "}\n", + "\n", + "/**\n", + " * Handle when a new output is added\n", + " */\n", + "function handle_add_output(event, handle) {\n", + " var output_area = handle.output_area;\n", + " var output = handle.output;\n", + " if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n", + " return\n", + " }\n", + " var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", + " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", + " if (id !== undefined) {\n", + " var nchildren = toinsert.length;\n", + " var html_node = toinsert[nchildren-1].children[0];\n", + " html_node.innerHTML = output.data[HTML_MIME_TYPE];\n", + " var scripts = [];\n", + " var nodelist = html_node.querySelectorAll(\"script\");\n", + " for (var i in nodelist) {\n", + " if (nodelist.hasOwnProperty(i)) {\n", + " scripts.push(nodelist[i])\n", + " }\n", + " }\n", + "\n", + " scripts.forEach( function (oldScript) {\n", + " var newScript = document.createElement(\"script\");\n", + " var attrs = [];\n", + " var nodemap = oldScript.attributes;\n", + " for (var j in nodemap) {\n", + " if (nodemap.hasOwnProperty(j)) {\n", + " attrs.push(nodemap[j])\n", + " }\n", + " }\n", + " attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n", + " newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n", + " oldScript.parentNode.replaceChild(newScript, oldScript);\n", + " });\n", + " if (JS_MIME_TYPE in output.data) {\n", + " toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n", + " }\n", + " output_area._hv_plot_id = id;\n", + " if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n", + " window.PyViz.plot_index[id] = Bokeh.index[id];\n", + " } else {\n", + " window.PyViz.plot_index[id] = null;\n", + " }\n", + " } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", + " var bk_div = document.createElement(\"div\");\n", + " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", + " var script_attrs = bk_div.children[0].attributes;\n", + " for (var i = 0; i < script_attrs.length; i++) {\n", + " toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n", + " }\n", + " // store reference to server id on output_area\n", + " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", + " }\n", + "}\n", + "\n", + "/**\n", + " * Handle when an output is cleared or removed\n", + " */\n", + "function handle_clear_output(event, handle) {\n", + " var id = handle.cell.output_area._hv_plot_id;\n", + " var server_id = handle.cell.output_area._bokeh_server_id;\n", + " if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n", + " var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n", + " if (server_id !== null) {\n", + " comm.send({event_type: 'server_delete', 'id': server_id});\n", + " return;\n", + " } else if (comm !== null) {\n", + " comm.send({event_type: 'delete', 'id': id});\n", + " }\n", + " delete PyViz.plot_index[id];\n", + " if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n", + " var doc = window.Bokeh.index[id].model.document\n", + " doc.clear();\n", + " const i = window.Bokeh.documents.indexOf(doc);\n", + " if (i > -1) {\n", + " window.Bokeh.documents.splice(i, 1);\n", + " }\n", + " }\n", + "}\n", + "\n", + "/**\n", + " * Handle kernel restart event\n", + " */\n", + "function handle_kernel_cleanup(event, handle) {\n", + " delete PyViz.comms[\"hv-extension-comm\"];\n", + " window.PyViz.plot_index = {}\n", + "}\n", + "\n", + "/**\n", + " * Handle update_display_data messages\n", + " */\n", + "function handle_update_output(event, handle) {\n", + " handle_clear_output(event, {cell: {output_area: handle.output_area}})\n", + " handle_add_output(event, handle)\n", + "}\n", + "\n", + "function register_renderer(events, OutputArea) {\n", + " function append_mime(data, metadata, element) {\n", + " // create a DOM node to render to\n", + " var toinsert = this.create_output_subarea(\n", + " metadata,\n", + " CLASS_NAME,\n", + " EXEC_MIME_TYPE\n", + " );\n", + " this.keyboard_manager.register_events(toinsert);\n", + " // Render to node\n", + " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", + " render(props, toinsert[0]);\n", + " element.append(toinsert);\n", + " return toinsert\n", + " }\n", + "\n", + " events.on('output_added.OutputArea', handle_add_output);\n", + " events.on('output_updated.OutputArea', handle_update_output);\n", + " events.on('clear_output.CodeCell', handle_clear_output);\n", + " events.on('delete.Cell', handle_clear_output);\n", + " events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n", + "\n", + " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", + " safe: true,\n", + " index: 0\n", + " });\n", + "}\n", + "\n", + "if (window.Jupyter !== undefined) {\n", + " try {\n", + " var events = require('base/js/events');\n", + " var OutputArea = require('notebook/js/outputarea').OutputArea;\n", + " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", + " register_renderer(events, OutputArea);\n", + " }\n", + " } catch(err) {\n", + " }\n", + "}\n" + ], + "application/vnd.holoviews_load.v0+json": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n function JupyterCommManager() {\n }\n\n JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n comm_manager.register_target(comm_id, function(comm) {\n comm.on_msg(msg_handler);\n });\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n comm.onMsg = msg_handler;\n });\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n var content = {data: message.data, comm_id};\n var buffers = []\n for (var buffer of message.buffers || []) {\n buffers.push(new DataView(buffer))\n }\n var metadata = message.metadata || {};\n var msg = {content, buffers, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n })\n }\n }\n\n JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n if (comm_id in window.PyViz.comms) {\n return window.PyViz.comms[comm_id];\n } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n if (msg_handler) {\n comm.on_msg(msg_handler);\n }\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n let retries = 0;\n const open = () => {\n if (comm.active) {\n comm.open();\n } else if (retries > 3) {\n console.warn('Comm target never activated')\n } else {\n retries += 1\n setTimeout(open, 500)\n }\n }\n if (comm.active) {\n comm.open();\n } else {\n setTimeout(open, 500)\n }\n if (msg_handler) {\n comm.onMsg = msg_handler;\n }\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n var comm_promise = google.colab.kernel.comms.open(comm_id)\n comm_promise.then((comm) => {\n window.PyViz.comms[comm_id] = comm;\n if (msg_handler) {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n var content = {data: message.data};\n var metadata = message.metadata || {comm_id};\n var msg = {content, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n }\n })\n var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n return comm_promise.then((comm) => {\n comm.send(data, metadata, buffers, disposeOnDone);\n });\n };\n var comm = {\n send: sendClosure\n };\n }\n window.PyViz.comms[comm_id] = comm;\n return comm;\n }\n window.PyViz.comm_manager = new JupyterCommManager();\n \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n var div = document.createElement(\"div\");\n var script = document.createElement(\"script\");\n node.appendChild(div);\n node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n var output_area = handle.output_area;\n var output = handle.output;\n if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n return\n }\n var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n if (id !== undefined) {\n var nchildren = toinsert.length;\n var html_node = toinsert[nchildren-1].children[0];\n html_node.innerHTML = output.data[HTML_MIME_TYPE];\n var scripts = [];\n var nodelist = html_node.querySelectorAll(\"script\");\n for (var i in nodelist) {\n if (nodelist.hasOwnProperty(i)) {\n scripts.push(nodelist[i])\n }\n }\n\n scripts.forEach( function (oldScript) {\n var newScript = document.createElement(\"script\");\n var attrs = [];\n var nodemap = oldScript.attributes;\n for (var j in nodemap) {\n if (nodemap.hasOwnProperty(j)) {\n attrs.push(nodemap[j])\n }\n }\n attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n oldScript.parentNode.replaceChild(newScript, oldScript);\n });\n if (JS_MIME_TYPE in output.data) {\n toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n }\n output_area._hv_plot_id = id;\n if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n window.PyViz.plot_index[id] = Bokeh.index[id];\n } else {\n window.PyViz.plot_index[id] = null;\n }\n } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n var bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n var script_attrs = bk_div.children[0].attributes;\n for (var i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n var id = handle.cell.output_area._hv_plot_id;\n var server_id = handle.cell.output_area._bokeh_server_id;\n if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n if (server_id !== null) {\n comm.send({event_type: 'server_delete', 'id': server_id});\n return;\n } else if (comm !== null) {\n comm.send({event_type: 'delete', 'id': id});\n }\n delete PyViz.plot_index[id];\n if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n var doc = window.Bokeh.index[id].model.document\n doc.clear();\n const i = window.Bokeh.documents.indexOf(doc);\n if (i > -1) {\n window.Bokeh.documents.splice(i, 1);\n }\n }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n delete PyViz.comms[\"hv-extension-comm\"];\n window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n handle_clear_output(event, {cell: {output_area: handle.output_area}})\n handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n var toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[0]);\n element.append(toinsert);\n return toinsert\n }\n\n events.on('output_added.OutputArea', handle_add_output);\n events.on('output_updated.OutputArea', handle_update_output);\n events.on('clear_output.CodeCell', handle_clear_output);\n events.on('delete.Cell', handle_clear_output);\n events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n safe: true,\n index: 0\n });\n}\n\nif (window.Jupyter !== undefined) {\n try {\n var events = require('base/js/events');\n var OutputArea = require('notebook/js/outputarea').OutputArea;\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n } catch(err) {\n }\n}\n" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.holoviews_exec.v0+json": "", + "text/html": [ + "
\n", + "
\n", + "
\n", + "" + ] + }, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "b0d583bf-d55a-4b5d-bd24-719874acb0e3" + } + }, + "output_type": "display_data" + } + ], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import panel as pn\n", + "pn.extension()" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "id": "9546da1e-b82c-4748-985b-a9a92ecb1dc7", + "metadata": {}, + "outputs": [], + "source": [ + "@pn.cache\n", + "def get_data():\n", + " return pd.read_csv(\"output_main.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "id": "bba1ffa5-851f-41cd-9c1c-391b5b9b9485", + "metadata": {}, + "outputs": [], + "source": [ + "df=get_data().fillna(\"NaN\")" + ] + }, + { + "cell_type": "code", + "execution_count": 104, + "id": "11557b76-3156-45f5-8241-019f4fa0933a", + "metadata": {}, + "outputs": [], + "source": [ + "VARIABLES = [\"dtype\", \"seq_desc_format\"]\n", + "def _selector_widgets():\n", + " for cat in (\"attn_bias_type\", \"attn_mask_type\", \"qkv_layout\", \"bias_shape\", \"is_training\",\"swa\", \"dropout\"):\n", + " yield pn.widgets.Select(name=cat, options=list(df[cat].unique()))\n", + "\n", + "\n", + "selector_widgets = list(_selector_widgets())\n" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "id": "9bcaaf1c-2759-4ecf-a400-1f39897d341b", + "metadata": {}, + "outputs": [], + "source": [ + "import seaborn as sns" + ] + }, + { + "cell_type": "code", + "execution_count": 113, + "id": "558f449f-dd9c-47b3-bef2-c5a1793b8777", + "metadata": {}, + "outputs": [], + "source": [ + "def make_plot(attn_bias_type, attn_mask_type, qkv_layout, bias_shape, is_training, swa, dropout):\n", + " subset = df[\n", + " (\n", + " (df[\"attn_bias_type\"]==attn_bias_type) &\n", + " (df[\"attn_mask_type\"]==attn_mask_type) &\n", + " (df[\"qkv_layout\"]==qkv_layout) &\n", + " (df[\"bias_shape\"]==bias_shape) &\n", + " (df[\"is_training\"]==is_training) &\n", + " (df[\"swa\"]==swa) &\n", + " (df[\"dropout\"]==dropout)\n", + " )\n", + " ]\n", + " subset = subset[subset.time < subset.time.quantile(.95)]\n", + " subset = subset[subset.time > subset.time.quantile(.05)]\n", + " if not subset.empty:\n", + " return sns.catplot(data=subset, kind=\"violin\", x=\"seq_desc_format\", y=\"time\", hue=\"dtype\", split=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 115, + "id": "921e4c12-f4b3-4393-a569-3e88fda9c0fe", + "metadata": {}, + "outputs": [ + { + "data": {}, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.holoviews_exec.v0+json": "", + "text/html": [ + "
\n", + "
\n", + "
\n", + "" + ], + "text/plain": [ + "Row\n", + " [0] Column\n", + " [0] Select(name='attn_bias_type', options=['AttnBiasType.NO_BIAS', ...], value='AttnBiasType.NO_BIAS')\n", + " [1] Select(name='attn_mask_type', options=['AttnMaskType.NO_MASK', ...], value='AttnMaskType.NO_MASK')\n", + " [2] Select(name='qkv_layout', options=['QKVLayout.BS3HD', ...], value='QKVLayout.BS3HD')\n", + " [3] Select(name='bias_shape', options=['NaN', 'BiasShape._1HSS']...], value='NaN')\n", + " [4] Select(name='is_training', options=[True, False], value=True)\n", + " [5] Select(name='swa', options=[True, False], value=True)\n", + " [6] Select(name='dropout', options=[0.0, 0.1], value=0.0)\n", + " [1] ParamFunction(function, _pane=Str, defer_load=False)" + ] + }, + "execution_count": 115, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "a3c3ac47-984b-4a04-ab5f-442bb10f1822" + } + }, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAArwAAAHpCAYAAABgEe+aAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjYsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvq6yFwwAAAAlwSFlzAAAPYQAAD2EBqD+naQAAnnNJREFUeJzs3XlcVFX/B/DPLMwMw77IpqiouKOoKELmkhjm8ki2qI/lrpU/F3Iry+2xHi3LUksz60kttcxKMzXKXEsJ99z3XQEX9p2ZOb8/iFuTgKDM3Bn4vF8vXsK95977hRHmM2fOPUchhBAgIiIiIqqilHIXQERERERkSQy8RERERFSlMfASERERUZXGwEtEREREVRoDLxERERFVaQy8RERERFSlMfASERERUZXGwCsjIQQyMjLAqZCJiIiILIeBV0aZmZlwc3NDZmam3KUQERERVVkMvERERERUpTHwEhEREVGVxsBLRERERFUaAy8RERERVWkMvERERERUpTHwEhEREVGVxsBLRERERFUaAy8RERERVWkMvERERERUpTHwEhEREVGVxsBLRERERFUaAy8RERERVWkMvERERERUpTHwEhEREVGVxsBLRERERFUaAy8RERERVWkMvERERERUpTHwEhEREVGVxsBLRGXKyMjAgAEDsH79erlLISIieiAMvERUppMnTyIxMRELFy6UuxQiIqIHwsBLRGVSKBRyl0BERPRQGHiJqEwMvEREZO8YeImoTEol/0wQEZF94zMZEZWJPbxERGTvGHiJqEzs4SUiInvHZzIiKhMDLxER2Ts+kxFRmTikgYiI7B0DLxGViT28RERk7/hMRkRlYg8vERHZOwZeIiqTSqWSuwQiIqKHwsBLRGViDy8REdk7Bl4iKhPH8BIRkb3jMxkRlYk9vEREZO8YeImIiIioSmPgJSIiIqIqjYGXiIiIiKo0Bl4iIiIiqtIYeImIiIioSmPgJSIiIqIqjYGXiIiIiKo0Bl4iIiIiqtIYeImIiIioSmPgJSIiIqIqjYGXiIiIiKo0Bl4iIiIiqtIYeImIiIioSmPgJaJyE0LIXQIREVGFMfASUbkZjUa5SyAiIqowBl4iKjeDwSB3CURERBXGwEtE5VZYWCh3CURERBVmE4F38eLFqFu3LnQ6HcLDw7Fv374y269btw6NGzeGTqdDSEgItmzZYrZfCIEZM2bA398fjo6OiIqKwrlz58zapKSkYODAgXB1dYW7uzuGDx+OrKysEq93/vx5uLi4wN3dvcK1EFUlDLxERGSPZA+8a9euxYQJEzBz5kwcOnQILVu2RHR0NG7dulVi+71792LAgAEYPnw4Dh8+jJiYGMTExOD48eNSm3nz5mHRokVYunQpEhIS4OTkhOjoaOTl5UltBg4ciBMnTmDr1q3YtGkTdu/ejVGjRt1zvcLCQgwYMACPPvroA9VCVJUw8BIRkT1SCJlvuw4PD0fbtm3x4YcfAgBMJhMCAwMxduxYvPrqq/e079evH7Kzs7Fp0yZpW/v27REaGoqlS5dCCIGAgABMnDgRkyZNAgCkp6fD19cXK1asQP/+/XHq1Ck0bdoU+/fvR1hYGAAgLi4OPXr0wPXr1xEQECCd+5VXXsHNmzfRtWtXxMbGIi0trdy1/FN+fj7y8/OlrzMyMhAYGIj09HS4uro+4E+QyLLOnz+PESNGAABWrVqFWrVqyVwRERFRxcjaw1tQUICDBw8iKipK2qZUKhEVFYX4+PgSj4mPjzdrDwDR0dFS+0uXLiEpKcmsjZubG8LDw6U28fHxcHd3l8IuAERFRUGpVCIhIUHatn37dqxbtw6LFy9+oFr+ae7cuXBzc5M+AgMDS2xHZKsKCgrkLoGIiKjCZA28d+7cgdFohK+vr9l2X19fJCUllXhMUlJSme2L/71fGx8fH7P9arUanp6eUpu7d+9iyJAhWLFiRam9r/er5Z+mTp2K9PR06ePatWsltiOyVRzSQERE9kgtdwG2auTIkfj3v/+Njh07Vto5tVottFptpZ2PyNrYw0tERPZI1h5eb29vqFQqJCcnm21PTk6Gn59ficf4+fmV2b743/u1+edNcQaDASkpKVKb7du3491334VarYZarcbw4cORnp4OtVqNzz77rFy1EFU1DLxERGSPZA28Go0Gbdq0wbZt26RtJpMJ27ZtQ0RERInHREREmLUHgK1bt0rtg4KC4OfnZ9YmIyMDCQkJUpuIiAikpaXh4MGDUpvt27fDZDIhPDwcQNH43CNHjkgfs2fPhouLC44cOYInn3yyXLUQVTUMvEREZI9kH9IwYcIEDB48GGFhYWjXrh0WLFiA7OxsDB06FAAwaNAg1KxZE3PnzgUAjB8/Hp06dcL8+fPRs2dPfPXVVzhw4ACWLVsGAFAoFIiNjcWbb76J4OBgBAUFYfr06QgICEBMTAwAoEmTJujevTtGjhyJpUuXorCwEGPGjEH//v2lGRqaNGliVueBAwegVCrRvHlzadv9aiGqav4+ywgREZG9kD3w9uvXD7dv38aMGTOQlJSE0NBQxMXFSTeDXb16FUrlXx3RkZGRWLNmDaZNm4bXXnsNwcHB2LBhg1kQnTJlCrKzszFq1CikpaWhQ4cOiIuLg06nk9qsXr0aY8aMQdeuXaFUKvHUU09h0aJFFaq9PLUQVSXs4SUiInsk+zy81VlGRgbc3Nw4Dy/ZtL/Pwztx4kT07t1b5oqIiIgqRvaV1ojItqWnp0ufc0gDERHZIwZeIirTF198IX3OwEtERPaIgZeIynThwgXp87y8PBkrISIiejAMvERUpszMTOlz9vASEZE9YuAlolKZTCazrxl4iYjIHjHwElGp/rmSIIc0EBGRPWLgJaJSXbp0yexrBl4iIrJHDLxEVKqzZ8+afc3AS0RE9oiBl4hKdfLkSbOvc3NzZaqEiIjowTHwElGJDAYDjh07braNgZeIiOwRAy8RlejMmTPIzc2BwdFD2sbAS0RE9oiBl4hKlJCQAAAwufhK2/IYeImIyA4x8BJRiX777TdAqYJRX0Paxh5eIiKyRwy8RHSP69ev4+LFiyh0rQmoVNL23LxcCCFkrIyIiKjiGHiJ6B7bt28HABg8g8y2m0yCU5MREZHdYeAlIjNCCPzyyy+AUg2De+179ufk5MhQFRER0YNj4CUiM2fOnMHVq1dR6F4bUDncs5/jeImIyN4w8BKRmbi4OABAoXeDEvdnZ2dbsxwiIqKHxsBLRJKCggJs27YNwkEPo2tAiW04pIGIiOwNAy8RSRISEpCZmYkCr/qAwvzPg1JRNDvDihUrMGrUKLz55ptylEhERFRhDLxkUTk5Odi5cycKCwvlLoXK4eeffwYAGEoYzuCoKgq8ycnJOHv2LK5evWrV2oiIiB4UAy9Z1PLlyzFr1ixs2bJF7lLoPjIzMxEfHw+j3hOmvy0nXMxRXRR4jUajtUsjIiJ6KAy8ZFHx8fEAgNOnT8tcCd3P3r17YTAYAAHoT3wP3YWdZvuLA6/JZJKhOiIiogenlrsAqtoUCgUAhiR78OuvvxZ9IkxQ5abds1+vYuAlIiL7xMBLRCgoKMCBAwdgdHQHFKoS27CHl4iI7BWHNJBFFffwCiFkroTKcuLECeTl5cHoWrPUNnqO4SUiIjvFwEsWpVQW/Rdjr6BtO3r0KADA4Opfahs9e3jtmsFg4AtPIqq2GHjJKvhEa9tOnjwJADA6+5baRq0UcFAy8Nojo9GIgc89h1mzZsldChGRLDiGlyyquIeXgde2nT17FiatC6DWltlOrzYhh0Ma7E5BQQGSk5KQnJQkdylERLJgDy9ZFMfw2r709HSkpqbCWMLcu/+kV5vYw2uHOO6aiKo7Bl6yKE5LZvuuX78OADDp3O7b1kktYDIxPNkbBl4iqu4YeMmiOKTB9iX9+Ta30Lrct63eQYAPpf1h4CWi6o6BlyyKgdf23bp1CwBg0jjdt23xTA1kXxh4iai6Y+Ali+KQBtt3+/ZtAIAoR+B1YuC1SwaDQe4SiIhkxcBLFsV5eG3fnTt3AAAmjf6+bZ3UfBztEXt4iai6Y+Ali+IsDbbv9u3bgFIFqMqekgwAnBz4ONoj9vASUXXHwEsWVdzDyx4m25WcfAsmByfgzxcnZeGQBvtUWFgodwlERLJi4CWLYg+vbcvNzUVKyt2iRSfKwcmBQxrsEV9wElF1x8BLFqVSqQBwDK+tunLlCgDA5OhervbOHNJgl9jDS0TVHQMvWRRvWrNtZ86cAQAY9Z7las8hDfaJY3iJqLpj4CWL4pAG23bkyBEAgNHZt1ztnTmkwS6xh5eIqjsGXrIoDmmwXQaDAfv3H4BJ61yuVdYAztJgrxh4iai6Y+Ali+LCE7Zr3759yMrKhMG9TrlmaAAAByWgAEOvvWHgJaLqjoGXLIpjeG3XDz/8AAAo9A6u0HHK8mVjsiEFBQVyl0BEJCsGXrIoDmmwTZcuXUJ8fDwMzr4wlfOGtWIMvPaHPbxEVN0x8JJFcUiDbfr0008BAAUBLSt8rOrPwMsbEe0HAy8RVXcMvGRRHNJge/bt24c9e/bA4OIPo2vNCh+vVBQFXT6m9oOBl4iqOwZesqjiIQ1c6ck25OTk4L333gcUSuTXbl/um9X+rnhIAx9T+8ExvERU3THwkkVxSINt+fDDD5GUlIh8/xYw6T0e6BwqBl67wx5eIqruGHjJooqHNHC8p/y2bt2KLVu2wOjkjQL/0Ac+T/GQBgZe+8HAS0TVnU0E3sWLF6Nu3brQ6XQIDw/Hvn37ymy/bt06NG7cGDqdDiEhIdiyZYvZfiEEZsyYAX9/fzg6OiIqKgrnzp0za5OSkoKBAwfC1dUV7u7uGD58OLKysqT9Z86cQZcuXeDr6wudTod69eph2rRpZk8cK1asgEKhMPvQ6XSV8BOpOooDL8ORvM6fP4933nkXUGmQW68zoHzwX/3iHl722tsPBl4iqu5kD7xr167FhAkTMHPmTBw6dAgtW7ZEdHQ0bt26VWL7vXv3YsCAARg+fDgOHz6MmJgYxMTE4Pjx41KbefPmYdGiRVi6dCkSEhLg5OSE6Oho5OXlSW0GDhyIEydOYOvWrdi0aRN2796NUaNGSfsdHBwwaNAg/Pzzzzhz5gwWLFiATz75BDNnzjSrx9XVFYmJidLHlStXKvknZN9405r8UlJS8Nrrr6OgIB859TpB6Fwf6nwcw2t/isfwKh/ihQ4RkT1Ty13Ae++9h5EjR2Lo0KEAgKVLl2Lz5s347LPP8Oqrr97TfuHChejevTsmT54MAHjjjTewdetWfPjhh1i6dCmEEFiwYAGmTZuGPn36AAA+//xz+Pr6YsOGDejfvz9OnTqFuLg47N+/H2FhYQCADz74AD169MC7776LgIAA1KtXD/Xq1ZOuW6dOHezcuRO//vqrWT0KhQJ+fn4W+dlUBRzSIK/8/HxMmzYNt5KTkV+rDYzugQ99TgZe+2MwGAAAGo1G5kqIiOQh68v9goICHDx4EFFRUdI2pVKJqKgoxMfHl3hMfHy8WXsAiI6OltpfunQJSUlJZm3c3NwQHh4utYmPj4e7u7sUdgEgKioKSqUSCQkJJV73/PnziIuLQ6dOncy2Z2VloU6dOggMDESfPn1w4sSJUr/f/Px8ZGRkmH1UdRzSIB+TyYQ5c+bg5MmTKPRqgAK/FpVyXhU4htfeFA9pKJ41hYioupE18N65cwdGoxG+vr5m2319fZGUlFTiMUlJSWW2L/73fm18fHzM9qvVanh6et5z3cjISOh0OgQHB+PRRx/F7NmzpX2NGjXCZ599hu+//x6rVq2CyWRCZGQkrl+/XmLtc+fOhZubm/QRGPjwvW22jkMa5PPxxx9j165dMLj4I6/uIw80BVlJ2MNrf4oDL3t4iai64oCu+1i7di0OHTqENWvWYPPmzXj33XelfRERERg0aBBCQ0PRqVMnfPfdd6hRowY+/vjjEs81depUpKenSx/Xrl2z1rchGw5pkMf69euxdu1amBzdkdvgMUBZeT17DLz2p3hIA3t4iai6knUMr7e3N1QqFZKTk822Jycnlzou1s/Pr8z2xf8mJyfD39/frE1oaKjU5p83xRkMBqSkpNxz3eJe2KZNm8JoNGLUqFGYOHFiiU8cDg4OaNWqFc6fP19i7VqtFlqttsR9VRV7eK1v7969WLToAwgHPXKCHwfUlft/TsFZGuxOceBVq2W/bYOISBay9vBqNBq0adMG27Ztk7aZTCZs27YNERERJR4TERFh1h4oml+0uH1QUBD8/PzM2mRkZCAhIUFqExERgbS0NBw8eFBqs337dphMJoSHh5dar8lkQmFhYalP9EajEceOHTML2tUde3it6/z58/jP7NkQCiVygrtBaJ0tdi328NqP4sDr4OAgcyVERPKQ/eX+hAkTMHjwYISFhaFdu3ZYsGABsrOzpVkbBg0ahJo1a2Lu3LkAgPHjx6NTp06YP38+evbsia+++goHDhzAsmXLABTNmhAbG4s333wTwcHBCAoKwvTp0xEQEICYmBgAQJMmTdC9e3eMHDkSS5cuRWFhIcaMGYP+/fsjICAAALB69Wo4ODggJCQEWq0WBw4cwNSpU9GvXz/pSWP27Nlo3749GjRogLS0NLzzzju4cuUKRowYYeWfou3iTWvWk5aWhtdefx35eXnIaRAFk5OXRa/Hx9R+cEgDEVV3sgfefv364fbt25gxYwaSkpIQGhqKuLg46aazq1evms0dGRkZiTVr1mDatGl47bXXEBwcjA0bNqB58+ZSmylTpiA7OxujRo1CWloaOnTogLi4OLNFIVavXo0xY8aga9euUCqVeOqpp7Bo0SJpv1qtxttvv42zZ89CCIE6depgzJgxePnll6U2qampGDlyJJKSkuDh4YE2bdpg7969aNq0qSV/ZHaFPbzWYTQaMXv27KLpx2q2gdGjdoWO1x9fD0Vh0TzVCkPRv8qcFDgd/rLURSqEECgoKOCNUHag+MWJopJuXCQisjcKwSQim4yMDLi5uSE9PR2urg+3GICt+vTTT7Fq1SrodDrExcXJXU6VtXz5cqxcuRKFHnWQV/+xCs/I4HT4SygNuSXuEyoHKIyF6B6Yi38H52DGfjdczix6rfztt9/Cy8uyPcn08MaMGYPjx48jKCgIy5cvl7scIiKr4ywNZFG8ac3yTpw4gc8//wImrQvy6j5aadOPlUd1mEu6KmC/BhFVdwy8ZFHFb6HyCdcyCgoK8Nbbb0MIE3KDOgJq6w4vYOC1D3zBSUTVHQMvWRTH8FrWunXrcO3qVRT4NoPJxff+B1SyzMxMq1+TKo6Bl4iqOwZesqjiHl4+4Va+tLQ0fLFqFYSDI/JrtpKlBgZe+8AXnERU3THwkkWxh9dy1q1bh7zcXOT7twRU8syUwCENRERkD2Sfloyqtn+O4X3zzTdx9epV1K5dG9OmTZOzNLuWn5+PjRs3Qjg4orBGQ9nqyMrKku3aRERE5cXASxb1z3k/r169irNnz8pUTdWxZ88eZGZmosC/JaCU79eYQxqIiMgecEgDWZSylEUL6OHs3LkTAFDo1UDWOtjDax+44AQRVXdMI0R2xmAwYP/+AzDp3CAc3WSthYHXPjDwElF1x8BLZGcuXryI3NwcGFz85S6FgddO8J0WIqru+FeQyM4Uj4E2OnnLWodSIRh47QQDLxFVd/wrSGRnrly5AgAw6T1lrUMJ9vDaCwZeIqru+FeQyM7cvHkTAGDSushah1IBZHGWBrugUqkAcD5sIqq+GHiJ7ExSUhKESgOotbLWoVQI5OXnw2AwyFoH3Z9aXTR1HVc8JKLqioGXyM7cvnMHJge93GVA+eeN/7m5ufIWQvdV3MPLFydEVF0x8BLZkYKCAmSkp0NobCDw/vkvx/HavuLAazQaZa6EiEgeXGmNyI7cuXMHACA0TpV+7iVLlpS4/aWx4wEA22/oEJ+sRWZhUddutqHo30mTJmH16tWVXg9VnuIhDQy8RFRdMfAS2ZGkpCQAgEnjbPVrF5gUKCj4awEDgaLPMzIyrF4LVUxx4OWQBiKqrhh4iezItWvXAAAmnWuln3v06NEl71A5lHkc7/y3fcWBt7CwUOZKiIjkwTG8RHbk0qVLAACTo7u8hfwNA6/tc3AoetHCIQ1EVF0x8BLZkdOnTwMKJUw6d7lLITvCHl4iqu4YeInsRE5ODs6dOweDkzegVMldjoQ9vLaPN60RUXXHwEtkJw4fPgyj0Qija4DcpZhh4LV9xYGXiKi6YuAlshN79uwBABjcAmWuxBwDr+1j4CWi6o6Bl8gOFBYW4tdff4XQOMHk5C13OWRniheeICKqrhh4iezA/v37kZmZiULPIEChuP8BVsQeXtvHHl4iqu4YeInsQFxcHACg0KuBzJXci4HX9jHwElF1x8BLZOMyMjKwd+9eGB09YdJ7yl0O2SEGXiKq7hh4iWzc7t27YTAYUOhdX+5SSsQeXtvHMbxEVN0x8BLZuF27dgEADJ71ZK6E7BUDLxFVdwy8RDYsLy8Phw4fhlHvDaFxkrucErGH1/Yx8BJRdcfAS2TDTp06BaPBAIOrv9ylkB1TKvmnnoiqN/4VJKthT2DFXb58GQBg0nvJW0gZ+LjaPgZeIqru+FeQrKawsFDuEuxOSkoKANjscAayDwy8RFTd8a8gWY3BYJC7BLvD3lMiIqKHx8BLVsMe3orz9i5aRlhRkCVzJaVjKLd9ChtbnY+IyNoYeMlqGHgrrnHjxgAAVcZNmSshIiKyXwy8ZDUMvBXXsGFD+Pr6QpNyCTDky10OERGRXWLgJaspKCiQuwS7o1Qq8fTTTwMmA7Q3DlvsOsJBB5PaESa1IwSK3v4WUBR9rdJY7LpkHRx2QkTVHQMvWU1+PnsoH0RMTAzq1KkDza2TUKVft8g1cpo/iexWA5DdagBMek8AgEnviexWA5Af2NYi1yTrYeAlouqOgZeshj28D8bBwQHTpk2Dg0YD/cWdUObclbsksjMmk0nuEoiIZMXAS1aTl5cndwl2Kzg4GK9NnQqFsRD6Mz8x9FKFMPASUXXHwEtWwx7eh9OlSxdMmTIFSmM+nE5vgSr9htwlkZ1g4CWi6o6Bl6yGPbwP74knnsB//vMfOKgU0J/9GQ5JxwCOz6T7MBqNcpdARCQrBl6yGgbeytGxY0csWrgQ3t7e0F3bD935bYCBP1sqHQMvEVV3DLxkNQy8ladJkyb45JNlCAsLg0PaVTif+J5DHKhUXNabiKo7Bl6ymtzcXLlLqFI8PDwwb948vPjii1Ab86E/+xO0l/cCRi7wQeYYeImoumPgJath4K18SqUS/fv3x7JlH6NevXrQ3D4N5xPrH6q316Rzg1HvBZPOrVztFQrFA1+LrINDGoioumPgJavJycmRu4Qqq379+vj4448xePBgqA25Rb29l359oOWI8+p3Rk6zPsir37nyCyVZcIYUIqruGHjJatjDa1kODg4YOnQoPv74YzRs2BCaO+fgfPw7qFMvy10ayYxDGoioumPgJavJzs6Wu4RqoUGDBliyZAleeOEFaBRGOJ7fDt357VAUWuYFB4c02L7CQo7rJqLqzSYC7+LFi1G3bl3odDqEh4dj3759ZbZft24dGjduDJ1Oh5CQEGzZssVsvxACM2bMgL+/PxwdHREVFYVz586ZtUlJScHAgQPh6uoKd3d3DB8+HFlZWdL+M2fOoEuXLvD19YVOp0O9evUwbdq0e5447lcL/YVDGqxHrVZjwIABWP7ZZ2jRogUcUi/D6fh6qFMuyV0ayaB4SIODg4PMlRARyUP2wLt27VpMmDABM2fOxKFDh9CyZUtER0fj1q1bJbbfu3cvBgwYgOHDh+Pw4cOIiYlBTEwMjh8/LrWZN28eFi1ahKVLlyIhIQFOTk6Ijo42mxZr4MCBOHHiBLZu3YpNmzZh9+7dGDVqlLTfwcEBgwYNws8//4wzZ85gwYIF+OSTTzBz5swK1UJ/YQ+v9dWqVQsLFizAuHHjoFMJOF7YAd2FXQ80trc07OG1fcUv1DUajcyVEBHJQyGEvMs0hYeHo23btvjwww8BFC2BGRgYiLFjx+LVV1+9p32/fv2QnZ2NTZs2Sdvat2+P0NBQLF26FEIIBAQEYOLEiZg0aRIAID09Hb6+vlixYgX69++PU6dOoWnTpti/fz/CwsIAAHFxcejRoweuX7+OgICAEmudMGEC9u/fj19//bVctfxTfn4+8vP/ChoZGRkIDAxEeno6XF1dK/qjswtfffWV9LMIDAyEo6Mjzp49i4YNG2LZsmUyV1e93LhxA3PmzMGJEycgNM7IqdcJJhff+x6nTrkIxws7S93v4OCArVu3VmKlVNlmz56N7du3w8PDA+vXr5e7HCIiq5O1h7egoAAHDx5EVFSUtE2pVCIqKgrx8fElHhMfH2/WHgCio6Ol9pcuXUJSUpJZGzc3N4SHh0tt4uPj4e7uLoVdAIiKioJSqURCQkKJ1z1//jzi4uLQqVOnctfyT3PnzoWbm5v0ERgYWGK7qurvQ0bI+mrWrImFCxdi2LBhUBbmwOnMFjgkPvzSxOzhtX3FQxrYw0tE1ZVazovfuXMHRqMRvr7mvUy+vr44ffp0icckJSWV2D4pKUnaX7ytrDY+Pj5m+9VqNTw9PaU2xSIjI3Ho0CHk5+dj1KhRmD17drlr+aepU6diwoQJ0tfFPbzVRVZWFmrUqCF3GdWaWq3GoEGDEBoaiv/Mno271/dDlXMHeUGPAsqy/xwsWbKkxO2xsbEWqJQqU/E7Swy8RFRdyT6G19atXbsWhw4dwpo1a7B582a8++67D3wurVYLV1dXs4/qQgGBwsJCmEwmuUshAC1atMAny5YV3dCWcgn60z8+8CwO7OG1fcU9vCqVSuZKiIjkIWsPr7e3N1QqFZKTk822Jycnw8/Pr8Rj/Pz8ymxf/G9ycjL8/f3N2oSGhkpt/nlTnMFgQEpKyj3XLe6Bbdq0KYxGI0aNGoWJEydCpVLdtxb6i5ODQFahgoHXhnh6emL+/PmYP38+4uLioD+9BTmNnoDQ6EtsP3r06BK3a7VaS5ZJlaA48PLFCRFVV7L28Go0GrRp0wbbtm2TtplMJmzbtg0RERElHhMREWHWHgC2bt0qtQ8KCoKfn59Zm4yMDCQkJEhtIiIikJaWhoMHD0pttm/fDpPJhPDw8FLrNZlMZr2U96uF/uKkLhonyiVObYuDgwNeeeUV/Pvf/4YyLx36M1sq3NPLEGX7/n6zLBFRdSRrDy9QNPPB4MGDERYWhnbt2mHBggXIzs7G0KFDAQCDBg1CzZo1MXfuXADA+PHj0alTJ8yfPx89e/bEV199hQMHDkh3/CsUCsTGxuLNN99EcHAwgoKCMH36dAQEBCAmJgYA0KRJE3Tv3h0jR47E0qVLUVhYiDFjxqB///7SDA2rV6+Gg4MDQkJCoNVqceDAAUydOhX9+vWT5rK8Xy30F2cHE5JzVQy8NkihUGDkyJFQKpVYtWoVHM/+jJzGPQBV+eZsZeC1fVxamIiqO9kDb79+/XD79m3MmDEDSUlJCA0NRVxcnHQz2NWrV6FU/tURHRkZiTVr1mDatGl47bXXEBwcjA0bNqB58+ZSmylTpiA7OxujRo1CWloaOnTogLi4OOh0OqnN6tWrMWbMGHTt2hVKpRJPPfUUFi1aJO1Xq9V4++23cfbsWQghUKdOHYwZMwYvv/xyhWqhIs4O7OG1ZQqFAsOHD0d6ejp++OEH6C7vQV69TkA5wiwDr+1jDy8RVXeyz8NbnWVkZMDNza1azMP7iF8e9iTp4Ovri+TkZM7Da6MMBgNefvllHDt2DLlBj8LgHXzfeXj1ej1XGLRx//rXv5CRkYGgoCAsX75c7nKIiKyOszSQVbiwh9cuqNVqTJ8+HXq9Ho5XE8o1npc9vLYvL489vERUvTHwklW4OBTd6GcwGGSuhO7Hx8enaJltYwE0Nw7ftz0Dr20TQqCwkGN4iah6Y+Alq3DVsIfXnvTq1QuBgYHQ3DkLFOaV2ZaB17YVFBSAI9eIqLpj4CWrKO7hZeC1D2q1GgMGDACECerUy2W2ZeC1bbxhjYiIgZesRK0E9GrBwGtHHnvsMej1ejhklrxUdjEGXtuWl1d2Dz0RUXXAwEtW4+Jg4hheO6LT6fDII4/ctx0Dr21j4CUiYuAlK3LXmNjDa2eeeeYZNGrUqMw2DLy2jYGXiIiBl6zIVWOSuwSqoIYNG2Ly5Mlyl0EPgYGXiIiBlyzs7wsSuGkZeImsLTf3/nMpExFVdQy8ZDE5OTm4evWq9LW7hlMjEVkbAy8REQMvWdC1a9fMvnbjkAYiq+OQBiIiBl6yoHPnzpl9zcBLZH05OTlyl0BEJDsGXrKYkydPmn3tzjG8RFbHwEtExMBLFiKEwKHDh822ubOHl8jqsrOz5S6BiEh2DLxkEdeuXUNSYqLZNlfetEZkdQy8REQMvGQhv/76KwDApHaUtikVgFLB0EtkTVlZWXKXQEQkuwcKvBcuXMC0adMwYMAA3Lp1CwDw448/4sSJE5VaHNmv7du3AwolTI7uZtvVXJSLyKqKe3iFykHmSoiI5FPhwLtr1y6EhIQgISEB3333ndR78Mcff2DmzJmVXiDZn/Pnz+PChQsodA+EUKrM9qn+7OE1mTiel8gaMjIyAKUaUKju35iIqIqqcOB99dVX8eabb2Lr1q3QaDTS9sceewy///57pRZH9unHH38EABR6B9+zT/VnD6/BYLBmSUTVVkZGBkxqrdxlEBHJqsKB99ixY3jyySfv2e7j44M7d+5USlFkv/Lz8/Hzzz9DOOhhdKt1z37Vn//jGHiJrCMtPR1CxcBLRNVbhQOvu7s7Ev9x9z0AHD58GDVr1qyUosh+/fbbb8jMzESBdzCguPe/V/GQBqPRaO3SiKodg8GA7KwsCAcGXiKq3iocePv3749XXnkFSUlJUCgUMJlM2LNnDyZNmoRBgwZZokayI5s3bwZQ8nAGgEMaiKwpLS0NACD+NlsKEVF1VOHAO2fOHDRu3BiBgYHIyspC06ZN0bFjR0RGRmLatGmWqJHsxO3bt3H48GEYXPwgdK4ltike0sAeXiLLS01NBQAIBwZeIqre1BU9QKPR4JNPPsH06dNx/PhxZGVloVWrVggOLrlHj6qP3bt3QwgBg2e9Utuo/xzSwB5eIsv7K/DqZK6E6MF07twZoaGhWLBggdylkJ2rcOAtVrt2bdSuXbsyayE7t2fPHgCAwaNOqW2UHNJAZDXFNxKbHPQyV0L08OrWrYvY2FjExsbKXQrZoQoHXiEEvvnmG+zYsQO3bt26Zz7V7777rtKKI/tRUFCAY8ePw6j3KvPt0+LAyyENRJZXHHiFxknmSoiI5FXhMbyxsbF4/vnncenSJTg7O8PNzc3sg6qnixcvorCgAEZn33K1Zw8vkeXdvn0bACDYw0t2IDs7G4MGDYKzszP8/f0xf/58aV/nzp1x5coVvPzyy1AoFFAoFMjOzoarqyu++eYbs/Ns2LABTk5OyMzMxOXLl6FQKPDVV18hMjISOp0OzZs3x65du8yOOX78OJ544gk4OzvD19cXzz//PKdarWIqHHi/+OILfPfdd/jxxx+xYsUKLF++3OyDqqdLly4BAEx6z3K1NxqNEEJYsiSiaq946XcTe3jJDkyePBm7du3C999/j59//hk7d+7EoUOHABS9e1yrVi3Mnj0biYmJSExMhJOTE/r3739P9li+fDmefvppuLi4mJ174sSJOHz4MCIiItC7d2/cvXsXQNFsJo899hhatWqFAwcOIC4uDsnJyXj22Wet982TxVV4SIObmxvq1Sv9piSqnpKTkwEAJq3LfVr+JTs7G87OzpYqiajaS0pKglDrAJWD3KUQlSkrKwv/+9//sGrVKnTt2hUAsHLlStSqVbSAkaenJ1QqFVxcXODn5ycdN2LECERGRiIxMRH+/v64desWtmzZgl9++cXs/GPGjMFTTz0FAPjoo48QFxeH//3vf5gyZQo+/PBDtGrVCnPmzJHaf/bZZwgMDMTZs2fRsGFDS3/7ZAUV7uGdNWsW/vOf/yA3N9cS9ZCdysjIAACICixhWjxHKBFVPiEEkpKS2LtLduHChQsoKChAeHi4tM3T0xONGjUq87h27dqhWbNmWLlyJQBg1apVqFOnDjp27GjWLiIiQvpcrVYjLCwMp06dAgD88ccf2LFjB5ydnaWPxo0bS3VR1VDhHt5nn30WX375JXx8fFC3bl04OJj3HBS//UDVS35+PgBAKMv/Xyo9PV169U5ElSstLQ35+fkwefjLXQqRRY0YMQKLFy/Gq6++iuXLl2Po0KFQKBTlPj4rKwu9e/fG22+/fc8+f3/+/lQVFQ68gwcPxsGDB/Hcc8/B19e3Qv+pqOoqHo+rvZoAZWEOTDo35NXvXOYxxb3CRFT5bty4AQAwaUteBIbIltSvXx8ODg5ISEiQpjxNTU3F2bNn0alTJwBF6wCUNMPPc889hylTpmDRokU4efIkBg8efE+b33//Xer1NRgMOHjwIMaMGQMAaN26Nb799lvUrVsXavUDz9ZKNq7Cj+zmzZvx008/oUOHDpaoh+xU8R8JZX4mVHlp5TqGgZfIcm7evAkAELryj6snkouzszOGDx+OyZMnw8vLCz4+Pnj99dehVP418rJu3brYvXs3+vfvD61WC29vbwCAh4cH+vbti8mTJ+Pxxx8v8Z3DxYsXIzg4GE2aNMH777+P1NRUDBs2DADwf//3f/jkk08wYMAATJkyBZ6enjh//jy++uorfPrpp1CpVNb5IZBFVXgMb2BgIFxd2WNA5v4a2lL+mRcyMzMtUwwR4fr16wDYw0v245133sGjjz6K3r17IyoqCh06dECbNm2k/bNnz8bly5dRv3591KhRw+zY4cOHo6CgQAqx//TWW2/hrbfeQsuWLfHbb79h48aNUmAOCAjAnj17YDQa8fjjjyMkJASxsbFwd3c3C9xk3yrcwzt//nxMmTIFS5cuRd26dS1QEtkjjUZT4WMYeIks59q1awAAkyPnRyf74OzsjC+++AJffPGFtG3y5MnS5+3bt8cff/xR4rE3btyAl5cX+vTpU+L+Jk2aICEhodRrBwcHc+GsKq7Cgfe5555DTk4O6tevD71ef89NaykpKZVWHNmPB3nLJysrywKVEBEAXLlyBUKlgVCXvvIhkb3LyclBYmIi3nrrLbzwwgsP1PlC1UOFA++CBQssUAbZu8LCwgofw8BLZBlGoxHXrl+HSecK8MZiqsLmzZuH//73v+jYsSOmTp0qdzlkwx5olgaif/preEL5n1xzcnIsUwxRNZeYmIjCggKYXD3kLoXIombNmoVZs2aVur9u3bpc1ZMAlDPwZmRkSDeq3e/Oet7QVj0lJSUVfVKBAf7Z2dkWqoaoeite6tvo6C5vIURENqJcgdfDwwOJiYnw8fGBu7t7iXPvCiGgUChKnCOPqjaj0YgzZ87AqHNDeXt4FRBcrY/IQi5fvgwAMDmyh5eICChn4N2+fTs8PT0BAMuXL0dgYOA9NymZTCZcvXq18iskm3f8+HFkZ2fD6NMYqqzb5TpGoQADL5GFXLx4EQCgvbYfpjvn77sIDBFRVVeuwFu8ygkADBs2TOrt/bu7d+8iKiqKY3yroS1btgAADB5B5Q68SjDwElnK+fPnIaCAKjcVUHAeUSKiCv8lLB668E9ZWVnQ6XSVUhTZj+vXr2Pr1q0wOXrA6OJX7uMUCqAgP9+ClRFVT3l5ebh+/Qag5OpQRETFyj1Lw4QJEwAACoUC06dPh16vl/YZjUYkJCQgNDS00gsk2yWEwKJFi2AymZBfs1WZ0x8tOeEMByWQWVjUptAEpKalYejQoVi+fLm1Siaq8i5evAghTBAqDRQmg9zl2JTx48fj1q1bVr2mj48PFi5caNVrEtG9yh14Dx8+DKAo5Bw7dsxscmeNRoOWLVti0qRJlV8h2azvv/8e+/btg8GtFgzudcpsm2P455sJRcE3LS3NMsURVVPnzp0r+kRZ4Vknq7xbt27hZmIShNbJKtdT5Ms7E82sWbOwYcMGHDlyRNY6AGDFihWIjY3l3/xyEkLghRdewDfffIPU1FQcPnwYsbGxCA0N5XoID6jcfxF37NgBABg6dCgWLlzI6cequaNHj+KDDz6EcNAhr24HTm5PZCOKA69g4C2R0Dohu8WzVrmW09GvrXIdW7Rz504MGTJEmjGkX79+6NGjh7xF2ZG4uDisWLECO3fuRL169eDt7W2R63Tu3LnEED1u3Djs2bMHx48fR5MmTUp80SSEwPz587Fs2TJcuXIF3t7eGD16NF5//XUARS9yir8HW1Dhv4h8+5kuXryI1157DUaTCbnBj0Fo9Pc/iIis4syZM0XjdzmGt0pKTU2Fg4MDnJ2d5S6lQhwdHeHoyGWuy+vChQvw9/dHZGSkbDUMGzYMCQkJOHr0aIn7x48fj59//hnvvvsuQkJCkJKSgpSUlFLPd/v2bbi4uMh2vxdv36UKuXTpEiZMnIisrCzkBj1aoRvVSsIVcIgqT0FBAS5eugSjo6fcpVAlMhgM2Lx5M5555hn4+/vjwoUL0r7r169jwIAB8PT0hJOTE8LCwpCQkFDiefbv349u3brB29sbbm5u6NSpEw4dOiTtF0Jg1qxZqF27NrRaLQICAjBu3Dhp/5IlSxAcHAydTgdfX188/fTT5f4eVqxYAXd3d+nrCxcuoE+fPvD19YWzszPatm2LX375Rdp/+vRp6PV6rFmzRtr29ddfw9HRESdPnizxGjt37oRCocC2bdsQFhYGvV6PyMjIoheBfxoyZAhiYmLMjouNjUXnzp2lrzt37oyxY8ciNjYWHh4e8PX1xSeffILs7GwMHToULi4uaNCgAX788cd7rr1582a0aNECOp0O7du3x/HjxwEULbTk6uqKb775xuzaGzZsgJOT099WKy2qcezYsbh69SoUCgXq1q1b4vebmpqKQYMGwcPDA3q9Hk888cRfQ5pQNHvWgAEDULNmTej1eoSEhODLL780u86uXbuwcOFCKBQKKBQKqUd+0aJF+L//+z/Uq1evxGufOnUKH330Eb7//nv861//QlBQENq0aYNu3bqV2B4omtHJ398fL774IuLj40ttZykMvFRup0+fxvjxsUhLTUVe3Udg8Kovd0lE9DcXL16E0WCA0ckyb3+SdR07dgwTJ05ErVq1MGjQINSoUQM7duxAy5YtARTNjtSpUyfcuHEDGzduxB9//IEpU6bAZDKVeL7MzEwMHjwYv/32G37//XcEBwejR48eUtj69ttv8f777+Pjjz/GuXPnsGHDBoSEhAAADhw4gHHjxmH27Nk4c+YM4uLi0LFjxwf+3rKystCjRw9s27YNhw8fRvfu3dG7d29pPv/GjRvj3XffxejRo3H16lVcv34dL774It5++200bdq0zHO//vrrmD9/Pg4cOAC1Wo1hw4ZVuL6VK1fC29sb+/btw9ixY/HSSy/hmWeeQWRkJA4dOoTHH38czz//PHJycsyOmzx5MubPn4/9+/ejRo0a6N27NwoLC+Hk5IT+/fvf8y758uXL8fTTT8PFxUXatnDhQsyePRu1atVCYmIi9u/fX2KNQ4YMwYEDB7Bx40bEx8dDCIEePXqgsLAQQNGMLW3atMHmzZtx/PhxjBo1Cs8//zz27dsnXSciIgIjR45EYmIiEhMTERgYWK6fzw8//IB69eph06ZNCAoKQt26dTFixIgye3gHDhyIVatWITU1FY899hgaNWqEOXPm4Nq1a+W65sPiIC8ql4SEBMyYORP5+fnIDXoUBu9guUsion8o7skyOnlDlWXd2Qiocty9exerVq3CypUrceLECfTo0QNLlixBr169zG4WB4A1a9bg9u3b2L9/v7Q4VIMGDUo992OPPWb29bJly+Du7o5du3ahV69euHr1Kvz8/BAVFQUHBwfUrl0b7dq1AwBcvXoVTk5O6NWrF1xcXFCnTh20atWq1Gt17txZ6i0sScuWLaXgDgBvvPEG1q9fj40bN2LMmDEAgNGjR2PLli147rnnoNFo0LZtW4wdO7bUcxb773//K60f8Oqrr6Jnz57Iy8ur0FvpLVu2xLRp0wAAU6dOxVtvvQVvb2+MHDkSADBjxgx89NFHOHr0KNq3by8dN3PmTKmXc+XKlahVqxbWr1+PZ599FiNGjEBkZCQSExPh7++PW7duYcuWLWY92wDg5uYGFxcXqFQq+PmV/C7quXPnsHHjRuzZs0ca9rB69WoEBgZiw4YNeOaZZ1CzZk2zyQTGjh2Ln376CV9//TXatWsHNzc3aDQa6PX6Uq9TmosXL+LKlStYt24dPv/8cxiNRrz88st4+umnsX37dgBFgXzIkCHSMWq1Gj179kTPnj2Rnp6Or7/+Gl988QVmzJiBzp07Y/DgwXj66actNvTFJnp4Fy9ejLp160Kn0yE8PFx69VGadevWoXHjxtDpdAgJCZEWPigmhMCMGTPg7+8PR0dHREVFmXXzA0BKSgoGDhwIV1dXuLu7Y/jw4cjKypL279y5E3369IG/vz+cnJwQGhqK1atXm51jxYoV0tsAxR9VbS5iIQS+++47vPrqVOQXGJDToGulhl0OaSCqPMWB18QeXrv1wQcfIDY2Fs7Ozjh//jzWr1+Pvn373hN2AeDIkSNo1aqVFHbvJzk5GSNHjkRwcDDc3Nzg6uqKrKwsqVf1mWeeQW5uLurVq4eRI0di/fr1MBiKprbr1q0b6tSpg3r16uH555/H6tWr7+ndrIisrCxMmjQJTZo0gbu7O5ydnXHq1Kl7Vmz97LPPcPToURw6dEh6zr2fFi1aSJ/7+/sDQIWno/v7OVQqFby8vKTebgDw9fUt8bwRERHS556enmjUqBFOnToFAGjXrh2aNWuGlStXAgBWrVqFOnXqPFBP+alTp6BWqxEeHi5t8/LyMrue0WjEG2+8gZCQEHh6esLZ2Rk//fRTpayKazKZkJ+fj88//xyPPvooOnfujP/973/YsWOH2RCS0ri5uWHkyJHYvXs39u7di0uXLmHQoEH46aefHrq20sgeeNeuXYsJEyZg5syZOHToEFq2bIno6OhS/3Pu3bsXAwYMwPDhw3H48GHExMQgJiZGGicDAPPmzcOiRYuwdOlSJCQkwMnJCdHR0cjLy5PaDBw4ECdOnMDWrVuxadMm7N69G6NGjTK7TosWLfDtt9/i6NGjGDp0KAYNGoRNmzaZ1ePq6iq9FZCYmIgrV65U8k9IPoWFhXjvvfeK5tpVa5HduAeM7rXlLouISnH69GlA5QCTzk3uUugBjRo1Cm+88QaSkpLQrFkzDB06FNu3by9xmEJFe8IGDx6MI0eOYOHChdi7dy+OHDkCLy8vFBQUAAACAwNx5swZLFmyBI6Ojhg9ejQ6duyIwsJCuLi44NChQ/jyyy/h7++PGTNmoGXLlg88zdikSZOwfv16zJkzB7/++iuOHDmCkJAQqZZif/zxB7Kzs5GdnY3ExMRyndvBwUH6vDggF//8lErlPR0txUMASjtH8XnKOm95jRgxAitWrABQNJxh6NCh5QrxD+Kdd97BwoUL8corr2DHjh04cuQIoqOj7/kZPwh/f3+o1Wo0bNhQ2takSRMAKFegzsvLw7p169C7d2906NAB3t7eWLJkCbp27frQtZVG9sD73nvvYeTIkRg6dCiaNm2KpUuXQq/X47PPPiux/cKFC9G9e3dMnjwZTZo0wRtvvIHWrVvjww8/BFDUY7hgwQJMmzYNffr0QYsWLfD555/j5s2b2LBhA4CiV0ZxcXH49NNPER4ejg4dOuCDDz7AV199hZs3bwIAXnvtNbzxxhuIjIxE/fr1MX78eHTv3h3fffedWT0KhQJ+fn7SR/GrPnt39+5dxMbG4ocffoBR74Xspv9irxGRDcvJycGlS5dh0HtxOWE7FhAQgGnTpuHs2bOIi4uDRqNB3759UadOHbz66qs4ceKE1LZFixY4cuRImeMm/27Pnj0YN24cevTogWbNmkGr1eLOnTtmbRwdHdG7d28sWrQIO3fuRHx8PI4dOwag6C3pqKgozJs3D0ePHsXly5elt68ras+ePRgyZAiefPJJhISEwM/P754hECkpKRgyZAhef/11DBkyBAMHDnzoJelr1KhxT3CuzHmKf//9d+nz1NRUnD17VgqCAPDcc8/hypUrWLRoEU6ePInBgwc/0HWaNGkCg8FgdoPi3bt3cebMGWmM8549e9CnTx8899xzaNmyJerVq4ezZ8+anUej0cBoNFb4+o888ggMBoPZDZTF565Tp+R5+YUQ+PXXXzFy5Ej4+flhwoQJaN68OY4ePYqEhAS89NJLZmOZK5usY3gLCgpw8OBBTJ06VdqmVCoRFRVV6h188fHx0qpvxaKjo6Uwe+nSJSQlJSEqKkra7+bmhvDwcMTHx6N///6Ij4+Hu7s7wsLCpDZRUVFQKpVISEjAk08+WeK109PTzf7jAkVvy9SpUwcmkwmtW7fGnDlz0KxZsxKPz8/PR/7fltPNyMgosZ3cjh49ipkzZyE1NQWFXvWRV/cRTmJPZOPOnDkDIUwwOvnIXYpNU+RnW21+3KKFJx78CTwyMhKRkZFYuHAhNmzYgBUrVuDdd9/F4cOHERISggEDBmDOnDmIiYnB3Llz4e/vj8OHDyMgIMDsrfViwcHB+OKLLxAWFoaMjAxMnjzZrJd4xYoVMBqNCA8Ph16vx6pVq+Do6Ig6depg06ZNuHjxIjp27AgPDw9s2bIFJpMJjRo1eqDvLTg4GN999x169+4treD6z97SF198EYGBgZg2bRry8/PRqlUrTJo0CYsXLwYArF+/HlOnTi16Z6OcHnvsMbzzzjv4/PPPERERgVWrVuH48eNljkeuiNmzZ8PLywu+vr54/fXX4e3tbTYrhIeHB/r27YvJkyfj8ccfR61atQAAXbt2xZNPPimNX76f4OBg9OnTByNHjsTHH38MFxcXvPrqq6hZsyb69Okjtfnmm2+wd+9eeHh44L333kNycrLZTX9169ZFQkICLl++DGdnZ3h6ekKpVOL8+fPIyspCUlIScnNzpRcFTZs2hUajQVRUFFq3bo1hw4ZhwYIFMJlM+L//+z9069bNrNf371atWoUXXngBTz75JL7++mspd1mLrCnmzp07MBqN9/SK+vr6lvofOCkpqcT2SUlJ0v7ibWW18fExf1JQq9Xw9PSU2vzT119/jf379+Pjjz+WtjVq1AifffYZWrRogfT0dLz77ruIjIzEiRMnpP/Efzd37lz85z//KfH8tkAIgW+++QYfffQRTALIqx2OQp+mXFSCyA4Uj9szOdeQuRLb9c+/+5bnUinX1Ol06N+/P/r374+bN29Kc/BqNBr8/PPPmDhxInr06AGDwYCmTZtKgfCf/ve//2HUqFFo3bo1AgMDMWfOHLObmtzd3fHWW29hwoQJMBqNCAkJwQ8//AAvLy+4u7vju+++w6xZs5CXl4fg4GB8+eWXpXbw3M97772HYcOGITIyEt7e3njllVfMOoE+//xzbNmyBYcPH4ZarYZarcaqVavQoUMH9OrVC0888QTS09PLNV7076KjozF9+nRMmTIFeXl5GDZsGAYNGiT1Yj+st956C+PHj8e5c+cQGhqKH3744Z7x18OHD8eaNWvMZo+4cOHCPb3t97N8+XKMHz8evXr1QkFBATp27IgtW7ZIQy+mTZuGixcvIjo6Gnq9HqNGjUJMTAzS09Olc0yaNAmDBw9G06ZNkZubi0uXLkkzLuzatUtqV/yCoHi/UqnEDz/8gLFjx6Jjx45wcnLCE088gfnz55dab9euXZGUlCTbwmUKIeNdQzdv3kTNmjWxd+9es1ejU6ZMwa5du0qcS1Cj0WDlypUYMGCAtG3JkiX4z3/+g+TkZOzduxePPPIIbt68KQ1WB4Bnn30WCoUCa9euxZw5c7By5cp7flF8fHzwn//8By+99JLZ9h07dqBXr1746KOPMGjQoFK/n8LCQjRp0gQDBgzAG2+8cc/+knp4AwMDkZ6eLvvKdVlZWZg3bx52794NodEjt14XGF0qPjxDf+J7qHLuwqj3Qk6zPtCd/RkO6ddLbe/q6oqNGzc+TOlkYefPn8eIESNK3e/h4YH169dbsSIqybRp0/Dbb78hq2V/CI3+nt9F58NrUC/Qn4sHkSw+/vhjvPHGG7h+vfTnA3u2c+dOdOnSBampqWbzDZfkiy++wMsvv4ybN2+WeDMiWYasPbze3t5QqVRITk42256cnFzqFBl+fn5lti/+Nzk52SzwJicnIzQ0VGrzz5viDAYDUlJS7rnurl270Lt3b7z//vtlhl2gaJB7q1atcP78+RL3a7VaaLXaMs8hh/Pnz2PGzJm4eeMGDC7+yKvfGcKBK+IQ2QshBE6cOAGTxpkrH5LNuXbtGrZs2fLAvcFVRU5ODhITE/HWW2/hhRdeYNi1MlnvbNBoNGjTpg22bdsmbTOZTNi2bVuJ44+Aoik//t4eALZu3Sq1DwoKgp+fn1mbjIwMJCQkSG0iIiKQlpaGgwcPSm2K74L9+xQfO3fuRM+ePfH222+bzeBQGqPRiGPHjpkFbVu3ZcsWvDR6NG7euIF8/xbIbRTNsEsl0igF9Oq/xtipVCp4eHjctzeDLC8pKQmpqakwcjgD2aDWrVvjypUrePvtt+UuRVbz5s1D48aN4efnZ3bvElmH7HciTZgwAYMHD0ZYWBjatWuHBQsWSMv3AcCgQYNQs2ZNzJ07F0DR2s2dOnXC/Pnz0bNnT3z11Vc4cOAAli1bBqBo1oTY2Fi8+eabCA4ORlBQEKZPn46AgABp4HiTJk3QvXt3jBw5EkuXLkVhYSHGjBmD/v37IyAgAMBfwxjGjx+Pp556Shrbq9FopDkPZ8+ejfbt26NBgwZIS0vDO++8gytXrpT59q+tyM3NxcKFCxEXFweotcgJ7gaje/lWWKHq6bGaeQh2M+CD40U34dSvX1/6vSN5Fd+5zxvWyBbdvn1b7hIsrnPnzvedV37WrFmYNWuWdQqie8geePv164fbt29jxowZSEpKQmhoKOLi4qSbzq5evWp2F19kZCTWrFmDadOm4bXXXkNwcDA2bNiA5s2bS22mTJmC7OxsjBo1CmlpaejQoQPi4uLMFoVYvXo1xowZg65du0KpVOKpp57CokWLpP0rV65ETk4O5s6dK4VtAOjUqRN27twJoGjKkZEjRyIpKQkeHh5o06YN9u7de99lD+V2+fJlzJw5E1euXIHRqQZy63eB0DrLXRbZASPXCbFJxfOQG50ZeImISiLrTWvVXUZGBtzc3Kx609pPP/2E9957D/n5+SjwbYb8WmGAUvVQ59QfXw9FYdGiHgpDHhQQEFBAqHVQmAqhMBlKPZY3rdm+4pvWugfmoo6LAR+fLOrhbdiwIXt4bcSIESNw/uJlZLZ6Dvizg4A3rRER/UX2Hl6yjtzcXCxYsKBo2T61BrkNusLgUfLk0BWlKMyD0mA+GbgCAgpDLgQnwK9SjCZOUWdrcnJycOHCRRicfaSwS0RE5hh4q4ELFy5g1qxZuHbt2p9DGDpDaC23mglVXRzSYHtOnjxZtOCEc9VY5ZGIyBIYeKswIQQ2bNiAJUuWoLCwEPl+ISio2Ya9QPTATAy8Nufo0aMA8EDzZhMRVRcMvFVURkYG5s2bh99++w3CQYfchl1gdLt39TeiijAKDmmwNUWBV8Eb1sph/Pjx98zBbmk+Pj5YuHChVa9JRPdi4K2Cjh8/jtmzZ+PWrVswuPojL6gTJ6OnSmG6fxOyooKCApw8eRJGvSeg4iT293Pr1i0kJ92El9Y6/5Pv5sv7btqsWbOwYcMGHDlyRNY6AGDFihWIjY1FWlqa3KXYhc6dOyM0NBQLFiwotU1OTg6ef/55bN26FZmZmUhNTUVoaChiY2MRGxtrtVrtBQNvFWIymfDVV1/h008/hUkI5NdsgwL/FoCCvXJUOTini205c+YMCgoKYPQoeWVKupeX1oT5kWlWudbEve5WuY4t2rlzJ4YMGYLLly8DKJqCtEePHvIWVcWsXLkSv/76K/bu3Qtvb2+4ublZ5Dp169a9J0Tn5eXhxRdfxMGDB3Hq1Cn06tULGzZsuOfY/Px8zJ49G6tWrUJSUhL8/f0xY8YMDBs2DEDRi7LLly9jxYoVFqn97xh4q4iMjAzMmTMHv//+O4TGCbn1OnNMH1U6juG1LX/88QcAwOjCwFtdpKamwsHBAc7O9jV3uqOjIxwduYpnZbpw4QKaNGlitg6BtRiNRjg6OmLcuHH49ttvS2337LPPIjk5Gf/73//QoEEDJCYmwmQq/R2WmzdvwsfHB2p15cdT3r1UBVy4cAEvvPACfv/9dxjcaiGrWR+GXbIIBl7bUhx4Dfx9r9IMBgM2b96MZ555Bv7+/rhw4YK07/r16xgwYAA8PT3h5OSEsLAwJCQklHie/fv3o1u3blJvYKdOnXDo0CFpvxACs2bNQu3ataHVahEQEIBx48ZJ+5csWYLg4GDodDr4+vri6aefLvf3sGLFCrNlyC9cuIA+ffrA19cXzs7OaNu2LX755Rdp/+nTp6HX67FmzRpp29dffw1HR0ecPHmyxGvs3LkTCoUC27ZtQ1hYGPR6PSIjI3HmzBmpzZAhQ6RVV4vFxsaic+fO0tedO3fG2LFjERsbCw8PD/j6+uKTTz6RVoF1cXFBgwYN8OOPP95z7c2bN6NFixbQ6XRo3769tChMdnY2XF1d8c0335hde8OGDXByckJmZuY934/BYMCYMWPg5uYGb29vTJ8+XVrNrXPnzpg/fz52794NhUJhVv/fXb16FX369IGzszNcXV2lAFrex6Fz5864cuUKXn75ZSgUCij+fMfYyckJH330EUaOHAk/v5JfcMfFxWHXrl3YsmULoqKiULduXUREROCRRx4psT0AfPLJJ6hVqxYmTZqEY8eOldruQTDw2rk9e/Zg9P/9HxITE5Ef0Aq5wd0Ate7+BxI9AAEOj7EVhYWFOHr0KIyOHvydr6KOHTuGiRMnolatWhg0aBBq1KiBHTt2oGXLlgCArKwsdOrUCTdu3MDGjRvxxx9/YMqUKaX2oGVmZmLw4MH47bff8PvvvyM4OBg9evSQwta3336L999/Hx9//DHOnTuHDRs2ICQkBABw4MABjBs3DrNnz8aZM2cQFxeHjh07PvD3lpWVhR49emDbtm04fPgwunfvjt69e+Pq1asAgMaNG+Pdd9/F6NGjcfXqVVy/fh0vvvgi3n777fuuZvr6669j/vz5OHDgANRqtfT2eUWsXLkS3t7e2LdvH8aOHYuXXnoJzzzzDCIjI3Ho0CE8/vjjeP7555GTk2N23OTJkzF//nzs378fNWrUQO/evVFYWAgnJyf079//noVfli9fjqeffhouLvdOFbpy5Uqo1Wrs27cPCxcuxHvvvYdPP/0UAPDdd99h5MiRiIiIQGJiIr777rt7jjeZTOjTpw9SUlKwa9cubN26FRcvXkS/fv2kNvd7HL777jvUqlULs2fPRmJiIhITE8v9M9y4cSPCwsIwb9481KxZEw0bNsSkSZOQm5tb6jGvvPIKFi5ciFOnTqF169Zo3bo1Fi1aVCnLU3NIgx3btGkT5s9/D0KprNSFJIhKwx5e23Hq1Cnk5+fD6NtA7lKoEt29exerVq3CypUrceLECfTo0QNLlixBr169oNGY35i4Zs0a3L59G/v374enpycAoEGD0v8/PPbYY2ZfL1u2DO7u7ti1axd69eqFq1evws/PD1FRUXBwcEDt2rXRrl07AEU9hU5OTujVqxdcXFxQp04dtGrVqtRrde7cWRq/W5KWLVtKwR0A3njjDaxfvx4bN27EmDFjAACjR4/Gli1b8Nxzz0Gj0aBt27YYO3Zsqecs9t///hedOnUCALz66qvo2bMn8vLyoNOV/4Vhy5YtMW3aNADA1KlT8dZbb8Hb2xsjR44EAMyYMQMfffQRjh49ivbt20vHzZw5E926dQNQFFhr1aqF9evX49lnn8WIESMQGRmJxMRE+Pv749atW9iyZYtZj+rfBQYG4v3334dCoUCjRo1w7NgxvP/++xg5ciQ8PT2h1+uh0WhK7WHdtm0bjh07hkuXLiEwMBAA8Pnnn6NZs2bYv38/2rZte9/HwdPTEyqVCi4uLqVepzQXL17Eb7/9Bp1Oh/Xr1+POnTsYPXo07t69KwX/WbNmmR2j0+nQr18/9OvXD7du3cKaNWuwYsUKTJo0CT169MDgwYPRu3fvBxrywB5eO7Vx40a8++67MKk1yG7Uw27DLle2ti98tGxH8Z33Bhd/eQuhSvXBBx8gNjYWzs7OOH/+PNavX4++ffveE3aBov8DrVq1ksLu/SQnJ2PkyJEIDg6Gm5sbXF1dkZWVJfXmPfPMM8jNzUW9evUwcuRIrF+/HgZD0dLw3bp1Q506dVCvXj08//zzWL169T29mxWRlZWFSZMmoUmTJnB3d4ezszNOnTol1VLss88+w9GjR3Ho0CGsWLFCeku9LC1atJA+9/cv+v2o6HR0fz+HSqWCl5eX1NsNAL6+viWeNyIiQvrc09MTjRo1wqlTpwAA7dq1Q7NmzbBy5UoAwKpVq1CnTp1Se8rbt29v9v1GRETg3LlzMBqN5foeTp06hcDAQCnsAkDTpk3h7u4u1VTex+FBmEwmKBQKrF69Gu3atUOPHj3w3nvvYeXKlWX28hbz8fFBbGwsDh06hO+//x7x8fHo27evNEykohh47dCvv/6K999/H8LBEdmNesLk5C13SVRN8PWJ7Th48CAABcfrVzGjRo3CG2+8gaSkJDRr1gxDhw7F9u3bSxymUNGbwAYPHowjR45g4cKF2Lt3L44cOQIvLy8UFBQAKOpRPHPmDJYsWQJHR0eMHj0aHTt2RGFhIVxcXHDo0CF8+eWX0p32LVu2fOBpxiZNmoT169djzpw5+PXXX3HkyBGEhIRItRT7448/kJ2djezs7HK/ne7g4CB9XhwYi39+SqXyno6WwsLCMs9RfJ6yzlteI0aMkGYkWL58OYYOHVquEG8p5X0cHoS/vz9q1qxpNntEkyZNIITA9evX73t8ZmYmli9fjsceewy9e/dG8+bNsXLlyvsOaSkNA6+duXnzJv47Zw6EUoWchtEQjpaZhoSoJMy7tiE3NxfHT5yA0ckbUGvlLocqUUBAAKZNm4azZ88iLi4OGo0Gffv2RZ06dfDqq6/ixIkTUtsWLVrgyJEjSElJKde59+zZg3HjxqFHjx5o1qwZtFot7ty5Y9bG0dERvXv3xqJFi7Bz507Ex8dLNw+p1WpERUVh3rx5OHr0KC5fvozt27c/0Pe5Z88eDBkyBE8++SRCQkLg5+d3zxCIlJQUDBkyBK+//jqGDBmCgQMHlqtnsCw1atS4JzhX5jzFv//+u/R5amoqzp49iyZNmkjbnnvuOVy5cgWLFi3CyZMnMXjw4FLP9c+bD4vHXatUqnLV0qRJE1y7dg3Xrl2Ttp08eRJpaWlSaCzP46DRaMrdq/x3jzzyCG7evImsrCxp29mzZ6FUKlGrVskLYRmNRvz444/497//DV9fX7z11lvo2rUrLl68iG3btmHQoEElvttRHhzDa2cWLlyIvNxc5NbrBJO+fG9jEVUWBl7bcOzYMRgNBhhqcDhDRd3NV1ptfty7+Uo8TP97ZGQkIiMjsXDhQmzYsAErVqzAu+++i8OHDyMkJAQDBgzAnDlzEBMTg7lz58Lf3x+HDx9GQECA2VvrxYKDg/HFF18gLCwMGRkZmDx5slkv8YoVK2A0GhEeHg69Xo9Vq1bB0dERderUwaZNm3Dx4kV07NgRHh4e2LJlC0wmExo1avRA31twcDC+++479O7dGwqFAtOnT7+nt/TFF19EYGAgpk2bhvz8fLRq1QqTJk3C4sWLAQDr16/H1KlTcfr06XJf97HHHsM777yDzz//HBEREVi1ahWOHz9e5njkipg9eza8vLzg6+uL119/Hd7e3mazQnh4eKBv376YPHkyHn/8cSn4de3aFU8++aQ0fhkoGjc9YcIEvPDCCzh06BA++OADzJ8/v9y1REVFISQkBAMHDsSCBQtgMBgwevRodOrUCWFhYQDK9zjUrVsXu3fvRv/+/aHVauHtXfSu8smTJ1FQUICUlBRkZmZKLxxCQ0MBAP/+97/xxhtvYOjQofjPf/6DO3fuYPLkyRg2bFip707MmTMH8+fPR79+/fDLL78gMjKy3N/v/TDw2pGzZ88iISEBBtcAGDzryV2OmSVLlpS4/aX/G1PidrJTTLw24cCBAwAAo2tNmSuxLz4+1l1+2beSrqnT6dC/f3/0798fN2/elObg1Wg0+PnnnzFx4kT06NEDBoMBTZs2lQLhP/3vf//DqFGj0Lp1awQGBmLOnDmYNGmStN/d3R1vvfUWJkyYAKPRiJCQEPzwww/w8vKCu7s7vvvuO8yaNQt5eXkIDg7Gl19+iWbNmj3Q9/Tee+9h2LBhiIyMhLe3N1555RVkZGRI+z///HNs2bIFhw8fhlqthlqtxqpVq9ChQwf06tULTzzxBNLT082mHCuP6OhoTJ8+HVOmTEFeXh6GDRuGQYMGVdoUWG+99RbGjx+Pc+fOITQ0FD/88MM9PZLDhw/HmjVrzGaPuHDhwj297YMGDUJubi7atWsHlUqF8ePHY9SoUeWuRaFQ4Pvvv8fYsWPRsWNHKJVKdO/eHR988IHU5n6PA1AU4l944QXUr18f+fn50pCQHj164MqVK1K74hcNxfudnZ2xdetWjB07FmFhYfDy8sKzzz6LN998s9San3/+eUyePLlCNxiWl0LwriHZZGRkwM3NDenp6XB1db1v+08//RSrVq1CTnA3GN0D79veWpwOf4mli0p+1fnS/42BQpQ+xsnFxQU//PCDpUqjSnD+/HmMGDEC3QNz4aAU+OFK0TLVDRs2xLJly2SurnoaNmwYLl6+isxWAwFlyW9v6k98D1XOXRj1Xshp1gfOh9egXqD/PdMiEVnDxx9/jDfeeKNcYzft0c6dO9GlSxekpqaazTdcki+++AIvv/wybt68+cBvz1PFsYfXjpw/fx6Aba6qNHr06JJ3KMoeJs7XW0QVc/fuXVy8eBEGt1qlhl0iW3Lt2jVs2bLlgXuDq4qcnBwkJibirbfewgsvvMCwa2W8ac2O5OXlFX3CJzmiamvfvn0AAAOHM5CdaN26Na5cuYK3335b7lJkNW/ePDRu3Bh+fn6YOnWq3OVUO+zhtSM1atQAACjysyB09x8CQVTZuNKa/KTA617yXc5EtqYyVsmydZ07d77vO5azZs26Z6EFsh728NqR4jsf1alXym5IZCEcgSIvg8GA/fsPwKR1gdDyRS8RUXkx8NqRjh07QqfTQXvrJGAyyF1OpeAYXqLyO3XqFLKyMmFwqwnIOFk9EZG9YeC1Iy4uLnjqqaegKMiG5uYfcpdDRFZWPBG9wa3kWVr0x9fD6fCXcDr8JZQ5RYsRKHNS4HT4S8CQb7U6iYhsDQOvnXnuuefg6+cHbeJRqDKT5C6HiKwoPj4eUKpgdC15wQlFYR6UhlwoDblQ/DlpsgICSkMuOIkyEVVnDLx2xtHREdOnTYNKpYTjhR1Q5Gfd/yAbxiENROVz69YtXLhwAQYXf0DJ+42JiCqCgdcONW/eHOPGjYOiMBf6sz8Dhjy5SyIiC4uPjwcAGNxry1wJEZH9YeC1U3369MGAAQOgzEuD/sxPdjs+jz28ROXzV+C1nVUWiYjsBQOvHRs1ahT+9a9/QZVzF/ozcVAUsqeXqCrKzc3FwUOHYNR7Qmic5C6HiMjuMPDaMYVCgdjYWMTExBSF3tOb7X5ML9k29sfL4+DBgygsKOBwBiKiB8TAa+eUSiXGjx+Pf//731DmpcPp9CYos+/KXVa5cUgD0f3t2bMHAMfvEhE9KAbeKkChUGDUqFEYN24clIW5cDq9GerUy3KXRUSVwGg0Yu/eeAgHPUx6L7nLISKyS5zbpgrp27cv/Pz8MHv2G8D57cj3b4mCmq0AhWVf1wgHHUx/fq4w5EEBAQEFhFoHhakQECYsWbKkxGMnTJhg0dqI7N2pU6eQnp6GwhqNuLoaEdEDYg9vFRMZGYmPPloC/4AAaBP/gOPZn6EozLXoNXOaP4nsVgOQ3WoATHpPAIBJ74nsVgNgcPEr81gOabAvfLSsj8MZiIgeHnt4q6CgoCAs+/hjvP322/jtt9/gdGIDcoM6wuhWU7aaRo8eXeJ2rVZr5UqI7MvevXsBpbrU1dWIiOj+2MNbRbm4uOCNN97A2LFj4SAKoT/7E7RXfgeMBrlLM8MeXqLS3bhxA1euXEGha02urkZE9BAYeKswhUKBp556CkuXLkXdoCBobp2E08kNUGUmyV0a2Su+PrEqLjZBRFQ5GHirgQYNGuDjpUsxYMAAqPKzoD+9BdrLewFDgdylsYfXzvDRsq7iwGtk4CUieigMvNWEVqvFCy+8gCVLFiMoqB40t0/D+fh3UN+9CDB0Ujnxf4r15OTk4Mgff8Do5A3h4Ch3OUREdo2Bt5pp0qQJPvlkGUaOHAmNwgDHizvhePYnKHLTZamHPbxEJTt06BCMBgMMbuzdJSJ6WAy81ZBarcbAgQPx+cqVaN++PdQZN+F8Yj001/YDxkK5yyMbxtcn1rNv3z4AgEHG2VWIiKoKBt5qzN/fH2+99RbmzJkDfz9faJOOwfnYt1DfOc9kQySzffv2Qai1MDl5y10KEZHdY+AlREZGYsWKFRg+fDh0SiMcL+2G/vRmKLPvWPzaHNJgX/hoWUdiYiKSkpJgcPG3+EqJRETVAf+SEoCim9qef/55fPHFF3jsscegyroFp5Mbob28B4rCPItdl4HXvgjBpW2t4dChQwAAo2uAzJUQEVUNDLxkxsfHBzNmzMDChQtRr149aG6fgfPxb+Fw6zSHORB7eK3k8OHDAAADV1cjIqoUDLxUopYtW2LZsmUYN24c9Fo1dFf2Qn9qE5Q5KXKXRjJi4LWOo0ePQjg4Qmhd5S6FiKhKYOClUqnVavTt2xervvgC3bp1gyr7NpxOfg/N9YOAyVgp1+CQBvvCh8vy7ty5g1u3bsHgVANQcAgJEVFlYOCl+/L09MTrr7+Od999F36+ftAm/gGnkxtL7O016dxg1HvBpHOToVKyNBMDr8WdPHkSAGBy9pG5EiKiqoOBl8otLCwMn332Pzz55JNQ5qbC6eRGOCSfMOv2y6vfGTnN+iCvfudynZM9vPbFJHcB1cDZs2cBAEanGjJXQkRUdTDwUoXo9XqMHz8e8+bNg7u7G3RXE6C7uJMLVlQT7OG1vEuXLgEAjHoPmSshIqo6GHjpgbRr1w7/+/RTtGjRAg4pl6A//SMUhTkPdC6Tif2G9oLTklnehYsXIRz0gFondylERFWGWu4CyH55eXnhvffew4IFC7Bp0yboT29BTsPuEFrnCp3HYDBAo9FYqEqqTOzhtaz8/HwkJSbC+BDTkS1ZsqTE7S+NHv3A5yQisnfs4aWHolarMXHiRAwePBjKvAzoz8ZBUZhboXMYDAYLVUeVzcjAa1G3b98GAJg0LjJXQkRUtdhE4F28eDHq1q0LnU6H8PBw7Nu3r8z269atQ+PGjaHT6RASEoItW7aY7RdCYMaMGfD394ejoyOioqJw7tw5szYpKSkYOHAgXF1d4e7ujuHDhyMrK0vav3PnTvTp0wf+/v5wcnJCaGgoVq9eXeFaqgOFQoGhQ4di0KBBUOZlwPHcLxWatoyB135w8IllJScnA0CF3yX5u9GjR5f4AQBpaWm8UZSIqiXZA+/atWsxYcIEzJw5E4cOHULLli0RHR2NW7duldh+7969GDBgAIYPH47Dhw8jJiYGMTExOH78uNRm3rx5WLRoEZYuXYqEhAQ4OTkhOjoaeXl/LZE7cOBAnDhxAlu3bsWmTZuwe/dujBo1yuw6LVq0wLfffoujR49KgW7Tpk0VqqU6GTp0KKKjo6HKvg3t9f3lPq6wkDe82QsTx/BalNTD66Cv9HMrAKSmpkrXICKqThRC5pf74eHhaNu2LT788EMARTcwBQYGYuzYsXj11Vfvad+vXz9kZ2ebBc/27dsjNDQUS5cuhRACAQEBmDhxIiZNmgQASE9Ph6+vL1asWIH+/fvj1KlTaNq0Kfbv34+wsDAAQFxcHHr06IHr168jIKDk9et79uwJX19ffPbZZ+Wq5Z/y8/ORn58vfZ2RkYHAwECkp6fD1bVqrKiUn5+PF154AZcvX0ZO455wSPwDDunXyzxm7dq18PX1tVKFVFHnz5/HiBEj0D0wFzdzVDh6t2i8dcOGDbFs2TKZq6tavv32W3zwwQfIbdAVBo86FT7e6fCXUBrKHlK0ePFiNGvW7EFLJCKyS7L28BYUFODgwYOIioqStimVSkRFRSE+Pr7EY+Lj483aA0B0dLTU/tKlS0hKSjJr4+bmhvDwcKlNfHw83N3dpbALAFFRUVAqlUhISCi13vT0dHh6epa7ln+aO3cu3NzcpI/AwMBSr2WvtFotXnnllaLPr5X+s/w7DmmwH0aOabConJyimU6EysFi1ygeNkFEVJ3IGnjv3LkDo9F4T++er68vkpKSSjwmKSmpzPbF/96vjY+P+SpGarUanp6epV7366+/xv79+zF06NBy1/JPU6dORXp6uvRx7dq1EtvZuyZNmqBLly5QZd+BMi/jvu2NxspZppgsz8ghDRYlBV6l5QLvjRs3LHZuIiJbJfsYXnuwY8cODB06FJ988slDvRWo1Wrh6upq9lFV/fvf/wYAqPLvH3jZw2s/OEuDZSkUln9BceXKFYtfg4jI1sgaeL29vaFSqe55iy05ORl+fn4lHuPn51dm++J/79fmnzfFGQwGpKSk3HPdXbt2oXfv3nj//fcxaNCgCtVSnQUHB6N+/frlasseXvthNCmsEsqqK7W6aGp0hbDc2JHipYuJiKoTWQOvRqNBmzZtsG3bNmmbyWTCtm3bEBERUeIxERERZu0BYOvWrVL7oKAg+Pn5mbXJyMhAQkKC1CYiIgJpaWk4ePCg1Gb79u0wmUwIDw+Xtu3cuRM9e/bE22+/bTaDQ3lrqe46dOhQrnZcac1+GIR1eiGrq+LACwsG3qtXryIzM9Ni5yciskWyr7Q2YcIEDB48GGFhYWjXrh0WLFiA7OxsaazsoEGDULNmTcydOxcAMH78eHTq1Anz589Hz5498dVXX+HAgQPS3eIKhQKxsbF48803ERwcjKCgIEyfPh0BAQGIiYkBUDTGtHv37hg5ciSWLl2KwsJCjBkzBv3795dmaNixYwd69eqF8ePH46mnnpLG5Wo0GunGtfvVUt21bNmyXO3Yw2s/jAy8FuXsXDT/rsKQf5+WD+fkyZNmL+6JiKo62cfw9uvXD++++y5mzJiB0NBQHDlyBHFxcdLNYFevXkViYqLUPjIyEmvWrMGyZcvQsmVLfPPNN9iwYQOaN28utZkyZQrGjh2LUaNGoW3btsjKykJcXBx0ur/Wpl+9ejUaN26Mrl27okePHujQoYNZUF25ciVycnIwd+5c+Pv7Sx99+/atUC3VWXBwcLnacSJ8+2HgkAaLKn4xrSjMseh1/vjjD4uen4jI1sg+D291lpGRATc3tyo1D+8/vfLKK2VO9QYAH374IV8k2LC/z8O7/5YGGSYdCgsLOQ+vBRw9ehTjxo1Dvn9LFNRqU+Hj7zcPr4/OiLv5KjRq0hRLlix5mFKJiOyK7D28VLW1atXqvm34mst+GAR7eC2p+J0tZb5lxthqVAL1XAtx+vRp5OaWvUAFEVFVwsBLsmPgtR9GBl6L8vHxgaOjHsrcVItdo5G7ASaTCadOnbLYNYiIbA0DL8mOAcp+FJr4eFmSQqFAvXpBUOWlAxaavSTYrRBA0Y1rRETVBQMvyY4Byn7wpjXLq1evHiBMUOZZppe3jkvRrCgXLlywyPmJiGwRAy/JjgHKPpgEpyWzhpCQEACAKrPkJcoflofGBJ1aVNmlzYmISsLAS7KTJtsnm2YQRUGXgdeyQkNDAVgu8CoUgKfGiLt37ljk/EREtoiBl2THwGsfCv9cH4SB17J8fHzg5+cHdWaSxVZc06sFsrKzLXJuIiJbxMBLslOpVHKXQOVQaGIPr7W0b98eCkM+VFm3KnSccNDBpHaESe0IgaLHSUABk9rRrJ1CAQgLLl9MRGRrGHhJdhqNRu4SqBwKGHit5pFHHgEAqFOvVOi4nOZPIrvVAGS3GgCTvmjVNpPeE9mtBpi1yzcq4KhzLOkURERVEgMvyY6B1z4U/tkhqFTyz4alhYaGQq/XQ512FbDAPNWp+Sp4enlV+nmJiGwVn7lIdgy89oE9vNbj4OCARx99FMr8TCizb1fqubMLFcgsVCAgIKBSz0tEZMsYeEl2jo58a9UeFBoZeK2pW7duAACHO+cr9bxXs4rGzNerV69Sz0tEZMsYeMkq9GoT3DQmKBVFb88qFQKqPz93cHCQszQqJw5psK5WrVrBy8sLDqmXAJOx0s57Pr1oVpTGjRtX2jmJiGwdn7nIKkY3y8IHHVJR27noibu2sxE1nYxwcXaWuTIqLw5psC6VSoXo6GgoDPkVvnmtLGfTi15gNm/evNLOSURk6xh4STY5BgWcXVzkLoPKqeDPHl4GXuvp1asXFAoFHG6frpTzGQVwJt0BderUhoeHR6Wck4jIHjDwkmxyDEo4s4fXbhT8OYaXQxqsJyAgAG3btoU6MwnK3NSHPl9ijhp5BgVCQ1tVQnVERPaDz1wkCyGAXIOCgdeOcEiDPP71r38BABxunaq0c7Zo0aLSzkVEZA8YeEkWJgEIAG5ubnKXQuXEldbkERERAT8/f2junAcM+ZVyzpCQkEo5DxGRvWDgJVkY/5xL393dXdY6qOI4pMG6VCoVnnqqL2AywOH22Yc+n7eXF3x8fCqhMiIi+8FnLpKFSRT1ErKH1/74+/ujYcOGqF27ttylVBtPPPEEdI6O0N46BQjTQ52rQXBwJVVFRGQ/1HIXQNVTcQ+vp6envIVQhb300ktctMDKnJ2d0bNHD3z77bdQp1yGwevBf/5BQUGVWBkRkX1gDy/JgoHXfnEpaHk8/fTTUCiU0CQdK7rr8wFxSWEiqo4YeEkWhj9vgPLy8pK5EqooLgUtD39/f3Tu3AmqnLtQZSY98Hlq1KhRiVUREdkHBl6SRXEPL5987Y9Wq5W7hGqrX79+AABN0vEHPgdvFCWi6oiBl2RhMCmgUqk4pMEO6XQ6uUuotho3bowWLVpAnX4Nyty0BzoHe+iJqDpi4CVZGETR9EgqlUruUqgClEol1Gre6yqn/v37AwAcHrCXl2Owiag6YuAlWRiFAn7+/nKXQRXkyN5d2bVv3x61atWCJuUCFIW5FT7eZHq4ac2IiOwRAy/JxtfXV+4SqIK0Oo7flZtSqcQzzzwDmIxwuHW6wsfn51fOam1ERPaEgZdk4+fnJ3cJVEFaLXt4bUF0dDRcXFyguX0KMBkqdGxWVpaFqiIisl0MvCQbfw5psDu8Yc026HQ69OnTB4rCPKhTLlXo2JSUFAtVRURkuxh4STY1a9aUuwSqIAZe2/Gvf/0LSqUSmuSTFVqIIjEx0YJVERHZJgZekg17eO0P5+C1HT4+PujYsSNUOXehzLpV7uMuX75suaKIiGwUAy9Zlb/eCI1SQKlUwtvbW+5yqIIYeG1L3759AQCa2+W/ee306Yrf6EZEZO8YeMmqXmyaBZUSqFu3LhQKhdzlUAUx8NqWkJAQ1K5dBw6plwFDXrmOuXLlClJTUy1bGBGRjWHgJavKLFQg16BArVq15C6FHgADr21RKBT41796F01RdudCuY/bv3+/BasiIrI9DLxkVUk5RSurMfDaJwZe2/P444/DwcEBDnfO3PfmNbWiaP+uXbusURoRkc1g4CWrSs4tCryBgYEyV0IPgoHX9ri6uqJDhw5Q5aZBmXO3zLZ+eiPqOBuQkPA70tLSrFMgEZENYOAlq0rKKfovxynJ7BMDr22Kjo4GADjcPX/fth0D8mEwGBEXF2fpsoiIbAYDL1lV8ZAG9vDaJwZe2xQWFgYPDw84pFwEhKnMth388qFTCXz37bcoLCy0UoVERPJi4CWrSspRwcnJCe7u7nKXQg+Agdc2qdVqdO7cGYrCPKgyk8ps66gW6BKQh1u3b+Pnn3+2UoVERPJi4CWrMQkgKVeFwMBATklmpzQajdwlUCm6dOkCAOVaaviJ2rnQqICVK1YgPz/f0qUREcmOgZesJiVfiUITpySzZwy8tqt58+bw9PSCQ+qV+w5rcNcKPF4rF7du38a6deusVCERkXwYeMlqkv+8YY2B135xSIPtUiqVePTRDlAY8qDMun3f9r3r5MJNI/DF55/j5s2bVqiQiEg+DLxkNcVTkjHw2i8HBwe5S6AyREZGAgDU6dfu29ZRLfB8wyzkFxTgrbfegtFotHR5RESyYeAlqykOvJySzH6xh9e2hYaGQqvVQp12/8ALAG1rFKC9Tz6OHj2Kzz77zMLVERHJh4GXrOZWDgOvveMYXtum1WrRunVrqHJT7zuOFwAUCmBo42z46Y1YvXo1fvzxRytUSURkfQy8ZDW3cpVwdnaCq6ur3KXQA+KQBtvXunVrAIDCWL45dh3VAhNaZMDFQWDevHkMvURUJTHwklUIALfyVAgIYO+uPWMPr+1r06YNgPIHXgDw05swJTQdTmoT3n77bSxfvhwm0/17iImI7AUDL1lFVoEC+UYF/Pz85C6FHgIDr+2rW7cu3NzcAVPFVlGr42LE9NZp8HE0YeXKlZgyZQpu3bplmSKJiKyMgZes4nZe0fhdf39/mSuhh8EhDbZPqVSiefNmUJRjDO8/+TuZ8J+wNLT2LsCBAwcwdMgQrFu3DgaDwQKVEhFZDwMvWcWdvKL/auzhtW/s4bUPISEhD3ysk4PA+JBMjGySBUVhNhYvXowhgwdj27ZtnLqMiOyW7IF38eLFqFu3LnQ6HcLDw7Fv374y269btw6NGzeGTqdDSEgItmzZYrZfCIEZM2bA398fjo6OiIqKwrlz58zapKSkYODAgXB1dYW7uzuGDx+OrKwsaX9eXh6GDBmCkJAQqNVqxMTE3FPHzp07oVAo7vlISip7Hfvqqjjw+vr6ylwJPQy1Wi13CVQOzZo1e6jjFQrgUf98zGufim61cpF48wbeeOMNDB40CD/88AOXIyYiuyNr4F27di0mTJiAmTNn4tChQ2jZsiWio6NLHTe2d+9eDBgwAMOHD8fhw4cRExODmJgYHD9+XGozb948LFq0CEuXLkVCQgKcnJwQHR2NvLw8qc3AgQNx4sQJbN26FZs2bcLu3bsxatQoab/RaISjoyPGjRuHqKioMr+HM2fOIDExUfrw8fF5yJ9K1XT3zyENDLz2jT289iE4OLhSzuPsIPB8wxy80z4VXQLykJx4A/Pnz8fTTz2FJUuW4Nq18s33S0QkN4UQQsh18fDwcLRt2xYffvghAMBkMiEwMBBjx47Fq6++ek/7fv36ITs7G5s2bZK2tW/fHqGhoVi6dCmEEAgICMDEiRMxadIkAEB6ejp8fX2xYsUK9O/fH6dOnULTpk2xf/9+hIWFAQDi4uLQo0cPXL9+HQEBAWbXHDJkCNLS0rBhwwaz7Tt37kSXLl2QmpoKd3f3cn2/+fn5Zj0jGRkZCAwMRHp6epWdquurr77C0qVLoVEKFJgU+OGHH+Di4iJ3WVQB58+fx4gRIwAAP/74IxwdHWWuiMrj8ccfR0FBAYx6L+Q06wOX/Z+hlpMBc8LTH/ic6QUKbLuuw46bOqQXFPWXNG/eHN27d0enTp34u01ENku2Ht6CggIcPHjQrAdVqVQiKioK8fHxJR4THx9/T49rdHS01P7SpUtISkoya+Pm5obw8HCpTXx8PNzd3aWwCwBRUVFQKpVISEio8PcRGhoKf39/dOvWDXv27Cmz7dy5c+Hm5iZ9BAYGVvh69qrApIBOq4Wzs7PcpdBD4JAG+yGtime6d9ztRyecMWO/Gz46UbHfRzeNQN96uXg/MhVjmmeihWcBTpw4jnfffRd9n3wS06ZNw44dO5Cbm1sZ3wIRUaWR7dnrzp07MBqN97zF7evri9OnT5d4TFJSUonti8fNFv97vzb/HHagVqvh6elZofG3/v7+WLp0KcLCwpCfn49PP/0UnTt3RkJCgjTx+z9NnToVEyZMkL4u7uGtLrxr1IBCoZC7DHoIDLz2Q6vVIjMzEwpxb+BNzFHhcuaDP5ZqJdDOpwDtfAqQkq9EfJIG8cla/Pbbb/jtt9+g1WjQPiICnTt3Rnh4OPR6/cN8K0RED43PXg+oUaNGaNSokfR1ZGQkLly4gPfffx9ffPFFicdotdq/el2qIS8vL7lLoIfEFyz2468eXstOKeapNaFnnTz0rJOHG9kq7Lulwb5bBuzatQu7du2CRuOAtm3boVOnToiIiOCwByKShWyB19vbGyqVCsnJyWbbk5OTS526ys/Pr8z2xf8mJyebzfeanJyM0NBQqc0/b4ozGAxISUl56Cmz2rVrh99+++2hzlGVeXp6yl0CUbVRfIOh4m9DGm7mqDD2Nw9kFha9cLmaVfS1i4Ppocb2FqvpZMSTQbl4MigXN7JV2H9Lg/23NdizZw/27NkDtVqFsLC26NSpEzp06MDwS0RWI9sYXo1GgzZt2mDbtm3SNpPJhG3btiEiIqLEYyIiIszaA8DWrVul9kFBQfDz8zNrk5GRgYSEBKlNREQE0tLScPDgQanN9u3bYTKZEB4e/lDf05EjR7iwQhkYeImsRxp+8rfAaxIKpBcoYRIKs68zCyv/qaCmkxExQbn4b7t0vNM+Fc/Wz0Ztx3z8/vvvePvtt/HkkzGYOnUqtm/fzmnOiMjiZB3SMGHCBAwePBhhYWFo164dFixYgOzsbAwdOhQAMGjQINSsWRNz584FAIwfPx6dOnXC/Pnz0bNnT3z11Vc4cOAAli1bBqDo7dbY2Fi8+eabCA4ORlBQEKZPn46AgABpLt0mTZqge/fuGDlyJJYuXYrCwkKMGTMG/fv3N5uh4eTJkygoKEBKSgoyMzNx5MgRAJB6ihcsWICgoCA0a9YMeXl5+PTTT7F9+3b8/PPP1vnh2SEPDw+5SyCqNoqHnyiEEZBvMh4AgK/ehF518tCrTh5u5yr/HPagRXx8POLj4+Gk16NrVBT69OmD+vXry1orEVVNsgbefv364fbt25gxYwaSkpIQGhqKuLg46aazq1evQqn8q+chMjISa9aswbRp0/Daa68hODgYGzZsQPPmzaU2U6ZMQXZ2NkaNGoW0tDR06NABcXFx0Ol0UpvVq1djzJgx6Nq1K5RKJZ566iksWrTIrLYePXrgypUr0tetWrUCULSwBVA0y8TEiRNx48YN6PV6tGjRAr/88gu6dOlS+T+oKqK807cRUeVSFNrOrAk1HP8a85uYrcSeZC32JpmwceNGbNy4ES1atMDQoUOlv7lERJVB1nl4q7uMjAy4ublVi3l4AeDNN99Ehw4dZK6IKurv8/Du3LlT3mKo3EaNGoWzZ88CAHIa94D+9JZS27ppTPigQ6q1SruHSQDH7jpg6w0djt4tGnvcrl07TJo0iYv5EFGlkH1pYao+qmqoJ7JFtWvXlt4tU+RnylxN2ZQKoKV3ISa1zMTstmkI8SzAvn37MHzYMFy8eFHu8oioCuC0ZGQ1DLxE1jNt2jQcOnQIEyZMgDI/CwCwZMmSEttOjX3RmqWVqa6LEZNaZmLHTS1WnAFmzpyBzz//glPiEdFDYQ8vWQ2nICKyruJZY5QFWTJXUjEKBfBYzXw08yjAtWvXkZGRIXdJRGTn2MNLVsNlhYmsy9vbGwCgKMgGAIwePbrEdm4aq5VUbnuSNDidpkFgrVr820FED42Bl6ymOq8yRyQHBwcHeHh44E5uttyllNuVTBXWXdDjaIoGLs7OeOXVV6FSqeQui4jsHAMvEVEVVqNGDaSet+0bv/KNwOE7Gmy7rsOZdAcAQNu2bTF+/HjUqlVL5uqIqCpg4CUiqsK8vb2l6clsSYEROJnqgIRbGhy8rUWeseimtPDwcDzzzDMICwuTuUIiqkoYeImIqjAvL68/PxNQKgAXB4EcgwJGAagUgF4t4OJgskotGQUKHL3rgMN3NDiaokH+nyE3ICAAUVFRePzxx9mjS0QWwcBLRGXi2jT2zdPTU/o8QG/EnPB0q13baAIuZqpx7K4Djt7V4FKmGsX/m+rUqY1HHumARx99FI0bN+a0Y0RkUQy8ZFFGo1HuEughMfDat78HXmu4m6fEsRQHHLvrgBOpGuQYioKsWq1C6zahaN++Pdq3b4/AwECr1kVE1RsDL1lUfn4+AECjscF5j6hcTCbrvN1NluHu7m7R8xeagDNpDvjjrgOO3dXgZs5fMyoEBATg8Xbt0K5dO4SGhkKv11u0FiKi0jDwkkUVB15OSWa/2Etv3zw8PCr9nCn5SvxxxwFH7mpwMtVBGovr6KjDI4+0Qdu2bdG2bVvUrFmz0q9NRPQgGHjJohh47R+HNNg3Nze3SjnPzWwlDtzW4sBtDS5n/vXUUa9ePYSHh6Ndu3Zo3rw5HBwcKuV6RESViYGXLKqgoAAAhzTYMw5psG+urq4PfOydXCXikzWIT9bienbR04VarUK7dm0QGRmJ9u3bw8/Pr7JKJSKyGAZesqjCwkIAYK+PHWPgtW8uLi4Vam8wAYfuaLDjhhYnUoteqGo0DujYMQKdOnVCeHg4l/olIrvDwEsW1bZtW2zduhVPPPGE3KXQA+KQBvumVqvh5OSE7OysMtvlGhTYfkOLn687IjVfCQAIDQ1FdHQ0OnbsCCcnJ2uUS0RkEQy8ZFGPP/442rZta5EbZ8g62MNr/1xcXEoNvIUmYNt1HTZe0SOrUAG9oyOefbY3evfuzanDiKjKYOAli2PYtW/s4bV/pQ1BuJChxqennHEjWwVnZyeMHPJv9OnTh0MWiKjKYeAlIqriSgqwv1zXYvU5Z5igwJNPxmDYsGEVHu9LRGQvGHiJqEwhISHo0KEDevToIXcp9ICKx99ez1Yj7poOidkq7Lipg6enB2bOnIWWLVvKXCERkWUx8BJRmbRaLd588025y6CH8PcbztacK/q8Zs0AvP/+Avj4+MhVFhGR1TDwEhFVcUVL+iqgUCjw3//+FwqFAiEhIRyrS0TVBgMvEVEV5+joCKDoBsTIyEiZqyEisj6l3AUQEZFlFfXwEhFVXwy8RERVXHEPLxFRdcXAS0RUxel0OrlLICKSFQMvEVEVp9Vq5S6BiEhWDLxERFUce3iJqLpj4CUiquI0Go3cJRARyYqBl4ioimPgJaLqjoGXiKiKY+AlouqOgZeIqIpzcHCQuwQiIlkx8BIRVXEMvERU3THwEhFVcQy8RFTdMfASEVVxarVa7hKIiGTFwEtEVMUx8BJRdcfAS0RUxTHwElF1x8BLRFTFcVoyIqru+LKfiKiK0+v1mDRpEvz9/eUuhYhIFgohhJC7iOoqIyMDbm5uSE9Ph6urq9zlEBEREVVJHNJARERERFUaAy8RERERVWkMvERERERUpTHwEhEREVGVxsBLRERERFUaAy8RERERVWkMvERERERUpTHwEhEREVGVxsBLRERERFUaAy8RERERVWmyB97Fixejbt260Ol0CA8Px759+8psv27dOjRu3Bg6nQ4hISHYsmWL2X4hBGbMmAF/f384OjoiKioK586dM2uTkpKCgQMHwtXVFe7u7hg+fDiysrKk/Xl5eRgyZAhCQkKgVqsRExNTYi07d+5E69atodVq0aBBA6xYseKBfgZEREREZDmyBt61a9diwoQJmDlzJg4dOoSWLVsiOjoat27dKrH93r17MWDAAAwfPhyHDx9GTEwMYmJicPz4canNvHnzsGjRIixduhQJCQlwcnJCdHQ08vLypDYDBw7EiRMnsHXrVmzatAm7d+/GqFGjpP1GoxGOjo4YN24coqKiSqzl0qVL6NmzJ7p06YIjR44gNjYWI0aMwE8//VRJPx0iIiIiqgwKIYSQ6+Lh4eFo27YtPvzwQwCAyWRCYGAgxo4di1dfffWe9v369UN2djY2bdokbWvfvj1CQ0OxdOlSCCEQEBCAiRMnYtKkSQCA9PR0+Pr6YsWKFejfvz9OnTqFpk2bYv/+/QgLCwMAxMXFoUePHrh+/ToCAgLMrjlkyBCkpaVhw4YNZttfeeUVbN682Sxs9+/fH2lpaYiLiyvX95+RkQE3Nzekp6fD1dW1XMcQERERUcXI1sNbUFCAgwcPmvWgKpVKREVFIT4+vsRj4uPj7+lxjY6OltpfunQJSUlJZm3c3NwQHh4utYmPj4e7u7sUdgEgKioKSqUSCQkJ5a7/frWUJD8/HxkZGWYfRERERGRZsgXeO3fuwGg0wtfX12y7r68vkpKSSjwmKSmpzPbF/96vjY+Pj9l+tVoNT0/PUq9bkVoyMjKQm5tb4jFz586Fm5ub9BEYGFju6xERERHRg5H9prXqZOrUqUhPT5c+rl27JndJRERERFWeWq4Le3t7Q6VSITk52Wx7cnIy/Pz8SjzGz8+vzPbF/yYnJ8Pf39+sTWhoqNTmnzfFGQwGpKSklHrditTi6uoKR0fHEo/RarXQarXS18XDpzm0gYiocri4uEChUMhdBhHZGNkCr0ajQZs2bbBt2zZp2i+TyYRt27ZhzJgxJR4TERGBbdu2ITY2Vtq2detWREREAACCgoLg5+eHbdu2SQE3IyMDCQkJeOmll6RzpKWl4eDBg2jTpg0AYPv27TCZTAgPDy93/REREfdMifb3WsojMzMTADi0gYiokvAmYCIqiWyBFwAmTJiAwYMHIywsDO3atcOCBQuQnZ2NoUOHAgAGDRqEmjVrYu7cuQCA8ePHo1OnTpg/fz569uyJr776CgcOHMCyZcsAAAqFArGxsXjzzTcRHByMoKAgTJ8+HQEBAVKobtKkCbp3746RI0di6dKlKCwsxJgxY9C/f3+zGRpOnjyJgoICpKSkIDMzE0eOHAEAKUi/+OKL+PDDDzFlyhQMGzYM27dvx9dff43NmzeX+/sPCAjAtWvXqnSPREZGBgIDA3Ht2jU+CdkpPob2rzo9hi4uLnKXQES2SMjsgw8+ELVr1xYajUa0a9dO/P7779K+Tp06icGDB5u1//rrr0XDhg2FRqMRzZo1E5s3bzbbbzKZxPTp04Wvr6/QarWia9eu4syZM2Zt7t69KwYMGCCcnZ2Fq6urGDp0qMjMzDRrU6dOHQHgno+/27FjhwgNDRUajUbUq1dPLF++/OF/IFVMenq6ACDS09PlLoUeEB9D+8fHkIiqO1nn4aWqj3MN2z8+hvaPjyERVXecpYGIiIiIqjQGXrIorVaLmTNnms1OQfaFj6H942NIRNUdhzQQERERUZXGHl4iIiIiqtIYeImIiIioSmPgJSIiIqIqjYGXyIpWrFgBd3d3ucugEigUCmzYsEHuMoiIyAIYeB/S7du38dJLL6F27drQarXw8/NDdHQ09uzZU2nXmDVrFhQKBRQKBdRqNby9vdGxY0csWLAA+fn5lXaditTx949ffvnFajVUhEKhwMqVK+/7+BR/H7///rvZ8fn5+fDy8oJCocDOnTtLvQ4fn/sr6fdEoVBIqyiWZdeuXXjsscfg6ekJvV6P4OBgDB48GAUFBWbt+Dg8mPIG/fI+DvZq586dUCgUSEtLM9s+ZMgQaaXO4q+LH1MHBwf4+vqiW7du+Oyzz2AymaxbNBGVGwPvQ3rqqadw+PBhrFy5EmfPnsXGjRvRuXNn3L17t1Kv06xZMyQmJuLq1avYsWMHnnnmGcydOxeRkZHIzMys1GuVp46/f3Ts2PGBzmWNJ8p58+aV6/EJDAzE8uXLzbatX78ezs7O5boOH5+ylfR7AuC+P5uTJ0+ie/fuCAsLw+7du3Hs2DF88MEH0Gg0MBqN97Tn42AZFX0cqrru3bsjMTERly9fxo8//oguXbpg/Pjx6NWrFwwGg9zlEVFJ5F3ozb6lpqYKAGLnzp1lthk+fLjw9vYWLi4uokuXLuLIkSNmbebOnSt8fHyEs7OzGDZsmHjllVdEy5Ytpf0zZ840+7rYqVOnhEajEa+//rq0LS8vT0ycOFEEBAQIvV4v2rVrJ3bs2CHtv3z5sujVq5dwd3cXer1eNG3a1Gx55uPHj4uePXsKFxcX4ezsLDp06CDOnz9fZh3Fjh49Krp06SJ0Op3w9PQUI0eONFuyefDgwaJPnz7izTffFP7+/qJu3bri0qVLAoBYu3at6NChg9DpdCIsLEycOXNG7Nu3T7Rp00Y4OTmJ7t27i1u3bknn2rdvn4iKihJeXl7C1dVVdOzYURw8eFDa/8+loevUqVPq4wNA6PV6AUB06tRJeny6desmpk+fLgAIDw8P6fEJDw8XGo1GODo6iqCgIPHoo4+KFi1aSOc8cuSI6Ny5s3ROPz8/sX//fiGEEMuWLRMajUZ6fFq3bi0aNmwoYmJiRF5eXpV8fFxcXAQAsWzZslIfn8DAwBJ/T95//31Rt25dIUTZvye//vqrCAwMFAqFQtSqVUuMHTtWZGVlCSH++j1p0KCB0Ol0om7dumL58uXC1dVVuLm5Sb8nAMT69euFEEW/J127dhUODg4CgFAqlSI8PFxcunRJehxq1aol1Gq10Gq1wsHBQbi5uYnRo0eLadOmSXUtXrxYNGjQQGi1WuHj4yOeeuopu/w9+fvjUJZff/1Vqu+fj4MQQty8eVP06NFDehxWr14t6tSpI95//32pDQCxdOlS0bNnT+Ho6CgaN24s9u7dK86dOyc6deok9Hq9iIiIkP7fF9uwYYNo1aqV0Gq1IigoSMyaNUsUFhaanfeTTz4RMTExwtHRUTRo0EB8//33Qggh/Xz//lG8rH3x4/HPx+eftm3bJl1DiKJl7mfOnCkCAwOFRqMR/v7+YuzYsff9GRKRZTDwPoTCwkLh7OwsYmNjRV5eXoltoqKiRO/evcX+/fvF2bNnxcSJE4WXl5e4e/euEEKItWvXCq1WKz799FNx+vRp8frrrwsXF5dyBV4hhOjTp49o0qSJ9PWIESNEZGSk2L17tzh//rx45513hFarFWfPnhVCCNGzZ0/RrVs3cfToUXHhwgXxww8/iF27dgkhhLh+/brw9PQUffv2Ffv37xdnzpwRn332mTh9+vR968jKyhL+/v6ib9++4tixY2Lbtm0iKChIetIQouiJwtnZWTz//PPi+PHj4vjx49ITTePGjUVcXJw4efKkaN++vWjTpo3o3Lmz+O2338ShQ4dEgwYNxIsvviida9u2beKLL74Qp06dEidPnhTDhw8Xvr6+IiMjQwghxK1btwQAodPpxMiRI8W1a9dKfXwAiHfeeUc0atRIPPHEE8LLy0v88ccfQqvVigULFggAYtKkSdLjo9VqRYMGDcSlS5fExo0bhZOTk/D395fO2axZM/Hcc8+JU6dOia5du4qaNWtKIbpjx45CpVKJ3bt3i927d4saNWoIpVIpTp06VWUfn6NHjwoHBweh1+vF7du3zR6f5cuXi8TERNGxY8cSf08++eQTodVqxcyZM0v9PTl//rxwcnIS0dHRonHjxmLPnj2iVatWYsiQIVI9Pj4+QqvVivj4eHHgwAHh4+MjlEqlGDNmjPR7AkAsXrxYCCFEjx49hJOTk4iJiRFbtmwRS5YsEVFRUaJRo0bi4sWLwtPTU9SuXVs4OTmJ/v37i//+97/io48+Enq9XvTq1Uu0bNlS7N+/X6hUKrFmzRpx+fJlcejQITFv3jyb/D0pfhz+Hpb/7ssvvxRarVb6v1iS4sfh/fffF2fPni3xcYiKihKhoaHi999/FwcPHhSdOnUSjo6O9wTemjVrirVr14ozZ86ImJgYUbduXfHYY4+Zfe/du3eXjtm9e7dwdXUVK1asEBcuXBA///yzqFu3rpg1a5bZeWvVqiXWrFkjzp07J8aNGyecnZ3F3bt3hcFgEN9++60AIM6cOSMSExNFWlqa9HiUJ/AKIUTLli3FE088IYQQYt26dcLV1VVs2bJFXLlyRSQkJJi96CMi62LgfUjffPON8PDwEDqdTkRGRoqpU6eKP/74QwhR1Nvh6up6TxiuX7+++Pjjj4UQQkRERIjRo0eb7Q8PDy934H3llVeEo6OjEEKIK1euCJVKJW7cuGHWpmvXrmLq1KlCCCFCQkLMngT+burUqSIoKEgUFBSUuH/mzJlCqVQKJycn6aNt27ZCiKKeSw8PD7PenM2bNwulUimSkpKEEEVPFL6+viI/P19qU/xE/umnn0rbvvzySwFAbNu2Tdo2d+5c0ahRoxLrEkIIo9EoXFxcxA8//CBtAyCmTJly38enuGdvwYIFokuXLqJ+/fqid+/e4sknnxRt27YVAMx6yf/5+HTr1k16DIQQwsXFRaxYsUIIce/jo1QqhYuLizh9+rQIDAwU48aNqxaPz9dffy0ACI1GIz0OxT/3sn5PPvroI/H/7d17UFTXHQfw791d9iEIBVQEuy6PBUQLqCGJxKBGQUXjqDGpNYTAlEhIB4mjjbSR+kgFp06rEVKbVHEdQZrE+jZKaRAyCMqisCyPFRAhTKOIJkRYARPg1z8st6xgJErUkN9nhhnuufeec+7+OOyPyz1nIyMjxbvwixYtopSUFLpx44YYh6ioKIqOjrYYJ3l5eSSRSKi9vZ2qqqoIACkUCjEOUqmUAPRJtJYsWUJERGq1mhwdHam7u1vcf+vWLVKpVLR06VJyc3Oj8PBw0mg01NnZKR7z0ksv0YQJE0gikZBSqSQAj1Uc7jZOeu5s301nZ6cYh9GjR1vEoUdPHHrrHQeTyUQAxP92EBHV1NT0G4eEhARx+8yZMwSAUlNTLa5dqVSK27NmzaKkpCSLttPS0iz+EL2zXrPZTADo5MmTRESUk5NDAKi5udminu+T8C5dulS8AfGXv/yFvLy87jpeGWMPl+zBH4r4aVuyZAnmz5+PvLw8nD17FidPnsSWLVuwa9cu3Lx5E2azGY6OjhbntLe3o7a2FgBgMpkQExNjsT8wMBA5OTkDap+IIAgCAKCsrAxdXV3w8vKyOKZn8hUAxMXF4Y033kBWVhaCg4OxZMkS+Pn5AQAMBgOCgoJgZWV11/a8vb3F5y8BiB9VajKZ4O/vD2tra3Hf1KlT0d3djaqqKjg5OQEAfH19IZfL+9Tb0wcAFsf2LmtqahK3r169ioSEBOTm5qKpqQldXV1oa2tDQ0ODRb2BgYHYuHHjd8YHAJYtWwapVIqbN29CIpHg+vXrSE9PxyuvvNKnrw4ODvjss88wevRomM1mdHR0WOxftWoVXnvtNaSlpeHbb78Vy8vKytDd3Y3W1laMGzcOVlZWSE1N/cnEBwCio6MxatQonDx5EgBw6tQpeHt733Wc1NXVQafT4dChQ1i6dCmsrKyQlJSEP/3pTwgNDYVer0dpaSmMRiOICF1dXbCxsQERobu7G3V1daiuroYgCJBKpWIcevoTHx+PhIQEsc3GxkYAgI+PD7KysiCTySCVSiGTySCRSNDR0YHy8nIEBQVBEARMmDBBrBcAnJ2dUVRUBG9vb2RkZGDp0qW4du0afv7zn2Pfvn0wGo2PPA79jZN7kUql0Ol02LRpE06dOoXCwkIxDnq9Hs7OzmIc9u3bJ553ZxxkMhkmT54s7tdqtbC3t7+v6+zo6EBLSwtsbW1RWlqK/Px8JCYmisd0dXWho6MDbW1tGDZsWJ96ra2tYWtra/F6Pajev49feuklvPvuu3B3d8fcuXMxb948LFiwADIZv+0y9ijwpLVBoFQqERISgj/84Q8oKChAZGQk1q9fD7PZDGdnZxgMBouvqqoqvPXWW4PStslkgpubGwDAbDZDKpXi/PnzFu2ZTCZs374dAPDaa6/h0qVLCA8PR1lZGQICApCSkgIAUKlU92xPLpdDq9WKX2q1+nv1t/cbfW+9k7ieN4w7y3rPgI6IiIDBYMD27dtRUFAAg8EAR0fHfif43Cs+ALB161YYDAZxYo5SqURoaGifus6cOYPMzEzY2tri+PHjKCkpQVBQEKjXJ3Rv2LABFRUVmD9/PiorK9HR0YFDhw7BbDZDIpHA2toazz//PJydnZGZmfmTic+IESPg4eEhxgEAPvzwwwGNE0EQMHXqVLz33nuoqKhAR0cHjEYjgNs/96+//jpiYmLg5eUFg8GA0tJS1NTUwMPDQ+zTneNk+PDh+O1vfyu2BwBRUVEAAHd3d/j7+2P9+vWYMWMGOjs7sWrVKlRXV8Pd3b3f16Wnn0QEuVyOiRMnoqKiAh9//DG0Wi3WrVuHtLS0AU1qehTjZCDGjBmD8PBwizi8//77AP4fh94x7C8OAzGQ6wQgXqvZbMbGjRst2i4rK0NNTQ2USmW/9fbUM5grK/T+faxWq1FVVYUdO3ZApVLhN7/5DaZNm2bxRzBj7OHhhPcHMH78eNy8eROTJ09GY2MjZDKZRRKi1WoxYsQIALfvJBUWFlqcf+fyWHdz4cIFZGZmYsmSJQCASZMmoaurC01NTX3aGz16tHieWq1GTEwMDh48iNWrV2Pnzp0Abt/9yMvLu69fyD4+PigtLcXNmzfFsvz8fEgkEnh7e3/v+u4lPz8fcXFxmDdvHiZMmACFQoHr169bHGNlZdXvDPI74wPcvjOn1WoRFxcHvV6PyMhISKXSPnfLCwoKIJfL4eTkhICAAHh6euLGjRt92vDy8kJoaChu3LiB8ePHQ6fTYdKkSeKb6+HDhzFlyhT8+te/xrBhw36S8em5Y/p9x4m9vT2cnZ3Fu5STJ09GZWUlHBwcoFAoLM6Xy+WQy+UgIgQGBgL4/zhpbW3FyJEjxWN76u6ps6GhAStXrkRWVhZWr16Nw4cPQ6vV4oknnkBeXt6AEiWZTIbg4GBs2bIFRqMRra2tMBgMj1Uc7jZO7qUnDj3X0hOHO2PYEwdvb290dnaipKRErOPixYtobm5+sIv8X9tVVVX9ti2RDOxtrueO+v2uOnHq1CmUlZWJv4+B23+kLliwAMnJycjNzcWZM2dQVlZ2X/Uzxh4MJ7wP4Msvv8TMmTORnp4Oo9GIuro67N+/H1u2bMHChQsRHByMwMBALFq0CFlZWaivr0dBQQHWrl2Lc+fOAQDefPNN7N69GzqdDtXV1Vi/fj0qKir6tNXZ2YnGxkZcvnxZXBJo+vTpmDhxongXzMvLC2FhYXj11Vdx8OBB1NXVQa/XY/Pmzfjkk08AACtXrsS//vUv1NXVobi4GDk5OfDx8QEAxMbGoqWlBb/61a9w7tw51NTUIC0tDVVVVfd8LcLCwqBUKhEREYHy8nLk5ORgxYoVCA8PF/8lOZg8PT2RlpYGk8mEwsJChIWF9bkDqlarERcXh/feew+nT59GXV0dNm/ejDVr1iA4OFiMD3D7cYH6+nrY2tpi5cqVWLhwIQCIj5ucPHkS1dXVKCoqwq1bt/D111+jtrYWycnJMJlMAG7/O7y2thbLli1DXFwcgoKCoNVq0dbWBh8fH3h5eWHKlCloa2vDkSNHsGnTJqjVavj7+yM9PR3A0IxPVlYWfH19YWVlhcuXL4vjRBAEjBkzBr/4xS/w5JNPYtGiRUhOToaHhweOHDmCtWvX4u2338Ybb7yBGTNmIDU1FZs2bcLy5ctRVlYm/qERHx+PgoICnDhxAmazGQUFBdDpdFi2bBlSUlIQGRkJOzs76PV6FBYWiomuIAjiuNXr9QAgjkuDwQCVSoXZs2dj165dyMzMhKOjI+Li4vDCCy+gpaUFn332GZqbm+8ah+PHjyM5ORkGgwGff/459u7dCyKCSqV6rMaJq6srsrOz0djYKCafer0e48aNwxdffAEA+OCDD8RHbWpra1FRUYH4+HhUVFRgwYIFFnGIjY2FwWBATU0Njhw5gtjYWADAuHHjEBwcjOjoaOj1epSUlCA6OhoqlUq8Y3u/1q1bh71792Ljxo2oqKiAyWTChx9+aPG4yr1oNBoIgoDjx4/j2rVr4uNO/bl16xYaGxvxxRdfoLi4GElJSVi4cCGef/55vPrqqwBuf8hMamoqysvLcenSJaSnp0OlUkGj0TzQtTLG7tMje3p4COjo6KDf/e53NHnyZHF5I29vb0pISKC2tjYiImppaaEVK1aQi4sLWVlZkVqtprCwMGpoaBDrSUxMpBEjRpCNjQ1FRETQmjVr+kxaw/+WypFKpeTg4EDPPvssbdu2rc9En2+++YbWrVtHrq6uZGVlRc7OzrR48WIyGo1ERBQbG0seHh6kUCho5MiRFB4eTtevXxfPLy0tpdmzZ9OwYcNo+PDhFBQURLW1tWI/BmPZq956JuOUlJSIZf1NHtHpdGRnZyduFxcXU0BAACmVSvL09KT9+/f3Wd7owIED9LOf/YwAkCAINGzYMFKr1QRAXBWhpaWFAJCDg0O/8elZtszOzk6Mz1NPPUVSqZRsbGxo6dKlNGfOHIv4yOVyksvlJJVKydnZmWJjY6m9vZ2ISFx1oHd8XFxcyM3Nja5evTok4+Ph4UGLFi0iuVxOSqVSHCe//OUvyd3dnWQyGanValqxYgWNGDGCAJCLiwuFhYXRiRMn6JVXXiE3NzeSSqUkCAJJpVKaOXOmxTjR6/Xk7u5usayUSqUSx0l9fT3Nnz+fFAoFjR07lnbv3k12dnZi3J2dnQkAvfvuu0R0e5y4urqSRCIhQRBIIpGQRqOh5cuX040bN6i0tJRcXFxIKpVaxOHNN98kjUZD/v7+lJeXR9OnTyd7e3tSqVTk5+dHH3300WM3To4ePUparZZkMpm4LFlP3T3LsBUXF4txUCgU5OjoSNOmTaOjR49a9FOv11NISAjZ2NiQtbU1+fn5UWJiorj/8uXLFBoaSgqFgjQaDWVkZNCoUaPo/fffF4/BHZPoBnrtmZmZ9Mwzz5BKpSJbW1t66qmnLFZFuLNeIiI7OzvS6XTi9jvvvEOjR48mQRC+c1mynp8xmUxGI0eOpODgYNq9ezd1dXWJxx06dIiefvppsrW1JWtra5oyZQp9+umnxBh7NASiXg8fssfChg0bcPjwYfG5QvZ44fg8Hh40Dq6urli5ciVWrlw5qP1iA/ef//wHarUan376KWbNmvWou8MYG8J4uihjjLGH4tSpUzCbzfD19cWVK1ewZs0auLq63ven0DHG2EBxwssYY+yh+Pbbb/H222/j0qVLGD58OJ555hns27fvO5faY4yxwcCPNDDGGGOMsSGNV2lgjDHGGGNDGie8jDHGGGNsSOOElzHGGGOMDWmc8DLGGGOMsSGNE17GGGOMMTakccLL2BCSm5sLQRDw9ddfP+qu4MKFC5gyZQqUSiUmTpz4qLvDGGPsJ4wTXsbYD2L9+vWwtrZGVVUVsrOzH3V3+iUIAg4fPvyou8EYY+wHxgkvY+wHUVtbi2effRYajQaOjo73Vcc333wzyL1ijDH2U8QJL2MP6J///Cd8fX2hUqng6OiI4OBg3Lx5EwCwa9cu+Pj4QKlUYty4cdixY4fFuXq9HpMmTYJSqURAQAAOHToEQRBgMBgG1PaJEyfg5eUFlUqF5557DvX19X2OOX36NIKCgqBSqaBWqxEXFyf2DwB27NgBT09PKJVKODk54cUXXxT3dXd3Y8uWLdBqtVAoFBg7diwSExPv2S9BEHD+/Hm88847EAQBGzZsAACUlZVh5syZ4msVHR0Ns9ksnhcZGYlFixYhMTERLi4u8Pb2Rn19PQRBwMcffyxex5NPPonq6moUFRUhICAANjY2CA0NxbVr18S6ioqKEBISghEjRsDOzg7Tp09HcXGxuN/V1RUAsHjxYgiCIG4zxhgbgogxdt8uX75MMpmMtm7dSnV1dWQ0Gumvf/0rtba2Unp6Ojk7O9OBAwfo0qVLdODAAXJwcKA9e/YQEVFrayuNHDmSXn75ZSovL6djx46Ru7s7AaCSkpJ7tt3Q0EAKhYJWrVpFFy5coPT0dHJyciIA1NzcTEREFy9eJGtra9q2bRtVV1dTfn4+TZo0iSIjI4mIqKioiKRSKWVkZFB9fT0VFxfT9u3bxTbWrFlD9vb2tGfPHrp48SLl5eXRzp0779m3K1eu0IQJE2j16tV05coVam1tJbPZTM7OzvTCCy9QWVkZZWdnk5ubG0VERIjnRUREkI2NDYWHh1N5eTmVl5dTXV0dAaBx48ZRZmYmVVZW0pQpU+iJJ56gGTNm0OnTp6m4uJi0Wi3FxMSIdWVnZ1NaWhqZTCaqrKykqKgocnJyopaWFiIiampqIgCk0+noypUr1NTUdM/rYowx9uPECS9jD+D8+fMEgOrr6/vs8/DwoIyMDIuyP/7xjxQYGEhERB988AE5OjpSe3u7uP9vf/vbgBPe3//+9zR+/HiLsvj4eIuENyoqiqKjoy2OycvLI4lEQu3t7XTgwAGytbUVk8DeWlpaSKFQDCjB7Y+/vz+tX79e3P773/9O9vb2ZDabxbJPPvmEJBIJNTY2EtHthNfJyYlu3bolHtOT8O7atUss+8c//kEAKDs7WyzbvHkzeXt737U/XV1dNHz4cDp27JhYBoAOHTp0X9fHGGPsx0P2qO4sMzYU+Pv7Y9asWfD19cWcOXMwe/ZsvPjii5DL5aitrUVUVBSWL18uHt/Z2Qk7OzsAgMlkgp+fH5RKpbg/MDBwwG2bTCY8/fTTFmV3nl9aWgqj0Yh9+/aJZUSE7u5u1NXVISQkBBqNBu7u7pg7dy7mzp2LxYsXY9iwYTCZTLh16xZmzZr1vV6T7+qvv78/rK2txbKpU6eiu7sbVVVVcHJyAgD4+vpCLpf3Od/Pz0/8vvexvcuamprE7atXryIhIQG5ubloampCV1cX2tra0NDQMCjXwxhj7MeDE17GHoBUKsW///1vFBQUICsrCykpKVi7di2OHTsGANi5c2efpFQqlT60/pnNZrz++uuIi4vrs2/s2LGQy+UoLi5Gbm4usrKysG7dOmzYsAFFRUVQqVQPrZ+99U6Ie7OyshK/FwSh37Lu7m5xOyIiAl9++SW2b98OjUYDhUKBwMBAngjHGGM/QTxpjbEHJAgCpk6dio0bN6KkpARyuRz5+flwcXHBpUuXoNVqLb7c3NwAAD4+PjAajejo6BDrOnv27IDb9fHxgV6vtyi78/zJkyejsrKyTx+0Wq14F1UmkyE4OBhbtmyB0WhEfX09Tp06BU9PT6hUqkFbUszHxwelpaUWE+by8/MhkUjg7e09KG30lp+fj7i4OMybNw8TJkyAQqHA9evXLY6xsrJCV1fXoLfNGGPs8cIJL2MPoLCwEElJSTh37hwaGhpw8OBBXLt2DT4+Pti4cSM2b96M5ORkVFdXo6ysDDqdDlu3bgUAvPzyyxAEAcuXL0dlZSVOnDiBP//5zwNuOyYmBjU1NXjrrbdQVVWFjIwM7Nmzx+KY+Ph4FBQUIDY2FgaDATU1NThy5AhiY2MBAMePH0dycjIMBgM+//xz7N27F93d3fD29oZSqUR8fDzWrFmDvXv3ora2FmfPnkVqaup9vVZhYWFQKpWIiIhAeXk5cnJysGLFCoSHh4uPKAwmT09PpKWlwWQyobCwEGFhYX3uWru6uiI7OxuNjY1obm4e9D4wxhh7TDzqh4gZ+zGrrKykOXPm0MiRI0mhUJCXlxelpKSI+/ft20cTJ04kuVxO9vb2NG3aNDp48KC4/8yZM+Tv709yuZwmTpxIBw4cGPCkNSKiY8eOkVarJYVCQUFBQbR7926LSWtERHq9nkJCQsjGxoasra3Jz8+PEhMTiej2BLbp06eTvb09qVQq8vPzo48++kg8t6urizZt2kQajYasrKxo7NixlJSUNKC+3TlpjYjIaDTSc889R0qlkhwcHGj58uXU2toq7o+IiKCFCxdanNMzaa33a5KTk9PnOnU6HdnZ2YnbxcXFFBAQQEqlkjw9PWn//v2k0Who27Zt4jFHjx4lrVZLMpmMNBrNgK6LMcbYj49ARPRoU27GWI/6+nq4ubmhpKSEP46XMcYYGyT8SANjjDHGGBvSOOFl7DEVExMDGxubfr9iYmIead+SkpLu2rfQ0NBH2jfGGGPsTvxIA2OPqaamJrS0tPS7z9bWFqNGjXrIPfq/r776Cl999VW/+1QqFcaMGfOQe8QYY4zdHSe8jDHGGGNsSONHGhhjjDHG2JDGCS9jjDHGGBvSOOFljDHGGGNDGie8jDHGGGNsSOOElzHGGGOMDWmc8DLGGGOMsSGNE17GGGOMMTak/Rc39Rt9oA4N2AAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "bound_make_plot = pn.bind(make_plot, \n", + " attn_bias_type=selector_widgets[0],\n", + " attn_mask_type=selector_widgets[1],\n", + " qkv_layout=selector_widgets[2],\n", + " bias_shape=selector_widgets[3],\n", + " is_training=selector_widgets[4],\n", + " swa=selector_widgets[5],\n", + " dropout=selector_widgets[6],\n", + ")\n", + "pn.Row(\n", + " pn.Column(*selector_widgets),\n", + " bound_make_plot\n", + ").servable()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60eba2e4-45c7-4d05-a090-802aaf0d494a", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1299c7d5-725c-4965-bceb-4ba7a2932322", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "06914d05-ec6f-4bff-82ed-6ec2dbdefd51", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/benchmarks/attention/requirements.txt b/benchmarks/attention/requirements.txt new file mode 100644 index 000000000..48bec5c1d --- /dev/null +++ b/benchmarks/attention/requirements.txt @@ -0,0 +1,4 @@ +seaborn +panel +watchfiles +pandas \ No newline at end of file diff --git a/transformer_engine/common/ck_fused_attn/src/ck_fused_attn_bwd.cpp b/transformer_engine/common/ck_fused_attn/src/ck_fused_attn_bwd.cpp index 2b717ace0..b6d95d542 100644 --- a/transformer_engine/common/ck_fused_attn/src/ck_fused_attn_bwd.cpp +++ b/transformer_engine/common/ck_fused_attn/src/ck_fused_attn_bwd.cpp @@ -4,6 +4,7 @@ * License for AMD contributions = MIT. See LICENSE for more information ************************************************************************/ +#include #include #include #include @@ -415,6 +416,12 @@ void log_bwd_config(const char* func_name, } +void dump_bwd_timings(const char* dump_path, float average_runtime, hipStream_t stream){ + std::ofstream file; + file.open(std::string(dump_path) + "aiter-bwd-timings.txt", std::ios_base::app); + file << average_runtime << "\n"; +} + hipError_t ck_attn_bwd( DType dtype, uint64_t b, uint64_t h, uint64_t hg, uint64_t s_q, uint64_t s_kv, uint64_t d_qk, uint64_t d_v, uint64_t bias_b, uint64_t bias_h, @@ -489,9 +496,10 @@ hipError_t ck_attn_bwd( if (env_p != nullptr && std::string(env_p) == "1") ck_fused_attn_log_config = true; } + const char* dump_path = std::getenv("NVTE_DUMP_AITER_RT"); // print kernel name on verbose mode - ck_tile::stream_config stream_config{stream, false, ck_fused_attn_log_config}; + ck_tile::stream_config stream_config{stream, dump_path!=nullptr, ck_fused_attn_log_config}; ck_tile::index_t shape_seqlen_q = seqlen_q; ck_tile::index_t shape_seqlen_k = seqlen_k; @@ -651,6 +659,9 @@ hipError_t ck_attn_bwd( uses_bwd_v3, is_v3_atomic_fp32, how_v3_bf16_cvt); + if(dump_path){ + dump_bwd_timings(dump_path, average_runtime, stream); + } if(average_runtime < 0){ //TODO: better error out system throw std::runtime_error("fused attn configs not supported in ck_fused_attn bwd pass."); @@ -841,8 +852,9 @@ hipError_t ck_attn_varlen_bwd( if (env_p != nullptr && std::string(env_p) == "1") ck_fused_attn_log_config = true; } + const char* dump_path = std::getenv("NVTE_DUMP_AITER_RT"); // print kernel name on verbose mode - ck_tile::stream_config stream_config{stream, false, ck_fused_attn_log_config}; + ck_tile::stream_config stream_config{stream, dump_path!=nullptr, ck_fused_attn_log_config}; std::string data_type_str = get_data_type_str(dtype); @@ -996,6 +1008,9 @@ hipError_t ck_attn_varlen_bwd( uses_bwd_v3, is_v3_atomic_fp32, how_v3_bf16_cvt); + if(dump_path){ + dump_bwd_timings(dump_path, average_runtime, stream); + } if(average_runtime < 0){ //TODO: better error out system throw std::runtime_error("fused attn configs not supported in ck_fused_attn bwd pass."); diff --git a/transformer_engine/common/ck_fused_attn/src/ck_fused_attn_fwd.cpp b/transformer_engine/common/ck_fused_attn/src/ck_fused_attn_fwd.cpp index c87a3db6c..b50c50098 100644 --- a/transformer_engine/common/ck_fused_attn/src/ck_fused_attn_fwd.cpp +++ b/transformer_engine/common/ck_fused_attn/src/ck_fused_attn_fwd.cpp @@ -4,6 +4,7 @@ * License for AMD contributions = MIT. See LICENSE for more information ************************************************************************/ +#include #include #include #include @@ -107,6 +108,12 @@ void log_fwd_config(const char* func_name, } } +void dump_fwd_timings(const char* dump_path, float average_runtime, hipStream_t stream){ + std::ofstream file; + file.open(std::string(dump_path) + "aiter-fwd-timings.txt", std::ios_base::app); + file << average_runtime << "\n"; +} + hipError_t ck_attn_fwd( DType dtype, uint64_t b, uint64_t h, uint64_t hg, uint64_t s_q, uint64_t s_kv, uint64_t d_qk, uint64_t d_v, uint64_t bias_b, uint64_t bias_h, @@ -166,9 +173,9 @@ hipError_t ck_attn_fwd( if (env_p != nullptr && std::string(env_p) == "1") ck_fused_attn_log_config = true; } - + const char* dump_path = std::getenv("NVTE_DUMP_AITER_RT"); // print kernel name on verbose mode - ck_tile::stream_config stream_config{stream, false, ck_fused_attn_log_config}; + ck_tile::stream_config stream_config{stream, dump_path!=nullptr, ck_fused_attn_log_config}; std::string data_type_str = get_data_type_str(dtype); @@ -272,6 +279,9 @@ hipError_t ck_attn_fwd( bias_type, has_lse, uses_fwd_v3); + if(dump_path){ + dump_fwd_timings(dump_path, average_runtime, stream); + } if(average_runtime < 0){ //TODO: better error out system throw std::runtime_error("fused attn configs not supported in ck_fused_attn fwd pass."); @@ -338,9 +348,9 @@ hipError_t ck_attn_varlen_fwd( if (env_p != nullptr && std::string(env_p) == "1") ck_fused_attn_log_config = true; } - + const char* dump_path = std::getenv("NVTE_DUMP_AITER_RT"); // print kernel name on verbose mode - ck_tile::stream_config stream_config{stream, false, ck_fused_attn_log_config}; + ck_tile::stream_config stream_config{stream, dump_path!=nullptr, ck_fused_attn_log_config}; std::string data_type_str = get_data_type_str(dtype); @@ -447,6 +457,9 @@ hipError_t ck_attn_varlen_fwd( bias_type, has_lse, uses_fwd_v3); + if(dump_path){ + dump_fwd_timings(dump_path, average_runtime, stream); + } if(average_runtime < 0){ //TODO: better error out system throw std::runtime_error("fused attn configs not supported in ck_fused_attn fwd pass.");