From 418cf7bb08eb890af0c9e90ed5f7ced9070462a5 Mon Sep 17 00:00:00 2001 From: stfrheitz <friedrich.heitzer@stud.h-da.de> Date: Tue, 21 Nov 2023 00:27:22 +0100 Subject: [PATCH] dfsdsf : --- Notebooks/friedrich/free_p1.ipynb | 144 --- Notebooks/friedrich/p3/GINN_P3.ipynb | 1388 +++++++++++++++----------- 2 files changed, 797 insertions(+), 735 deletions(-) delete mode 100644 Notebooks/friedrich/free_p1.ipynb diff --git a/Notebooks/friedrich/free_p1.ipynb b/Notebooks/friedrich/free_p1.ipynb deleted file mode 100644 index 63e5706..0000000 --- a/Notebooks/friedrich/free_p1.ipynb +++ /dev/null @@ -1,144 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": true, - "ExecuteTime": { - "end_time": "2023-10-27T20:14:13.177295Z", - "start_time": "2023-10-27T20:14:12.730428Z" - } - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "import matplotlib.pyplot as plt" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "outputs": [], - "source": [ - "class Model:\n", - "\n", - " def __init__(self, theta=None):\n", - " if theta is None:\n", - " theta = [0.12, 1.6, 3.2, 1.5, -1.1, 3.2, 1.4, 0.54, -3.3, 3.8,\n", - " 2.6, -4.5, -3.4, -2.0, -3.3, 1.7, 3.2, 7.2, -6.0, -1.8]\n", - " self.theta = theta\n", - "\n", - " def forward(self, x):\n", - " a = 1/(1+np.exp(-(self.theta[0]*x[0]+self.theta[1]*x[1]+self.theta[2])))\n", - " b = 1/(1+np.exp(-(self.theta[3]*x[0]+self.theta[4]*x[1]+self.theta[5])))\n", - " c = 1/(1+np.exp(-(self.theta[6]*x[0]+self.theta[7]*x[1]+self.theta[8])))\n", - " d = 1/(1+np.exp(-(self.theta[9]*a+self.theta[10]*b+self.theta[11]*c+self.theta[12])))\n", - " e = 1/(1+np.exp(-(self.theta[13]*a+self.theta[14]*b+self.theta[15]*c+self.theta[16])))\n", - " hat_y = np.tanh(self.theta[17]*d+self.theta[18]*e+self.theta[19])\n", - " return hat_y\n", - "\n", - " def predict_list(self, x):\n", - " return np.asarray([self.forward(q) for q in x])\n", - "\n", - " def predict_mgrid(self, x, y):\n", - " return np.asarray([self.predict_list(list(zip(i, j))) for i, j in zip(x, y)])" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-10-27T20:19:12.157699Z", - "start_time": "2023-10-27T20:19:12.152231Z" - } - } - }, - { - "cell_type": "code", - "execution_count": 14, - "outputs": [ - { - "data": { - "text/plain": "<Figure size 640x480 with 1 Axes>", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ8AAAGJCAYAAABVbT4SAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAC8NElEQVR4nOydeZwcdZn/P33Ofd/3TCaTzEzOmYRkZiKXBDlNgiBRUUERVnFVBNddPH7iyaK7iqDgqqygnLoQAUEhIAkJuZOZnvvO3Pd0z9H3Vb8/hm9NdXVVd1V1dU9PUu/Xa14wPdXV1Z3q+tTzfJ/n+agoiqKgoKCgoKAQQdQrfQAKCgoKChcfivgoKCgoKEQcRXwUFBQUFCKOIj4KCgoKChFHER8FBQUFhYijiI+CgoKCQsRRxEdBQUFBIeIo4qOgoKCgEHEU8VFQUFBQiDiK+CgoKCgoRBxFfBQUFBQUIo4iPgoKCgoKEUcRHwUFBQWFiKOIj4KCgoJCxFHER0FBQUEh4ijio6CgoKAQcRTxUVBQUFCIOIr4KCgoKChEHEV8FBQUFBQijiI+CgoKCgoRRxEfBQUFBYWIo4iPgoKCgkLEUcRHQUFBQSHiKOKjoKCgoBBxFPFRUFBQUIg4ivgoKCgoKEQcRXwUFBQUFCKOIj4KCgoKChFHER8FBQUFhYijiI+CgoKCQsRRxEdBQUFBIeIo4qOgoKCgEHEU8VFQUFBQiDiK+CgoKCgoRBxFfBQUFBQUIo4iPgorAkVRK30ICgoKK4h2pQ9A4eKCoii43W7YbDZoNBpotVpoNBpoNBqoVKqVPjwFBYUIoaKUW1CFCOH1euFyueDxeOBwOEBRFC04arUaWq1WESMFhYsERXwUwg5FUfB4PHC73fB6vVCpVHA6nVCr1aAoiv4hfwOWxEin09HRkVqtVsRIQeECQhEfhbBCURQd7RBGRkYwMjKClJQUpKamIjU1FTqdjt6eKUYAoFKpaDEikZEiRgoKqxtFfBTChtfrhdPphNfrhVqthtvtRltbG4xGI0pKSmCxWDA3Nwer1YqkpCSkpaUhLS0NKSkp0GqXliMVMVJQuDBRxEdBdkiazeVygaIoqNVqzM/Pw2AwICEhARs2bIBKpaJ/HA4HTCYT/eNwOPzESKPR0PvmEiOSnlPESEFhdaCIj4KscKXZBgYG0NfXh7Vr16K0tBQURcHpdNLiw8Zut/uIkdPpRHJyMi1GycnJnGJEftRqtV8BgyJGCgrRhSI+CrJAIhES7ZCigubmZlitVmzduhUpKSkAltNxfOLD3q/NZsPc3BwtRm6320+M1Go1vT0zKgKW03SKGCkoRA+K+CiEDOndcbvdAJYu9rOzs2hubkZGRgY2bNhAr+EA4sSH67WsVquPGHk8HqSkpNBilJSUxClGCwsLGBsbQ1VVlU+PkVarlXQsCgoK0lGaTBVCgkQ7zEiju7sbQ0NDqKqqQkFBgawXdZVKhYSEBCQkJKCgoAAURdGFCyaTCUNDQ6Aoiq6iI2Kk1WpBURTMZjNUKhXcbjdcLhctOuw1I0WMFBTCixL5KEiCmWYj1Ww2mw0GgwFerxdbtmxBYmIi73MdDkdYLvBEYEwmE+bm5jA3NwcASE1NhV6vh9FoRH19Pf26zMiIpAuJGJE+I5KmU1BQkA9FfBREQ9ZhyOK+SqXCxMQE2trakJ+fj/Xr19MFAXzPdzqdABD26IKiKCwuLsJkMmFqagqLi4vQ6XR0VJSamoqEhISgYsSevqCIkYJCaCjioyAKEu2cOHECRUVFyMnJQWdnJyYnJ7Fx40bk5OQE3UckxYeJ0WhEV1cXqqur6fWi+fl5aDQaer0oLS0NcXFxvGIEcI8CUsRIQUEcypqPgiC4endsNhuOHz8OvV6PhoYGxMXFrfRhBkWlUiElJQUpKSkoLS2lCxFMJhMmJyfR09MDrVbrI0axsbF+Ta8ul8unaEIRIwUFcSjioxAUdu8OaQzt6+vDmjVrUF5evmoutuxAX61W08UJZWVl8Hg8tBiNj4+jq6sLer3eT4yY1XQkGnS5XADgJ0akmk5BQWEZRXwUAkKiHVJU4HK50NbWBpvNhuLiYlRUVKz0IcoKMwUHLL3/+fl5mEwmjI6OorOzEzExMT5iFBMTQz+fKUYkMlKr1ZzVdAoKFzOK+Chwwu7dUavVmJubg8FgQFJSEtLT0xEfH7/CRxl+NBoN0tPTkZ6eDgBwu910Fd3w8DDa29sRHx9PFy+kpaVBr9fTzw8kRsxqOkWMFC42FPFR8IOrd6e/vx/9/f2oqKhASUkJmpqaVp0bqRwXeK1Wi8zMTGRmZgIAXC4XLUaDg4Noa2tDQkKCjxgxJ3YDy59vf38/YmJikJubq3gZKVx0KOKjQMPVu+NwONDc3Ay73Y6dO3ciOTkZwNKFfLWJDyC/fbdOp0NWVhaysrIALIkR6TE6f/48WltbkZiYSKfoUlNT6eIFm81GH5PT6aR7nxRjPYWLAUV8FAD4FxWo1WpMT0+jpaUFWVlZqK2t9RmRs1rFJ9zodDpkZ2cjOzsbAOB0Oumy7t7eXh/7CIfDgdjYWJ8hqcBSZORwOOhydEWMFC5EFPFR8LG3Ju6inZ2dGBkZwYYNG5Cfn+/3HEV8hKHX65GTk0P3PzHtI0hVndFo9LOP0Gg0PpO6FTFSuNBQxOcihm1vrVarYbVaYTAYAAANDQ1ISEjgfG4o4uN2uzEyMoL4+HikpqYGnIZwoUHWeHJzc+H1ehETE4OEhASYTCZ0dHTw2kdwiREzTacY6ymsNhTxuUjhSrONj4+jra0NhYWFWL9+fcDeHanis7CwgKamJrps2+l0+kykZtojyE20RWsURUGn0yEvLw95eXl+9hFjY2Oc9hFcYmS32wEoLq8KqwdFfC5C2L07Ho8H7e3tmJ6expYtW+j1ikCo1WqfarhgUBSF4eFhdHV1obS0FMXFxfRFk6ShRkZG4PV6/ewRLuQLJ/O9qVQqxMfHIz4+Hvn5+X72ESMjI5z2EcHESHF5VYhGFPG5iGD27pAROQsLCzAYDIiNjcWuXbsQGxsraF9iogi3243W1laYTCbU1tYiPT2dngZALrZMewQiRoODgwDg09AZHx9/wVw4g31+fPYRpJqOaR9BKumIlxFTjLxeLy1GisurQrSgiM9Fgtfrhdvt9hmRMzg4iJ6eHqxZswZr1qwRdQESKj7z8/MwGAyIi4tDQ0MDYmJieJ+nUqmQmJiIxMREFBUVwev10vYIMzMz6O3t9Zu7thrmyQVC7GfO/HzY9hEDAwMA4CNGiYmJfmLk8Xjg8Xhgt9sVMVJYMRTxucDhsrd2uVxoaWnB4uIitm/fTo+SEYNKpQqYdqMoCkNDQ+ju7pYkbsDSXXpycjKSk5NRUlICr9dLj7ohc9cCjbrhOuYLCZVKhaSkJCQlJdFpTGIfYTQa0d/fT8+uY9pHsIekEjEyGo1YWFhAYWEhPX1BcXlVCBeK+FzAkDtjl8tF2wQYjUY0NzcjNTUVu3btorvvxRIo8nG5XGhtbcXc3By2bdtGj6YJFbVa7Td3jayHkFE3ZLoAudiy31+0FRzI7fLKFmsiRtPT0+jt7eW0jyBiZLfbMTExgby8PLjdbsXlVSGsKOJzgUKinfPnz8PpdGLDhg3o7e3FwMAA1q9fj6KiopAuIHziQ+a/JSYmYteuXT5zzuRGo9EgIyMDGRkZAJZH3ZhMJvT398NisdANnWlpaRfdBVOtVouyjyCCw46MuCzHFZdXhVBRxOcCg927o9Fo4Ha7cfr0aTidTtTV1SEpKSnk12FXu1EURa8hrV27FqWlpRG/2LNH3TAbOru6umC326FSqdDf3083dK7khTPSUVgw+4j5+XmoVCq0t7fz2kcoYqQgF4r4XEBw9e5YrVZMTU2hoKAA27Zt8xmREwrMyMfpdKK1tRULCwuS15DCAbOhEwCmpqbQ0dEBm81G99CQsuX09HR6cT6SrGQ0xraPGBkZwdjYGGJiYgLaR/CJEaC4vCoIRxGfCwSv1wun00n37ni9XnR2dmJiYgLJycnYuHGjrK9HxMdkMsFgMCA5ORkNDQ2i0mxyr3kEg8xR27BhA91DQyKjoaEhAMuVYmlpaUhISAjr8UXT+hOw9G+q1+tRXl4OQLh9BFuMFJdXBSEo4rPK4bK3tlgsMBgMUKvVKCsrw8LCQlhe22w248yZM7TNwmpYUyEXfGYPTWFhoU/Z8uzsLF0pxl6cl/s9RtNnxr4ZkGofobi8KghBEZ9VDJe99ejoKDo6OmiX0dHRUczPz8v6uk6nEyMjI7DZbNi5cydSUlJk3f9KwC5bZi/Od3d3+9hpp6enByzrFkK0RT7k5oUP9pqa0+mkCzyC2UeQ/bON9ZhipLi8Xlwo4rNKYUY7KpUKHo8HbW1tmJ2dRU1NDX23GqwfRyxGoxEGg4G+EF8IwsMF1+I80067o6ODTkGRHyll69F0oRWbBtXr9YLtI8i5EkyMFJfXiwdFfFYZbHtrlUpFD+uMj4/Hrl27fO7IiUWCHK9L3EzXrVsHlUqFqampkPe7WuCy0yYXWiF3/VxEW+Tj9XpDutAHso/o6uqCw+HwEyPmRHMuMSI3TomJiYoYXWAo4rOKYNtbq1QqDAwMoLe3F+Xl5SgrK/P7YoodAMoFcTO12WzYsWMHUlJSMDIyEnUXz2DIedHSarV+KShyoe3p6YHdbqcvtOnp6fQ06mhG7gIQdrUhc2J3IPsIciwAMDk5iYmJCWzevFlxeb3AUMRnFcBlb+10OtHS0gKLxYJLLrkEqampnM8N1UZgdnYWzc3NSEtLQ01NDX03L4eorQThEkz2XT9zWndbW5tPWTeZRg2s7rSbWOLi4hAXFyfKPgJYLt8Glqs6Fcvx1Y8iPlEOV+/O7OwsWlpakJaWhoaGhoBrDVJFgqIo9PX14fz585wTEaLNGyfaiI2N9fPpMRqN9CggkuKanZ1FQkICEhMTV/yiGcnSd6H2EWTK+sLCAm0fQY5VcXld3SjiE8Vw2Vv39PRgcHAQlZWVKCwsDPrFklJw4HA4YDAYYLfbsXPnTvoOlL1fqeKjUqnobvoL3a8H8L3QMsu6SeR67tw5usAhPT09bGXdwYh03xUTPvuI/v5+2vZDiH0En8srKetWJnZHD4r4RCFc9tY2mw0GgwEejwf19fVITEwUtC+xBQczMzNobm5GZmYmamtreRfNpYoPaX4dHR0FRVEXnEWCEIjo6vV6FBUVISsrix4AOjU1hZ6eHuh0Op/PRajPUiispPiwIfYRZB2ourraxz7i/PnzUKlUQe0jFJfX6EURnyiDK802OTmJ1tZW5OXlobKyUtTCtdDIx+v1ore3F4ODg6iqqkJBQUHAL6UU8bFarWhqagIA7Ny5E1qtlr7oEouE2NhYepFeavlyoGOONsgFkTkAlF3W3dnZSX8u5CccA1ujSXwI5OaLqw+LqymYOaEiPj4+qBgpLq8rhyI+UQTb3trr9aK9vR0TExPYuHEjXTUkBiGRj91uh8FgEDV4VKz4TE5OoqWlBfn5+aisrKQ9ZJizxcg4F2b5MrM0NzU1NeSKsWhap+I7Fq6ybvK5sCcLpKenCyrrFno80Tb6hu+YuLye2PYRWq3WR4zi4uIUl9coQhGfKIDdu6NWq2E2m2EwGKDVarFr1y7J6ahgBQfT09Nobm5Gdna2qMGjYiKq7u5uDA8PY+PGjcjLy6MfZ8Me5+J0OulF+s7OTjidTp9BoCTnv5oRcmHj+lyIGPX29sJmswXsnxFKNIqP0N4jsfYRzIndbDFyOByKy2sEUMRnhSFFBcePH8emTZsQHx+P4eFhdHV1oaSkBGvXrg3pgsAnEl6vFz09PRgaGkJ1dTUKCgpE71dIRNXU1AS3242GhgYkJCSIeg29Xk/3iZCKMeLSOTw8DIqifC4m4R4EGi2wJwswy7pJ/wyzrDs5OVnQORStaTcpQso3oWJubo5O8TLHJbHX1dgur3wFDIqxnnQU8Vkh2L07FosFTqcTPT09mJubQ21tLW2SFgpcaTdSvOB2u0UVLwTbLxNmRFVdXe13ARGb/mJWjJFqKLPZDKPRiNnZWfT19fnc2aanp/st0kfbRUKuiz1XWTcRo5GREXi9XqSkpNDraHxl3aFOOAgHXq9XlnU/diqTuODOzc35rKsx03Rc9hGkEGhsbAyJiYnIyMhQXF4loojPCsBVVKBSqdDU1ISUlBRZHUBJhEIudFNTU2hpaUFOTg6qqqokr6HwRT4URdGOqVVVVSgsLAz1LfC+PlmAJjl/skg/NjZGFy+QC260eAyFGy6RtlgsPqOAVCqVzx1/fHw8/e8ZbRdOsv4pN2wXXLH2EbOzs9BqtZzGeuw0nQI3ivhEGHbvDgD09/fD5XKhuLgYVVVVsl4AyGt4PB709vZieHgYGzZsQH5+fkj75RIf0h/kcDhkc0wVCtP+YM2aNfTFxGg00sULCQkJ8Hg8MBqNktdF5CQSxQ+kZDkxMRFFRUV0lZjRaPRZmE9LS4PVao26UvdIrUPx2UdwFXmkpqbC7XbTIkOOU3F5FYciPhGCq3fH4XCgpaUFNpsNsbGxyM7Olv3Ok5zsp06dAkVRktZeuGCLD3MMT6D+oEjBvpg4HA5MTEygr6/PZ12EREYrVbwQ6UiDWSVGFuZJxDg7O4vBwUFMTk76uZeuFCuVCgxmH2GxWNDX14eFhQWfQbKKy6twFPGJAFxptpmZGbS0tCAjIwM1NTU4efJkWGalTU9PAwCSkpI4116kwkznkWnXXGN4ooWYmBhkZmbi/PnzaGho8CleGBoa8ileSE9Pp1NR4SQayr6ZEePi4iJduEHGALW3t9N3/OQiK2fvVTDClXYTC7vI49ixY8jJyYHL5UJPTw9nxSGXGCkur8so4hNm2L07FEWhq6sLw8PDqK6uRn5+Pl1FI6f4eL1edHV1YXR0FACwbt06WdNMpIru7NmzsFgs9LTraIaICde6COkRmZmZ8SleIJFRuCYMRJNQUxTltxbCTD/19fX5efTI0XsViGgRHzYURSEzM5M+54XaRygur8so4hMmmL07JG9Nqsy8Xq9flZmc4sOcJFBfX48jR47IHlWZzWb6whBsuCkX0fSlUqlUPg2LHo8HCwsLMBqNdCVUXFxcyMZx0Q5XwQE7/cR1kWVOok5JSZFVLKJVfJhrtkBo9hFAcJdXs9mMpKSkFU2Byo0iPmHA6/XC7Xb7pNnGx8fR3t6OgoICrF+/3u8LJZf4TExMoLW11ed15JxATVEUBgcH0d3dDQCoqamJKiEJhpDPQaPR+E1eYBvHJSUl0VFRKE2d0fTZCVnc57rIks+G2CIwy5VDHRwbjY2vQPD+Iyn2EYHEaM+ePbj77rtxxx13RODdRQZFfGSEecIw7a07OjowNTWFzZs30zljNqGKj8fjQWdnJ8bHx7Fp0ybaV0aOfRNcLhdaW1sxPz+PLVu2oLGxMeR9rgbYxnHk7t9oNKKjowMul8tv8oKQC240rPkwkSKG5CLLtEUgn83g4CAA+IiR2EbgaOw9It9zoaLIZx9BhqQS+wi23xNTjKxWqyyFQtGEIj4ywWVvvbi4CIPBAL1ej127dgVcNwhFICwWC5qamugUWHx8vM/f5Yh85ufn0dTUhISEBDQ0NND7k3r3Hm0XFDEw7/6ZFxKTyYShoSEA8LFHiETxghyEeqFn2iIQ6wiylkYagZlRpRDriGhMu5HvaSg9cuzPifRizc3N0QUwqampGB0dRXJyMiwWi+hm8Pfeew8/+9nPcPbsWYyPj+PAgQPYt29fwOccPnwY9913H9ra2pCfn49vfvOb+OIXv+izzUsvvYTvfve76OvrQ3l5OX784x/jpptuEvsxKOIjB8zeHZKrHRoaQnd3N8rKylBeXh70Sy1VfMbGxtDW1oaioiKsW7eOdwijVGGjKIoe97NmzRqsWbMGKpWKNu+Ktrv3QIRDAPguuOw+GiJE6enpPnn7aBIludOA7LU09ry17u5uxMTEBCzrjkbxYabT5YDdi0Wmd5hMJjz99NN47rnn4PV68ZOf/ARdXV248sorsWnTpqCvb7FYsGXLFnzuc5/DzTffHPQ4zp8/j+uvvx533XUXnnnmGbz//vu45557kJWVRT//+PHj2L9/P374wx/ipptuwoEDB3Drrbfi6NGj2Llzp7j3Ta2mq0eUwdW7Q1JTCwsL2Lx5Mz3OIxgtLS2Ii4vD2rVrBW1P0nmTk5PYtGkTbzoPAN59911s3bpVdJe/2+1GW1sbZmdnsWXLFp9xPy6XC++88w52794tqaeHVAFG8sJis9lw4sQJXHnllRF7TaY9gslkwsLCAt05Pzk5iQ0bNsgyRkkOTp48ifLycro3KtyQz4YMj11cXKQ/G/Jz4sQJbNmyhdPQcKWw2+04duwYrrzyyojcPDgcDlRVVeG2225DV1cXjhw5gn379uGZZ54RvA+VShU08vn3f/93vPrqq+jo6KAf++IXvwiDwYDjx48DAPbv34+FhQX8/e9/p7e59tprkZaWhueff17U+1IiH4lw9e6YTCY0NzcjOTkZDQ0NokbkiIlOzGYzmpqaoNVq0dDQELQrXayhHAAsLi6iqakJMTEx2LVrl98dKfnSSb13WVxchNVqRVpaWkQnDUT6Xos9U+z9gh3wAJgBoAHQ+cF2Xre047p08rQchwkg8gUQ7M+GWdZNCjsAYGRkBNnZ2bJZR4QKqXSL1Gel0+lgMpnw5S9/GRUVFXC5XDCZTLK/zvHjx/GRj3zE57FrrrkGTz75JFwuF3Q6HY4fP46vf/3rfts88sgjol9v5f8lVyFerxdOp9MnJdDX14fz589j3bp1KC4uFn1iChWf0dFRtLe3o7i4GBUVFYIiB7FW2uQ1SktLsXbtWs73wuxXEAOzWo7MxmKuj/ANvVwtvJe+HQCg0vm+B8oV+HMSuj17uyM5l9D/H6oQrXT1HddUgffffx8ejwc9PT2w2+0BK8QihdRJ21Kx2+3weDz0uCqdThcw0yGViYkJn0IlAMjJyYHb7cbMzAzy8vJ4t5mYmBD9eor4iICk2Ug1GxmRQ4zYdu7cKTk9oFar6SiKC7fbTVfNbd26lf6CCt23EJFgpvKCvQa5SIkRNbfbjZaWFszPz2Pbtm2Ii4uDw+GA0WikZ7CRjntyRxwJ+2ixvJe+HZo4f9H32JY/i2Biw0bo9uztiBiptSq8X7ADABCTo8f2c0dFvT6w8uLDRqfTgaIorF27FnFxcXS5stFopMuV2RVikUjjsnt8wo3FYgEASdPnxcL+9yfXDebjXNtIOW8U8REIV5ptenoaLS0too3YuCDrRVyQqjmdThe0ao5v38FEglTMaTQaQak8gtDIh6TxYmNj0dDQQL9fslhPhl6S5k7iuUKaO0lkFMpnLPXCSi7qTHFh/n804HVTUGuX3p9j0on3C3Zg1+gpUfuItp4acm6RY2L3zrCrDEmFGDlfwuXvFOnIx2KxQK1Wh33oa25url8EMzU1Ba1WS69L8m3DjoaEoIiPAEjH+8mTJ3HFFVeAoih0dnZiZGRElgnRALdAUBSF0dFRdHR0oLS0FOXl5ZIuDsFKrcfHx9HW1obCwkLeijmufQot4SYVecw0HilJZ8I0AAN8mzv7+vro+VkkKhJqkiYGZhqLidhIJhKwU3DMdSMSCWmTtdjZcUzQ/qIt8iHfB65/Y64qQ1IhRtaMmHPrhJR1C2UlIp9IGCXW19fjtdde83nsrbfewvbt2+mJHvX19Th48KDPus9bb72FhoYG0a+niE8AmL07KpUKDocDZrMZLS0tAIBdu3b59dRIhS0+zEqzmpqakCqQ+CIfr9eLzs5OjI2N+TWmCiGY+JD9j4+PY8uWLaLz1OzmTrvdTldGtbS0wOv1+qwXCf2CcgkMl7iodKqoFB0COTa2CJEIiHCyqkGQAEWb+LAjn0Aw/Z2Ki4vh9Xrpkndio63T6XidS8UQ6cjHbDZLEh+z2Yze3l769/Pnz6OpqQnp6ekoLi7GAw88gNHRUfzxj38EsFTZ9qtf/Qr33Xcf7rrrLhw/fhxPPvmkTxXb1772NVx22WV4+OGHsXfvXrzyyit4++23cfSo+DSvIj48MF1GAdDpnhMnTgTsqZEKUyAWFhZ8UlShrntwFRxYrVYYDAbaZkGKiAYSH5vNhqamJlAUhfr6ellEOjY2Fvn5+XSXOLnTZTqZsvtpTqyth2txKcqiXBS0AMgRk4s2n8BEs/AwYYoQW3jcC25ok7U4WdWAHe3vB7yARZv4kHNWyjGp1WqkpKQgJSXFx0bbZDL5OJcyxUhodepKRT5iOXPmjE9bwX333QcAuP322/HUU09hfHycbooGgLKyMrzxxhv4+te/jl//+tfIz8/Ho48+6tMj1NDQgBdeeAHf+c538N3vfhfl5eV48cUXRff4AIr4+MG2tyaFAO3t7QAgW5qNjVqthtvtxtDQELq6ugQ3pwrdN1MkiJtpXl4e1q9fL7ub6czMDAwGQ0C31FDfF/tOl6RGW8qvxgzZhhERcAnJahEXIZD3yk69AUsCBACnqnch/s//Q4szOw0VbaNsAqXdxMIu6yZmg0yzuMTERJ9p3Xzriyux5iNlSgZZIuDjqaee8nvs8ssvx7lz5wLu95ZbbsEtt9wi6li4UMSHAVdRwcLCAgwGAx19kPWIcLz24uIiLBYLamtrZW08JJGP1+tFT08PhoaGsHHjRuTl5cmyXwJFUXTJeXV1NQoKCkI9dEFwFQQAF5a4SMHrpqCJ00Cfvvw1t976L3A9+yv09PRAr9f7FHNEY+QTrgiDbTbINIvr7e3l9OchgrMSkU8kKt0ijSI+H8Blbz04OIienh56rMzbb78dFsO3+fl59Pb20ikwucemq9VqOJ1OnDp1Cm6328/OIZT9kjsrp9OJ5uZmWK3WkErOgyF0veZig/kZqHQqaOKW78ydRrePAMV/+9+x5W//oKcLENM4ABgYGEBWVlbIlYVyEMnROmyzOLvdThcvMJ1v09LSYLfbIyo+ZM3nQuOiFx++ETktLS1YXFzE9u3b6bE0wXpxpLw2mQGXnZ0Ns9kcFr8Op9OJgYEB5ObmhsXNdH5+Ho2NjUhOTkZ9fb0grxuhd9hcUU2wdNrFDOk/olxeqHTLF0giQJo4NeYHLH5pKKfTiaNHj9LRK7OyMBw+PUJYydLv2NhY5OXl+VgiEDGamZmhsyQkMgpnc7TValUinwsNrjSb0WhEc3MzUlNTsWvXLp8LqUajkS3yITPg5ubmsG3bNtoSQU4oikJvby/m5uaQlZWFTZs2ybp/AJicnMTw8DDWrl2L0tLSkL+ARGwIXP00iuBww258pVwfLNh/IEJEgNIq/KNSEuVUVFRAr9fTd/7Mhk5mD00kJlFEyxoU2/m2q6uLnjhAyrpVKpVP8YKck8ylFhxEOxet+HDZW/f29mJgYADr169HUVGR38kjly8O055g165d0Ov1mJ2dlTWl53A40NzcDJvNhuzsbNnvnMjnNzIygm3btgkeoMqE+fkq6bTQUOlUdLEBu+KNGQU5jW4swIzuPddi3av/WN6G1cnOvvO3Wq10mfvAwAB9sSWRkVw9NEyicaI1sHRcsbGxKCoqopujiXUEc5I5u8dIKmazWYl8LgTYvjtqtRp2ux0GgwFutxt1dXX0DCU2Go0mpLQbmWvW09PjFynIaaNtNBphMBiQlpaGmpoadHV1ySpsFosFjY2NoCgKGzZskCQ8AHBq3S44jR/414uchaawBPtzA+AjQjE5y+XDTuPSOR+f6V+6zzVGhX4NRkMn82JLemi6u7uh1+tpIeKyRpBCNIsP87iYZd2lpaV05aXJZKIndcTExPh8PmKGDlssFtET6VcDF5X4sHt3VCoVpqam0NraGrAsmBCKQDidTtpqgbmOJMe+CRRF4fz58+jr6/OJ3qRMteaD2HQXFhZienpa0PoO4d2srT6/hzILTWEJdqEBgRQcMIsN9OlaWngWRsy++wkgPmy4emhIpRgpXkhISKAjI6nTqKNt3A/B4/EEvE5w2bCTHqOhoSG0tbXRnw8p6w70PbLZbCgsLJT9faw0F4X4cNlbM7v7N27cSHvSB0Jq5GMymWAwGJCUlMRrtRDqepLT6URLSwvMZjN27NiBlJQU+m9yFEp4vV50d3djZGSE/ryMRqMgUQskOgryQYRIm+z7tWZXuxHO1H6IHkAqRnzYaDQaZGRk0O0BZOS/yWTinEYttHghWtZ82Ijt8yGz0bg+n76+PlitVp+y7tTUVJ/9K9VuqxR2UYFKpfIboim0+17sRZyiKAwMDKC3txcVFRUoKSnh/TKFEvnMzc2hqamJ9hFi30WJtVRgQ9KSLpcL9fX19BdByH4V4YkczEo3AJzVbtYZO+IzY1GwPQ+jZ8bpv4cyTYANGflPypaZlWLM4oVgNhrRmnYLtc+H/fk4HA768+nq6oLD4UBKSgpMJhPcbrfkPp/HH38cP/vZzzA+Po4NGzbgkUcewaWXXsq57R133IGnn37a7/Hq6mq0tbUBWGpK/dznPue3jc1mkzSF5YIWH67enZGREXR2dqKkpARr164VdRKJiU6Ykcgll1wStDmVpMbENPoFWkPi2rcUjEYjmpqakJmZ6Te5O9B4HbboAIrwhBOutR++ajcAMA2YULB9uck4lMgnGGQaNRmLZLFY6Eq6QANAo1V85J5wEBMTg9zcXOTm5oKiKLrS8O2338avf/1rWCwWTE1NYXFxEVdddRVqamqCvv6LL76Ie++9F48//jh27dqF//mf/8F1111He4Gx+eUvf4n//M//pH93u93YsmULPv7xj/tsl5ycjK6uLp/HpI7/uiDFh6t3hwzqNBqNkgd1Co18yIJ/amoqZyTCt29A+IlNSrXn5+c515CYSIl8mFEbX/Ufn/hwCY/XTSlFBWGCPVonWLUbYKfXfkjVW6SmG6hUKiQmJiIxMZG3eCEmJoaeuBCNhHPCgUqlosX6m9/8Jr7xjW/gsssuw/bt23H8+HE89NBD2Lp1Kw4dOhRwPz//+c9x55134gtf+AIA4JFHHsGbb76JJ554Ag899JDf9mQNj/DXv/4VJpPJL9JRqVSCliiEcMGJD1fvzvz8PAwGA13aLLUSJ1jkQ1EU+vv70d/fL9rRVIz4kMGj8fHxguy6xUY+TGFjrx8x4RIftvDw2UMrwiMPwardCHF5etjGnfTv1hk70kqXn7NSo3UCFS9MTk7C4XDg5MmTdIouGqy0IznbTaVSwWw2Y//+/fjIRz4Cj8eDycnJgM9xOp04e/Ys/uM//sPn8Y985CM4dkyYvcaTTz6J3bt3o6SkxOdxs9mMkpISeDwebN26FT/84Q9RU1Mj7k19wAUlPuzeHQB09ZccTZCBIh9mX02gC3agfQOBnUEpiqLThmTkj5D3IybyWVxcRGNjoyBhY4vPu1lbfca6eGwenwsguSgqwiMffNVu+vSlaFufuvwVz9i41Fy6OGYFsJR6SytdipijpbKMWbyg1WqxuLiInJwcGI1Gv+KFcHk6BWMlZ7tpNJqgg41nZmbg8Xgk212Pj4/j73//O5577jmfxysrK/HUU09h06ZNWFhYwC9/+Uvs2rULBoMBFRUVIt/VBSI+FEXB4XBgYGAAhYWF0Gq1PrPGhKy5CIGv2m12dhbNzc1IS0vD1q1bRZUfE4LZUjP9fcQOHhUa+YyOjqK9vV3wRG2m+DRedqmP8HAeh1bll35ThEg+KBcFbbLWZ9KBc85NC5B11o74jFgk5cdjccyK9DUZoD4436JtqCiw9F3QarW8xQtMT6dIjLlhHlekp1pLKTiQanf91FNPITU1Ffv27fN5vK6uDnV1dfTvu3btQm1tLR577DE8+uijoo9v1YuP1+ulm0a7urqQn59Pi0F6ejp27dolW5hOBnQSmFOc+dZFhEL6cbjEx2w2o7GxEXq9XpK/T7BKOjLaZ2JiAlu3bqXN24QcM0VRaLzsUiwOWH33afMXab4UnII8kBJrj83LK0CEwh3LE8e791yL3Gf+HJXiw44wuIoXyOQFdvECsY2QE7KeHKnIh7xHMaXWmZmZ0Gg0kuyuKYrC//7v/+Izn/mMoHT+JZdcgp6eHsHHxmTVig+7d4fcifT29mJsbAxVVVUoKCiQ9QvFXPNxOBwwGAyw2+2yTXHmEgkSjUipziMESrtZrVY0NTVBpVKhoaFB1JdVrVZj/KZb4Zh2+b6eTgWtTkv7yADcwqNEPfLAte5DKguJCBEBItEPwTRggjZWG7WRT6DznVm8QNxLyWSBiYkJn+IFsmYkZrIAFyTSj1TkY7VaQVEU79QVLvR6PbZt24aDBw/ipptuoh8/ePAg9u7dG/C5hw8fRm9vL+68886gr0NRFJqamiTPjFyV4sMekaNSqWCz2QAspcDksgxgQ9Z8ZmZm0NzcjIyMDNTW1soaWRGR8Hg86OjowOTkpKhohG+/XGm36elpNDc3Iy8vD5WVlaKFTaVSQZ+gg2PahbjcpS+1fXZZiHzuxDkykW6X2/9BBVEwhYdEm77rbl6/gaNcRGNDJ/OmUghqtRqpqalITU1FWVkZPVnAaDSKNozjg1nIFAksFgsAiL6e3XffffjMZz6D7du3o76+Hr/97W8xNDSEL37xiwDgZ6FNePLJJ7Fz505s3LjRb5/f//73UVdXh4qKCiwsLODRRx9FU1MTfv3rX0t6b6tOfJi9OyRVNT4+jra2NqjVamzatClsQ/jUajW9IB+OyIqID2mCVavVoqMRLrhM38gQ1VCcWamvfgVuu5sWHi74ensoF/dFUekFEg5XxAMsiRCXADHTb/MjJqQULpfnR2vkE8oxsScLOJ1Oer2ou7sbDocDycnJdFQkpHhBTndVIVgsS/YXYit09+/fj9nZWfzgBz/A+Pg4Nm7ciDfeeIOuXmNbaANLA49feukl/PKXv+Tc59zcHO6++25MTEwgJSUFNTU1eO+997Bjxw7O7YOxasSHq3fH6/WitbUVk5OT2LRpEzo7O8PWG2C32zE4OAiHw4H6+npRYbBQ1Go1ZmZmMDg4iMLCQqxbt06Wk5xt+mYwGGCz2QIOUQ1G42XcndJCIM2PTEhaTukFEgaf8BD06Vp6kGjK2gSYR5cyA2mlqTANzPmk3tx2d9SKj5wXeb1ej5ycHHrdgxQvGI1GjIyM+BQvpKenIyEhwe8zYd70RgKy3iPl9e655x7cc889nH/jstBOSUmB1Wr13/gDfvGLX+AXv/iF6OPgY1WID1fvjtlshsFggE6no6ODnp4eWc3eCCQ9lZiYCI1GExbhIRHdwMAANm/eHHRhUAwk8iFjeFJSUtDQ0CB7vwQz5QbwefHwCw8Xihj5E0h44gqW75D16VrEpS/9nlgQh4Qs/0Xrihu2ouf1JvT19cHlcmFxcTEiFWNCCPeEA6HFC0zbiEhXul2odgrAKhAfr9cLp9PpcyIODw+jq6sLpaWlKC8vpx8P1fKA67V7enowNDSE6upq6PV6v9EScmC1WmEwGODxeFBZWSmr8ABL4uN0OnH69OmgM+bE4LYHXrNJKPRNFbisy/82jkkne3NBKGXa3L09pK+HvcZjMzpoAWJinbUDMAEA0krTsP7MARzdcC3OnTvHedFdCSI5XoeveMFoNPrYIiQmJoKiKDidzpCLF4RgtVovyKGiQBSLD0mzkWo2MiKHuH9y9brIKT42m432+CEFDHIbvgFL5Y8tLS30yAopPUKBcLvdOH/+PFwuF3bu3CmLLwhXyo0d9cRmBH4fxGeGHR0Rfx8hkAvvxSpCwHJvD5NgRQZrrliL/kO9fo/Hx8dj27Ztfl40sbGxtOV2sPH/crKSs92YxQvA0vdobm4O4+Pj8Hq9OHr0KF28QD6XcEREFotFVlfUaCIqxYcrzTY3N0fbEhD3TzZyiQ8RBLbHj5zixoyqyKL/qVOnZI3czGYzXbig1+vDakjFtma2GW2S9qNP1/mJiWuRP8K6mIUH4O/tYf5Ooh/LtIVOvTHXfAhqtZqzYmxubg5GoxF9fX2w2WxISkqioyKh9ghSiKZ1KK1WS8+DtFqtqKmpodeLmJOoiRglJSXJ8rkoabcIwox2yIlH5qUFSxmFKg7Es2Z4eJizCkwut1G2RQE5ueR0M52YmEBLSwuKi4uRm5uLM2fOyLJfqTBTbgTuNSF/MWFPTiAlxRez8Gji1D6WCUBgAWJiGphB7qZCTLSM0I9NNfVCVef/vSIXXXLhJROXjUYjbY/AbOrkWqSXSjROtSZrPlzFC2S9iFm8QERa6ucitsF0NRE14sPVu0PmpdntdkHz0kIRH7Lu4vV60dDQwPkPLkfkQ3qEuCwK5BAfr9eLrq4ujI6O0oULi4uLsokaSbkx13vya/JhGjTJsn8h0GL0wVIEs5n1YiCQbw+7uZQIEN/az/yICWuu3oyZtkFBF8fY2Fjk5eUhLy/Pb5G+v78fWq2WFqL09PSQ7LSjUXz4phvExcWhoKAABQUFoCgKZrMZJpMJs7Oz6Ovroz8Xpm2EEBTxCTNse2u1Wk1XmGVlZQlu5JQqDpOTk2hpaaGbLflyt0QcpKQDmL01VVVVnLa4odpd2+12NDU1wePx+JjkyWWjPTY2FvI+CEKjHiEjedhrHhebGDEtEwh8UQ8z9cZG7DnNtUhPmjpHR0fR0dERkp12NIqPkGo3lUqFpKQkJCUl8RYvxMbG+hR18K2jSZ3rthpYUfFhjsghJxpFUejs7MTw8DCqq6tRUFAQfEcfIFZ8mFHChg0bkJeXF3B7ctKJLbdkTrwO1FsTit317OwsDAYDsrKyUF1d7XN8oTqZMj+nDCxVR013TgPgjnrY6z1cKbdwcqGKUaAiAq4oKGfrsmfVTJfRL/ohqbfZjuGl54aYLmPOVQP47bRJVBRsXSQaxUfKXDe+4gVS0t3a2orExEQf2wjy/VXEJwxwFRWQ1BcA3tRXIDQajc/gz0CQmWbktYRYaZOTzuPxCBYfYiyXlpaGmpqagHd+UtJuFEXRthGVlZUoLCz0u4iQ45YSsTkcDjQ1NcHtdqOhoQFTpctFC/k10iYjcCE16gm8T98UFLA6JygIGY9DSF0n/UIl9+I+n502s6mTeffPruqKFpsHJnL0+bDX0ZxOJ526JMULRqMRp0+fhtFolDSBRIyF9qFDh3DllVf6Pd7R0YHKykr695deegnf/e530dfXh/Lycvz4xz/2mR0nlhURH1InT+4iVCoVxsbG0NbWhsLCQqxfv17SSSc08pmYmEBrayvy8/NFzTRjRj7BYIqCUGM5seLjcrnQ3NwMs9kccE1MrEsqwWQyoampCRkZGdiwYQPUrzwGANDF6UISnpUUAPaFPFrESKVT8RRbBD43SVOpbdSBxLI4uKwu6OKXUjjWWSviM5ZuqsouL8VEyziApdQbAKSV+rr5hruyjN3UaTabYTQaMT09jd7eXuh0OlqI0tPTo3LeXDgmWuv1eh8bbZvNhqNHj6KjowOnTp2C2+1GT08Pdu/ejauuugrV1dUBPxexFtqErq4unwHJzHmSx48fx/79+/HDH/4QN910Ew4cOIBbb70VR48exc6dOyW9bxW1Ql61JOrxeDxob2/H9PQ0Nm3aRN8lSWFoaAjT09PYtm0b5989Hg+6urowNjaGjRs3SrKDffPNN/GhD30oYFTmdDrR0tICs9mMrVu3CjaW6+zshNfrRXV1ddBtFxYW0NjYiMTERGzatClgw5vL5cI777yDq666SlCPBkVRGBoaQnd3t49wDnx2D4Al8WEiJO1WXL+8xuVYtPu9ptPiH7F6PohcZprnAADpVb7pSmPHIv974JikIAQhYiRnoyvXtAKyT6HCAwAa/fK2RHwA0OKTlLt0USEClJCVgLTSTKSsL8XEiTba1yfvt3+R8jZCxuPx0OtFRqMRZrMZAJCbm4ucnJyw9dGIhdgHSDFPk8InPvEJVFRUoKCgAO+88w5GR0fR0tIS8Dk7d+5EbW0tnnjiCfqxqqoq7Nu3j9NCm0Q+JpOJ1/ds//79WFhYwN///nf6sWuvvRZpaWl4/vnnJb23FUu7qVQqLC4uoqmpCbGxsdi1a5donxo2gSIf9rBOIWk2vtcIFJ2QETbJycloaGgQ1ZBHGmmDMTIygo6ODsFupsy0WzA8Hg9tWrd9+3bRvUFs4Sm7ogJ2k1nUPoBl4QGAzM2pAACvy/ffNr0qyWc7AJjvtUgWHmD5gq/SqTnXithiEcr4H74xOSqdyscBlgum8ACAx+mlBYgZ/bBZv2cbul496/d45oYSzLQNCjnssKDRaOi1IGDpBu7o0aO015TT6URKSorPetFKREUejydiTbbA0vJAZWUlvvjFL+Lf/u3fgn6HQ7HQrqmpgd1uR3V1Nb7zne/4pOKOHz+Or3/96z7bX3PNNXjkkUfEvSEGKyY+Q0NDaG9vF2UHHQw+8SEpvaKiopCHdfIVBVAUhcHBQfT09Ei27A4mbCRKnJqaQk1NDZ0zFnLMQPB0odVqRWNjI7RaLerr60O6GSi7YunOUIrwhELK2gR4Pctf0IVu8a9PFu2ZhQvuBXfQYZ5LzxUWFQXal1jhCQRJvS1OLCApNxnz5+WrWAwn5AJfUVGBmJgYnz4aMo2ZvV4UCSJdBMEerxPsmiLFQjsvLw+//e1vsW3bNjgcDvzpT3/CVVddhUOHDuGyyy4DsLRUIdWWm48VE5+4uDhJd9aBYIsP0xNny5YtIaX0mK/Bvoi7XC567E8o7ynQmg8RBo1GIzpKJCdsoLsmUtqen58vas2NnXKLS49D7mb/MvJAcKXc5CKZtQAvRYwA/wo6LqdWNnyFDuESHjHRz8jxzoCvs9KQc5WsCcfHxyM+Ph6FhYXwer1YXFyE0WjE5OQkbRpHoqJApcuhIqbYSA4iYaG9fv16rF+/nv69vr4ew8PD+K//+i9afMTuUwgrJj7Z2dmCUkxiYIoPGS2j1Wpl8cQhsCOfhYUFNDU1IS4ujnfsj1D4SqKnpqbQ3NyMgoICycUYfMLGtAIPxduHIFZ45IYZ9XARTIzY/TJ8kEZXtVbFOf6HLSJMIeKr4tMlLX8dhYhbqJgGZuj/J+XW0QI5V7kubmq1GikpKUhJSfEZAcQsXWaPAJJLMCI51Zo08YqZoh+KhTaTuro6PPPMM/Tvubm5Ie+TTVQ0mcoFER9iPV1cXIyKigpZw2QS+VAUhZGREXR2dqKsrAzl5eWy9EkwBcLr9aK3txeDg4PYuHFj0D6kQHAJG6mWs1gsIXn7ENbesB3mYelheCDY6z0A/NZ7pMAUI7FREREYpmi4Ft0BoxdNnAYaxrYE5j7IdlwCpNKoQAUQWK7oh1n1xkVu3QZMnGhD9ta1vNtEGjGmbezSZYfDQZd0t7e3w+12+4y6CcUyIhzVboEQG/mEYqHNpLGx0ed6U19fj4MHD/qs+7z11ltoaGgQvE82F5T4AEsX1M7OzpCtp/lQq9VwuVxoaWnBzMwM53RtqTBTeg6HAwaDgTavC7XRjD3lgDiyJiQkoL6+PmKLqFyVbtFCKqOabr7XImkfJC3GZRnBnlHHFhy+7YkIxRcvpVrFCBATsu6zGgjFMTQmJsandNlqtdJVdGyfnvT0dFEp7Ej7+UixVBBrof3II4+gtLQUGzZsgNPpxDPPPIOXXnoJL730Er3Pr33ta7jsssvw8MMPY+/evXjllVfw9ttv4+jRo5Lf24pWu8nJ4uIiDAYDKIqSNc3GhqIo9PT0ID4+Hg0NDSFX6DEhkQ/pr0lLSxM8WkjovoHlAgwxERspsy64vAZTp1r9/r72hu0hHyNBjogmVFLWLn3h+UQo2NoMsYwgIsQWHiZxuXrYJvjXvTI2p8A25wj4emIprK/0W/fxulxY+WLmJUiPT6jXCZVKhYSEBCQkJKCoqIhz1E1cXJxP8UKg71skIx+v1ytpzUeshbbT6cQ3vvENjI6OIi4uDhs2bMDrr7+O66+/nt6moaEBL7zwAr7zne/gu9/9LsrLy/Hiiy9K7vEBLoDIh6Ioeo5UYWEhBgcHw2byNDY2hoWFBaSnp2Pbtm2yn4QqlQo2mw1nzpwR3JgqZt+kWm58fFzWyFBO4Yk2mL1FgfqK+CAi5F7gXsOJy9XT/w0kQGyCRT8Eoak3gufxB6C5x78XJNKEa7oB16gbMgKIbRmRnp6O5ORkn+OIZORjsSzd+EhJh4ux0P7mN7+Jb37zm0H3ecstt+CWW24RfSx8rGrxcbvdaG9vx8zMDGpqapCWlobBwUHZK1KYVXOpqanIzMyU/YvhdrsxODgIp9OJHTt2yO69o1Kp0NHRAZVKhfr6+oiVpkaSYMUGoUKEaK6He21Im8x9zsWlxwDpwOKANeD+AwlQXGqMX/TDJUBrriyn/1+fFI+B97r99jV/fgwpZUuFJaaBGaSsL/X5e2NjI33xXSlL7UhNN9BqtcjKyqJvxOx2O13S3dLS4meN4Ha7Ixb5WK1L54sy1VpmQj2xSINqTEwMnf4iaxput1u26IfdnNrT0yOr4RuwVJlHyqjJtFs5MZlMsNvtSEtLw7Zt20ISZnbKjRn1SCk2kFpmLXdqTq0J/4UuqXRJ8IkIkaiHCVuAUiuEp1xKLy31f+yydZwCxMVsxzCyNq9BVlYWjEYjBgYGoFaraSEK1SJBDCs1VDQ2Nhb5+fk+I4CY1gherxcDAwPIzs4O++dhsVig0+ki9plHmlUX+TCrzEpLS7F27VpayFQqVdBGTTGQGXDMEmc59w8sr7+UlJQgMzMTzc3Nsu2b2fiq1+tRUlISFSNKxMJV6RYJNDxl11lblm4Opg3LPU4Box4WSaXxAaMgIkBs4QkU/XAJD6Fq/y50vPg+AN+ig4rbb0TP038DsFzxBgCFhYV0Pw1ZH2FaJDAttcN1PkXDRGu2NYLH48Hhw4cRExNDTxkhnweZRi3H+izBbDbLas4Xbawq8XG73WhtbYXJZOKtMpPD8I1pIcCeAReK7QH7NTo7OzE2NkY3wM7Pz8smbG63G21tbTAajdi+fTs6OjpC9vThKzbgI9LTDSIBs3Eza0uajwCJIak03seQjw1XVMRHIOEhVO3fhcG3G/0eTyvNxHzXAFLWlyKjqsjnb8z1kTVr1tAWCUzraJKSkjtFF40TrcnxlJWVISYmxufz6O7ult1Km4jPhcqqSbsxmzkbGhp4Q1GNRhNS86rNZkNTUxNdNcdeG9FoNHC5XJL3H+g15LLRtlgsaGxshE6noz8ruQzlmGRsLA++kUikptPCvd7DB4mCTOcXZN1vTBK3+LCjnzVXVsDjkJa6tHX3itqeaZFApi+TEmZmio5cfEOpBI3WidbAsghxWUaQz2N4eKlplynOcXFxot4TKbOOts9BLqI+8mFOWBYyBy6UyGdqagotLS3Izc3ldTTVaDSw26X3qszMzMBgMCAnJwdVVVU+ryGH+PBNQwjFUI6UWV+oyLHek1aW7CdAXCm3cON1e6DW+p63tvFpxOXJ2/PGNfKGpOjGxsbQ2dnpk5JKS0sTlaKLhrQbG/L94XsfbCttMgJoenqaTn0zRwAFW5e+kC20gRUWH5VKFfBunDkzbdu2bfTE20BIWZPxer3o6enB0NBQ0BEzUgWCoij09/ejv78/oI22HDbdXNMQwhH5iCWUBtP1H6uj/5/rTt/rcqPrVf+0klT41nv4iEuLR1xaPMbOyTPhIX1NBoz9s9yv9UH0s+bKpeGtmhi9oOinZHcNZ+pNDgKl6JgpqfT0dGRkZARN0UWj+JCbWiHfTZVKheTkZCQnJ6O0tBQej4ceATQ4OIi2tjZe91KC2Wy+YF1MgSiOfObn59HU1ISEhARRM9PEpt3sdjsMBgNcLpegSQJSIivi72OxWLBz504fwyYmUh1HnU4nmpubYbVaecfkSBVNp9OJgstr/B7Pv+UGODrlH06p0amxdt8un8e8AiPN9XtYx/nB++36m0GWYxNCfm2uIAFKX7N0IzXVPhW2Y3EuWqFP8i+pX5xYitBSyvKReXkdZg6fkP21A6XoBgcHg04ZiEbxIT0+UtJgGo0GGRkZ9Do10720s7MTLpeLXi9KS0tDQkKCpOkGq4moEx9mhVZ5eTnKyspE/WOLEYeZmRk0NzcjMzMT27ZtE1SpIvYiPj8/j8bGRiQnJwcdY8O0PhD6xSOmcklJSQH3LyXttri4iHPnzkH69KbAMMus1912NQDAawncCyOF9Tdu8fldqhjxTYlmI1SA+MjbEnyGH4l65CYcUw6CpejIlAFmSioaCw7knG7A5V5KxPnIkSP4j//4D1RWVkKlUuH8+fMoKysTvG8xFtovv/wynnjiCTQ1NcHhcGDDhg148MEHcc0119DbPPXUU/jc5z7n91ybzRbSul5Upd2cTidaW1uxsLAg2ZpAiPgwJzlXVVWhoKBAsMAJFTdmSbhQERXqu0MgA1SFrIWJTbuNj4+jtbV16aQXty4tGH2CHqX7Lg/PzgPAFKNwRUX5tbkwDQSvhMuuzuaNfgKl3qRS8bk96PnDq7LuUyxCUnSxsbFQqVRYWFhYMeM4NuGabsAW58rKShQUFOC3v/0tzp49i3Xr1qGkpAS7d+/Gz3/+84AN4mIttN977z1cffXV+MlPfoLU1FT84Q9/wEc/+lGcPHkSNTXLmYTk5GR0dXX5PDfU0WJRE/kQB9CkpCQ0NDRIbhINJg4OhwPNzc2w2WwBU2B8CIl8iBuo2MGjQsWHlGmLGZMjNPIh61/Dw8N0CbjrsKDDF81KCA+byn21AIDu1+RfCym7Yj3OH+oKvqFIKj66HeYR6ek6vuGicQ0fgu3Y0YiP2GFXjVmtVvT29tJRfSiDQOUkUnPd9Ho9PvzhD+Ott95CaWkpHn74YRw+fBjHjh0LOrPy5z//Oe6880584QtfALA0NPTNN9/EE088wWmhzXYi/clPfoJXXnkFr732mo/4qFQqn5YTOVhx8aEoCgMDA+jt7UVFRQVKSkpCussJJD7MgZ01NTWSGsKCiRuZiKDRaEQPHiWDFAPt3263o6mpCV6vV9SYHCGRj9PphMFggN1uR319vV++mfT45N9yg6DX5GPNZ5aq57yL80G3FbreEyrrPrr0Ret+rVF0sUGoMKMfISm3YHBVvAVjvmsAGRs2h/zachAfH4+kpCRotVpUVlZicXERs7OznCk6uRs7A7ESE62zsrKQmJiIG264ATfcEPh7F4qFNoEY9bGLu8xmM0pKSuDxeLB161b88Ic/9BEnKayo+LhcLjQ1NWFxcRGXXHIJPewvFLgKDpgCF+rAzkBNppOTk2hpaUFhYaFku+5AkZXRaERTUxOysrJQXV0t6osQLPIhd5lkbSocX+iYpFgU7PuI7PuVk3UfrYE2Pg5dLwtfhI9L478B4Ip+SLGBEISm3oRWvAFA7nVXYOLvhwQfw0pA1j2ZxnEkRTc3N8dZRUcaO8OVoou0l4/ZbMaaNWsEby/FQpvNf//3f8NiseDWW2+lH6usrMRTTz2FTZs2YWFhAb/85S+xa9cuGAwGVFRIX3tcUfHp6OiAWq3Grl27ZPOT0Wg0cDiWm/BIpdni4iJ27NiBlJSUkPfPvogzU1XsiQhi4RIfZhHG+vXrUVRUJPoLFijyISN+hKwdhUJEhSfEfqn1H6sTJUChkl0dusV7IAL1+oSr4i0U+AoOdDqdzyBQq9VKrxcJqaILhUhX4EmtdpNqd/3888/jwQcfxCuvvEKnQIElV9O6uuVWh127dqG2thaPPfYYHn30UdHHR1hR8dmwYQMAeb19mGkxudaRmLAjH4fDgaamJrhcLtTV1cli+sYUH+ZIoVCiQy5R83q96O7uxsjIiOC1I6EpN/ZonYLb9wPz0kbRrBRMARJa6caFnGs/FR8VZ1/BLrd2Dg9DX7Q8RsfW3Yu4dUsOpu42+eYKhorQCQdkob6goIBOGbG9euRK0ck9LT8YYvt8QrHQfvHFF3HnnXfiL3/5C3bv3h1wW7VajUsuuQQ9PT2Cj42LFRUfnU4X0igcLkjabXBwEN3d3Vi7di1KS0tlEzhm5GM0GmEwGGh/H7lN38iYHL1eH3CkkBDYaTen04mmpiY4nU7O9R3C/A/+BfEFS5Fc9o6NvPsPNNG64Pb9Eo965Yl0BMRFOKreAF9rhWjD6/WKzoYwU3RlZWU+Kbqenh7Y7faQUnQrseYTCQvt559/Hp///Ofx/PPPB11XApYiqaamJmzatEnwsXGx4gUHckPKM81ms+Ry7UCQ9FV/fz/6+vokp8H4IOJGxuSEsn7EPm4yk470HqWkpMjmlMrHahYewvqP1aH/H2dD2ke4Kt8uVORIcbFTdMxeGuLkybSLCJaii/Saj5TxOmIttJ9//nl89rOfxS9/+UvU1dXRUVNcXBy9RPH9738fdXV1qKiowMLCAh599FE0NTXh17/+dUjv74ISn4WFBfT29sLr9eJDH/pQWHwwyLrJ0NCQbEUSTFQqFUZGRjAzM8M5JkcqJKIi6ztSGnjFEinh8brkjZ65WHPttpAFCAhcbFB4TT3G/3lK0H4SC7MDllsHq3hjFx2wDeVWmnCsrzBnr0lJ0UUy8qEoChaLRbSLqVgL7f/5n/+B2+3Gl7/8ZXz5y1+mH7/99ttpx9O5uTncfffdmJiYQEpKCmpqavDee+9hx44dIb3HFW8ylQOmlTaxJgiH8CwuLqKxcakfZNu2bZLsbQPhdDphs9ngdDp5x+SEwvz8PKanp2W10OYjNo2VLhCw3hOO6QZysu4TH0b3C/8UtG1qqX8BQdkV6zE/NC3ptdPXZCBjQ+AudzEVb4HwulwYHRpCenr6ik1VDveEA3aKjthpB0rRRXrNR2zajSDGQvvQoUNB9/eLX/wCv/jFL0QfRzBWfeTDtNKura2FWq2GwSB/1zqZJlBaWoq+vj7ZU1Wk1FmlUqG8vFxW4XE6nRgdHYXT6eS0iZBCsLluGXtC6wWKVtgCFKjMejVCGk1NJhP6+/uh1WrpYaDp6emyVaUGI9KWCmw7ba4UnVarRWJiImw2W9BmTzlQplpHMWazGU1NTbRvTWxsLBYWFmS1ufZ6vejo6MDExAQdMZw/f17W12COyTEajbJ+6cj6jk6nQ2pqakjCoy0uke24FKIL765roH7/Tfr3LVu2wOv10gv2ZBJzUlISLUbJyclhi05WerAoV4qus7MTZrMZJ06cCHujq8fjgc1mU6ZaRyNk7aK4uBgVFRX0iSqX0yiwdPdD0mwNDQ303Y5cVtpMYaupqUFmZqasbqZE1MrLy6HVajE1JX4kC6l0E0PS7V8ATOOinycEOdJKoSIm/RYN8E23DlbpRszhSLe7w+Ggo4GWlhZ4vV6kpaXRUZGc0cBKiw8TkqIjgpObm+uXoktOTqZFWY5GV7N5qVVB7tR7NLHq1nw8Hg86OzsxMTFBzx5jotVqJXviMJmenkZzczNyc3NRVVXl80WQQ+DsdjsaGxtpN1PyxZXDUI45+42I2sjIiGyiphA+Acr78A7BRQdiYTeaaq++Ee6Df+PcVtP4d3hqrvN5LCYmBnl5ecjLy/MxS5ucnER3dzdiY2NpIQo1Gogm8SGQardAKTriYMpsdJUiylbr0vqnEvlECVarlR40yLxgMyELgh6PR9LJzzRlq66uRkFBAedrhHIhJ2NysrOzZXczJU2vbrfbZ/ZbpMzkkm7/QthfQy5UIuefsZGy3lP88esw9Je/h/S6QpEy4w0A4mtqEOwMZJulBVqwF2IexyYaLRX4BJHtYErsIiYmJmhRZtpFCLkuWSwWxMTERGxu3Uqwat7ZxMQEWltb/eyh2YQiPmSwps1mC1htJjXyYc6Yq6ysRBGj05y5b6niQ9Z3UlNT/Zpe5YioLga08cLvUouuqcPwmyvXgMpVbi214i2mzwBXCMfCjgasVqufeRyzcCHYtJFIFxwIQUi1m0ql4q2i6+3t9UnRpaenIzk5mfN9ms3mFas0jBRRn3bzer3o6urC6OiooLlpQiZDc0FG8aSkpKChoSGgcEmJfMiYnLm5uYD9QVJFYmRkBB0dHbwTHYJZlnNBvfyIqO1XU9Sz2ii47WOwN8lj+xBoxI5csM3j5ufn6bRUe3s7bSGdkZGBlJQUv5vJaEy7SenzkZqiu9Ar3QAguv51WdhsNpw8eRJGoxH19fWCBnaqVCpRbqYURWFoaAinT59GSUkJtm7dGjRiEhv5kAoZUuocqDFVrPh4vV60t7ejq6sLNTU1vI2j4Y58Eovk9fqINrSp3ANpS26/WZb9F15TL8t+QoXMd1MvGmXbJxn2WV5ejksuuQQf+tCHUFxcDKfTiba2Nhw5cgQGgwEjIyOwWq2gKCoqxUeOCQckRbdp0yZceuml2Lp1KxITEzExMYETJ07g2LFj+MpXvoLDhw/zRkWBePzxx1FWVobY2Fhs27YNR44cCbj94cOHsW3bNsTGxmLNmjX4zW9+47fNSy+9hOrqasTExKC6uhoHDhwQdUx8rHjaje+OnCz45+Tk+K2LBEOo+LjdbrS1tcFoNGLbtm1+Hhah7h9YtlkoKiryqcrjgzkGJxh86ztciLHRJoLM9D00Xf1ZZHXxu8qpPnyjoH0ThHj5SGaF04tcDaZMIrnuE42wLaTNZjOMRiOmp6fR09ODmJgYuFwuzM/PIyEhIWrWPeSecMBcNyMpuomJCdhsNvzpT3/C+Pg4Lr30Ulx99dX4yEc+gu3btwf8LMS6mJ4/fx7XX3897rrrLjzzzDN4//33cc899yArKws337x0U3X8+HHs378fP/zhD3HTTTfhwIEDuPXWW3H06FHs3LkzpPcfXbcWWJ603NTUhKqqKmzcuFH0P7gQcSDRiN1uR0NDg2DhIfsPdiGnKArd3d1obm7Gxo0bA65TMREaoczNzeHYsWOIjY1FXV1d0P4doQUHXq8Xra2t6OvrC7ptuImUkVykSNkpbCJ13oe5x5bEbhVn3pX2xeVxKc5F4dMjqLjIpXtUKhWSkpJQUlKCmpoaXHbZZVi3bh2ApVTykSNHcPbsWZw/fx4LCwsRKZrhI9yz3bRaLQoLC/H73/8e3/rWt7Bjxw7cfvvtaG1txfXXX4+BgYGAz2e6mFZVVeGRRx5BUVERnnjiCc7tf/Ob36C4uBiPPPIIqqqq8IUvfAGf//zn8V//9V/0No888giuvvpqPPDAA6isrMQDDzyAq666ys8BVQpRJT52ux2nT5/G1NQU6uvrkZ8vbeJuMPEhIW5WVhYuueQS0aN4gqXdnE4nzpw5g8nJScHpQua+g4nPyMgITp8+jbKyMmzevFmQOAuJfOx2O06dOgWz2YyGhgbBx3yxI1fqTU6I8CTd/ZWA22mvXo5YvbuuCesxCUGj0SAzMxMAsHXrVtTV1SE3N5duKD969ChaW1sxPj7u49sVbiKdCrRarUhPT8cXvvAF/PnPf8b09DTKy8t5tycuph/5iK9nViAX0+PHj/ttf8011+DMmTN09oVvG6HOqIFY8XiWpN1mZ2dhMBiQmZkZsj0Bn/gwixc2bdoU1OMi0P75LuTMidFSHEEDiQ+zKbW2thYZGRmi9hvornFubg6NjY3IyMjAhg0boNFowNw6UMrNtO/LSF8YFHwsFwrqXP8y/GiAGfEASwLkeP5JAL69PrrhbriK1kX8+IJBURRdah0bG+s3aWB2dpae5ZiQkOBTuBCu2WvkOxmp2W4Wi8WnxyfY60pxMZ2YmODc3u12Y2ZmBnl5ebzbCHVGDcSKiw/pqzl//jwqKytRWFgYcnkhl/jY7XY0NTXB4/EE9K8RAl/kQyrOQpkYzSc+5Pi9Xi9vj5OU/QLLkxAqKipQUlIi+rhXUngiMdFaCCW334zBp1+K+OtylVs79YnQO808z4h+yHnKjjK4LLVJ5VhHRwdcLhdSU1Ppcu74+HjZSpUjLT5ijeQIYl1MubZnPy7VGTUYKy4+BoMBc3Nz2LlzJ5KTk2XZJ1t8mFEVuauXc/+k4mxyclJ0RMKGSyS4ohKxcBV2kEhwbGyMnoSgEBnEFB0U3PaxkF8v5pN30tFPtMMnPmx0Oh1ycnKQk5NDWxAYjUbMzs6ir68POp2OjorS0tJCGopKvu+RTLuJuUGW4mKam5vLub1Wq6WvYXzbSM0aMVnxNZ+ysjLU19fLJjzAsjgQ07dz586hoqICmzZtkuXOhRn5kHLwxcVFNDQ0hCQ87H0DwPDwML2+E8rxs0WN5IhnZ2dRX18vWXhM+74cfKOLBDnXfviKDsSi9obSOroykJskMRd6lUqFxMREFBcXY+vWrbj00ktRWVkJrVaL8+fP4+jRozhz5gz6+/slzU8k6z2RavoU2+fDdDFlcvDgQd712/r6er/t33rrLWzfvp0War5t5FgTXvHIJzU1VdYJ0cCS+DidTjQ2NmJxcRE7duygXfnk2r/X66UjKq4xOVIhIkGiqampqZCjKWA58iGlrefOnUNycjJqamqCrksFK7Nmow/TUNHVQLAy63AT+y9fX9HXlwMiDKFc6DUaDTIyMujvjd1up1N0IyMjAJabOzMyMqLSxVRs2k2si+kXv/hF/OpXv8J9992Hu+66C8ePH8eTTz6J559/nt7n1772NVx22WV4+OGHsXfvXrzyyit4++23cfTo0ZDf44qLTzjweDyYmJhAWloa6uvrg47yEItKpcLi4iLOnTuHqqoqFBYWyrZvtVoNt9uNkydPgqIo1NfXyzItmHxxyJiisrIylJeXX9DjO1YLkWow9Xz+36D5359x/s3W3Qt1+Rafx+bHB5GSF3kbDTJaR85zMzY2Fvn5+cjPz6eHos7OztLz18jE6oyMDKSmpvrdSEbSxRSQJj5iXUzLysrwxhtv4Otf/zp+/etfIz8/H48++ijd4wMsTfN/4YUX8J3vfAff/e53UV5ejhdffDHkHh8gCsRH7ovfyMgIJiYmkJSUhNraWtn373a7MT4+DqvVip07d8oaUQFLuV6r1Yr8/HxZ1qfYtLa2YvPmzbLkbOUk2l1Mw8FKNpsyR+wYP/lvSH+eW5RcLlfEDOQI4S5pZjd3ulwuev5aV1cXnE4nUlJS6MKFhISEFYl8pBRFiXExBYDLL78c586dC7jPW265BbfccovoYwnGiouPXHg8HnR0dGBychJ5eXkA5Bc2s9lMu40mJSXJKjwURWF4eBjd3d3QarXYtGmTbMdP5soBwPbt25GWlhbS/oiLaaTXe+T08gl1ojUfKZdfivnDgUeahAshKTempYLP4+vWAn0GOHZdA8wtpU3TJ9rxdmc/kpKS6BSWHF41wYj0RGudTofs7GxkZ2eDoijYbDbMzs7CaDTSbq4JCQnwer1wOp2yZ1K4sFgsF7SXD3CBiI/VakVTUxNUKhUaGhowOTkJk8kk62uQdFVRURGSk5ODdhuLgbm+U1lZid7eXtm+4BaLBY2NjXQjrVAnU+rlR+CW+TOMdsRMtObDXLwJQPjEJ3ZrTUgDRs3X3YHEvz/l81j2TAfv9g0NDfSFeHh4GCqVih6CmZ6eLrpBWwgrOdFapVLRQ1GLiopoN9fh4WG4XC4cPXo0Im6uF8Ng0RUXn1BPsqmpKbS0tCAvLw+VlZVQq9WiZq8Fw+v1oqenB8PDw3Rj6tTUlGz7J6ZywNIX3eVyyTYAdGZmBgaDAQUFBaioqMDBgwdXdDyJgj+B1nuEGstx9frIRUxMDL1WwtXkmZiYSKenuKZTSyGahooSKwiHwwGPx4MNGzZwurkSMZJjfZaUjSuRT5RCURR6enowODiIDRs2+IzikUt8iL+P3W5HXV0dvQAol402MZXLyspCdXU1NBoN3G53yPtm+gYxDfHEDBcNRExlJSYqrxL+hPmLK4KSE6E9PomF2ZC73Va9aATzW8Ru8nQ6nfSFuLW1FV6v1ycqknohjibxIRAvH7abq9lsxuzsLKamptDT00Mbx5HCBamTWqxWq+AsxWplVYqPw+FAc3Mz7HY76uvr/apC5BCfQGNyQrXRJus7XV1dWL9+PYqKiugIkJRaS+0i9ng8aGtrw+zsrF+J+UVhKBeG98dnpyAGoUNFpeK+4TZZ9ze17nJkdwcur+eaTj07O0vbasfFxfnYagstnolG8eGqdiNrv0lJSbK7uSqRTwQQe4E1mUxoampCWloab49KqOIzPDyMzs5OXmO2UCIfj8eD9vZ2TE9Pcy7+ky+dFPGx2+04d+4cbTPOzsdLMZQzXf1ZUdsrAJrP3APPnx5f6cPwI95mhDVO2PT2rOEzovbNdyGenZ1FZ2cnXC6XX3qK7/yORgttIdVuQt1cyWfAV7jgdrvhcDgkjddZTay4+AiFoigMDg6ip6cn6AwyqeJDKuaCNXZKjXxsNhuampoALK3vcDW2kbsrsXd/JpMJjY2NyM7ORnV1NedzL4rIZ5WhTk6Fd2Euoq/JVXQgN8wLMUVRsFqtmJ2dpUff6PV6OipKS0vzuYmMRgttKX0+fG6uZAYkcXMlkSH5zprNS3P5lMgnCiClwiaTSVCpsBTxsdlsdBk1nzAw9y82NUbWdwKJA7Ac+YgRCRKpsVN4bKREPtGO5ZO+5cVqyv/fPe65RyJ0NKuPSEy2VqlUSEhIQEJCAoqLi+HxeDA3N4fZ2Vn09vbS6SlSzh3pnhohhHpMxM2VOLoy18va29vh8XiQmpqKtrY2ulUk1Go3k8mEr371q3j11VcBAHv27MFjjz3G66Tscrnwne98B2+88Qb6+/uRkpKC3bt34z//8z991tSvuOIKHD7sm5Ldv38/XnjhBVHHF/XiQ3prYmJiOFNJXIgVn9nZWTQ1NSE3NxdVVVVBTzJmdBLsboi4gnZ3d6OyshJFHzT28SFGfJgWC0KcWIVGPm63Gxrwp9xEFRtIhM9IznTr/T6/a73Be3+st90HAIh/9uehH9gFyui2m1BwVh575GCwR98w01OkhUGr1WJychLp6ekRb3Llwuv1ytrfw14vs1gsmJ2dxZ///Ge89957iIuLw7/+67/i2muvxYc//GFJPYWf+tSnMDIygn/84x8AgLvvvhuf+cxn8Nprr3Fub7Vace7cOXz3u9/Fli1bYDKZcO+992LPnj04c8Y3DXvXXXfhBz/4Af27lOKSFRefQJHD+Pg4WltbUVJSgoqKCsFRhtDIhKIonD9/Hn19faLG5DAFIpD4MBf/hTZ3krEiwUSCaaEt1GJByH5tNhvOnTuHaLOSG997P3QIrck0kiK00us+ctgqTK27HKG1IwuDnZ7q7u6GyWTCwMAA2tvbI97kykU4ozEyFDUxMRF//etfcfToUXzmM59BbGwsvvWtb6G3txd/+9vfcM01ws3+Ojo68I9//AMnTpygR+H87ne/Q319PV3oxCYlJcVviOhjjz2GHTt2YGhoyMeKOz4+XpRJJhcrLj6AfzrI6/Wis7MTY2Nj2LJlC7KzxQ1rJILg8Xh4Sx3dbjdaWlowPz8vevAoOQk9Hg/vXRlJ46nVatTX1wcdXMjefyCRWFhYwLlz55Camort27cLzkUHM5Qj60a5ubnApODDDSvje+/n/ZuQqIcL6233waPSIvmF/5Z6WFFrJCcG5ogdJn3FV6N86CDHM8IPMZBLTk7Ghg0b4HA4It7kykUkZ7t5vV4kJCTgkUcegUqlwuDgYNCsBpvjx48jJSXFZwZbXV0dUlJScOzYMU7x4WJ+fh4qlcovVffss8/imWeeQU5ODq677jp873vfE71GFRXiw4Q0XVIUhYaGBkm17uQkcbvdnOJDUnmxsbFoaGgQHU6T0ep8AiE2jce1f759k2hQimFdoP0y142Ki4tBdb4l6pjF4F2cD7qNOjYWo9eEd3zPwifuD0mA5GBwy8dQdOR/RT2HPeVg4pb/QKZtmHd7tdcFr9r3Jmnoxm9ireF5nmcssxLDRZkZC74mV+aivdxNrlxEch2KDBUlnwEZDCqGiYkJzpv27OxswS6kdrsd//Ef/4FPfepTPpY3t912G8rKypCbm4vW1lY88MADMBgMflFTMKJKfEhHfk5OTkgWBYHEYWJiAi0tLaJTeVyvwV5XYlbkCVnfCbRv9rFTFIXu7m4MDw9LigaB4IZyclg3yMH5j3wdepV8c9wCEQ0CpOALX6VnpJpcxRxTOAg0WufBBx/E97///YDPP336NADuJQ2hRVIulwuf+MQn4PV68fjjvunju+66i/7/jRs3oqKiAtu3b8e5c+dQW1sbdN+EqBGfvr4+9Pf3y2ZRwOU2SsbkyDHVmd3rw1zfueSSS3grSoTAFh+XywWDwQCbzeYzaUGO/TY1NcHhcKC+vt4nyjTWXse5j5Nlt6MEI5JeXwjnPxJ5PxpFgKILoRd6viZXpk2ClCZXLsiEg0gQyE7hX//1X/GJT3wi4PNLS0vR3NyMyUn/3Pn09HTQa5/L5cKtt96K8+fP45///GdQo8/a2lrodDr09PSsLvGhKArnzp3DwsKC7FbabvfSwBGHwwGDwQCn04n6+npZBvax3UzJ+o7Qirxg+yYiQYzfEhISUFdXF1LlDzMaZO+XmZ6cHegK6filoE6IR9+uf4n46xIWPnE/0l+Vt0BgpYsOpBDTZ0BR+coeg5T1Fa4mVxIViW1yleuYpGKxWHiXGzIzMwW5DtfX12N+fh6nTp3Cjh1LrrgnT57E/Px8QBdSIjw9PT149913BWVC2tra4HK56BJxoay4+KhUKhQVFSElJUXWkkoS+czNzaGpqQmpqamora2VPGuJb/9kfYc52DRUiPhMTU2hubkZxcXFIaUImfulKArT09MwGAyy7VcOVlJ4CMY998guQKHSW34D1va9HnS7mbiigOs+qwmv1xvytUCr1frYJJAm15mZGfT29iImJoa3yZWLlVjzCYWqqipce+21uOuuu/A///M/AJZKrW+88UafYoPKyko89NBDuOmmm+B2u3HLLbfg3Llz+Nvf/kabcgJAeno69Ho9+vr68Oyzz+L6669HZmYm2tvbcf/996Ompga7du0SdYwrLj7A0iKY3J33arUaU1NTGB0d5R2TE+r+JyYmMDExIbubqUqlwvj4OKamprBx40bRdxSBmJ6exuzsrN8w1kCIsdCWQlvdVxELR1hfQwgOlXxrBGIYvvTzoosO5CZ7piOivT6BkDrXkA+uJlcyg42ryTUhIcHv9SMd+cgxWufZZ5/FV7/6VXzkIx8BsNRk+qtf/cpnm66uLszPLxUAjYyM0A2pW7du9dnu3XffxRVXXAG9Xo933nkHv/zlL2E2m1FUVIQbbrgB3/ve90R/PlEhPnLfeXs8HjidToyNjQlqvpSyf7vdDrvdHvL6Dhu32w2bzQabzSZrGtLr9cJsNsPtdosuLQ8XbXVfXZHX9aj4T/vxvfcj7xV513+kDhXt/1B4o8HeLZ8UVPFmt9tFtQqESrgX9zUajU/6it3kSppgSeGCTqeLeOQjx9JAeno6nnnmmYDbMAuQSktLg05AKSoq8ptuIJWoEB85IesvXq8Xa9askV14rFYrXQq+du1aWYWHvW+5hMfhcKCxsREejwfFxcWShedk2e2yHA+wcsIjBDkFSPOZe4Du4J480YKjfIvfY8ePH0d8fDwdGYSzpBmI/FRrdpPr3NwcLUSkydXj8cBqtXJGRXJjsVhkv25FIxeU+JBS7by8PMTExMh+ArP3L+dJyFw7knPfxBoiLS0NcXFxEUsdBEKd5Ct+sZrgKbdQpxtIRYydwpKLaWSZuOU/wv4aH/rQh2A0GjE7O4u2tjZ4PB6f6cxyR0UraanAnDy9du1aOBwOzMzMYGFhAR0dHejq6hI0mToU5Eq7RTtRMb0v1AstRVHo6+tDY2MjKisrUV1dDa1WK5vbKBnDw96/HOtUpDfo3LlzWLduHW0qJ8e+x8fHcerUKRQXF2Pz5s2i9utRC7svSV8YFH1cLRvki6CCQamkneKBJitEA7FbayQ9L95mBACke5adT3XD3X7b9RVfvfx3nQ45OTmorq7Grl27UFtbi6SkJExMTOD48eM4efIkent7YTKZZDlvo8nPJyYmhi5NbmhowObNmxEXF4eRkREcPXoUp06dQl9fn2zvHbg4vHyACyDycbvdaG5uxuLios8aiVxupmSi9tzcnM9aSaiGcsDSl6y9vR1TU1M+s99CtT6gKAq9vb0YHBz0aUiVy8k0FI5X3I3EKCguEIIcJnKrHa8qsIGay+WCyWTCzMyMbFGR3AUHoUK+5xqNRnCTaygR4cXgYgqscvEhvSrx8fGor6/3CYHlEB+yBqPValFfX+/TvxNqdGK329HU1ASv1+tn4RCK+JCZdaRvinkHFWy2mxzoTeO8fztecXdYX1tuhi//AooO/36lD2PF+a8DsfjGTdxTxnU6nU9JM7vRU8paUTRFPsCyvxD7mII1ucbHx9NClJKSIjjlbTablcgnUki5yyFjckpLS7F27VpOt1GnU/o6AVnfyc/Px/r16/1OvFAin7m5OTQ2NiIjIwMbNmzwOymlig+ZSK3T6fzEGAgt8pGz2EAhejiy9h5c2hu4t2n7BmHfT76oiLlWlJaWRosRX2QQbeIjpNItUJNrR0eHqCZXuardop2oEB8xkJHrIyMjAWecSY18mDYL1dXVKCjgnl6s0WjgcrlE7390dBTt7e0B3ViliA9zIjVfs6scqcKSGGmjdVZb1ENo33Uvqt9/ZKUPg5Ng6z5CbBXyEkw+v5/RfQjbXUd9HqtJ6gQgfril1Kgo2sRHSo+P1CZXsq0S+UQZYsbkSBEfvvUdLsReyJkDPGtqagKOyFCr1fRoICGwJ1IH2q+UyGd6/eUIpdhstQpPJImGRtNwIiYqijYn01CPR2iT69jYGLKzs2E2m0Ne8xHrYgoAd9xxB55++mmfx3bu3IkTJ07QvzscDnzjG9/A888/D5vNhquuugqPP/64pCb7qBAfIWk3kqpKT08XNCZHrPhYLBY0NjZCr9cLslkQs+bjdDphMBg4B3hyIVQkxE6kvhBttJlwWWiH5XVEevmcXH8Xdnb9LkxHIwwuWwVCuG20uQgUFTmdTnR0dCArKysifUXBkHu6AV+T6x/+8Ae8+OKLUKvVeOihh7B//37s3r1bUs+PWBdTwrXXXos//OEP9O/s6+C9996L1157DS+88AIyMjJw//3348Ybb8TZs2dFf0bRc3vBA7GhPn36NMrKyrB582ZB89nEiM/09DSOHz+OjIwMbN++XVDtvtDIZ3FxEcePH4dGo0FdXZ2gOxoh4uNyuXD27FnMzs6ivr5e0ADAYPulKAoneuW9gB/P/ris+1sJ2nfdu6Kv31t+A+fjA2uFO1sKpTRhjP7/BVUkfEyXo6LS0lJs27YNGo0G+fn5cLlcaGtrw5EjR9Dc3IzR0VHYeezVw0m4IzHS4PrYY4+hu7sbKpUKOTk5+NGPfoTs7Gw/S4NgEBfT3//+96ivr0d9fT1+97vf4W9/+xu6ugIPDY6JiaGLKHJzc32Eb35+Hk8++ST++7//G7t370ZNTQ2eeeYZtLS04O233xb9vqMi8uGD2BTMzMyIHpMjpM+Hoij09/ejv79f1KwzQFjkQ4oiysrKUF5eLriwIphIBJpIHYhAkQ8p+0ZslaB9CeGtlNuQtELNoauBwS0fW+lDiEooikJmZiYSEhJkq6ALhUiuQXm9XlgsFjz00ENISUnB6Oio6NcOxcX00KFDyM7ORmpqKi6//HL8+Mc/ptfVz549C5fLRc+KA4D8/Hxs3LgRx44dE2XzDUSx+FitVjQ1NdE2BWJr5oNFPkwbbSkz1AJFPqTPZmBgQJJ3UCBhC2UiNZ+oOZ1OevyOmuNj7neWCn6NC5VoLjwIB0ULrRhO3rgir8282MtVQRcKkfTyMZuXCkTIejZfwVMgpLqYXnfddfj4xz+OkpISnD9/Ht/97nfx4Q9/GGfPnkVMTAwmJiag1+vpfkRCTk6OYHdUJlEhPuwL6PT0NJqbm0OyKQgkDmR9JyYmRpKNNsAvEMym17q6OklVK3xOpgMDA+jt7RUdpQXaL4mikpKSsHHjRiyM9oreLxdvpdwm+bmRcjFVWOJEzG7UOcSnTcIBicz5vvPh6CsKRqQnWickJHAed7hdTPfv30///8aNG7F9+3aUlJTg9ddfx8c+xh+lS20KjgrxAZZ7UKSmwdjwpd2IR05hYSHWrVsn+eTkEjemqHH12YjZN1MkvF4vnX4MZSI1O+3GjKLWrl17QRcjrDakTLQO1dMne6YDU5n+add22zo4utW4Yp1N8r6FQs57Id/LSEVFKzHRmutiHgkXUyZ5eXkoKSlBT08PACA3NxdOpxMmk8kn+pmamgpoUMdH1IiPy+XiHJMjFXbajbm+I4dHDjvyIU2pBQUFIYka4Cs+ZCI1RVGor68PKa1A9kuKOLq7u7Fhwwbk5eXB4/HIMnrHmZaHQ94P8/49Ub86RutwcbGk3rh6fSIFOQel3EmHKyqK5JrPSrqYspmdncXw8DB9rdy2bRt0Oh0OHjyIW2+9FcDS/MjW1lb89Kc/FbxfQlSIj9frxYkTJxAXF4eGhgZZHE01Gg0oioLX64XX6/UZOSOHVQERN2Y6LFBTqhiISDAnUm/cuDHk0J9El+3t7ZicnMT27duRkpJCC0809VYEQ+tdmdSc2DJrQjjKrdtn81GdMRZ8w1WEmMgnEHJGRZFe8wnVtkGKi6nZbMaDDz6Im2++GXl5eRgYGMC3vvUtZGZm4qabbgIApKSk4M4778T9999PN8h+4xvfwKZNm7B7927RxxkV4qNWq7F582YkJSXJNlCQnCwLCwtoaWlBbGxsSKkwNiTt1tLSgtnZWVkN2tRqNZxOJ06dOoXy8nKUlZXJ8rmQShqKolBXV4fY2Fg6OlSr1bK8RqCo50Kgtfzj2Nj3l5U+jAuWUCKfQAiNitLT05Gamuojfl6vV3BFaaislIupRqNBS0sL/vjHP2Jubg55eXm48sor8eKLL/qsW//iF7+AVqvFrbfeSjeZPvXUU5LEOSrEBwBSU1NlnbhMPozTp0+juLgY69atk/WEdrvdtMFUQ0ODz9DRUKAoCmNjY3A4HKitreUdHyQWi8VC1/jv2LEDKpWKFh6VSsX72bzaW42NxVZZjuFipE3lb87GR6SmHMTbjLDGpSPdMwWjJvj5VRnfB8NCBdxud9gvwhRFyXYjxEegqKi9vd0vKvJ4PGHx7eEiUNpNDGJdTOPi4vDmm28G3W9sbCwee+wxPPbYYyEfY9SIj5wQfx8AqKioQGlpqaz7N5lMOHfuHADgkksukS0kJ+Xfc3Nz9J2aHBCjuoyMDMzPz9PpN65JvQDgVuuggvjig5cXrkZ6YmSmDCiEF3aZdYzWiyNHjiAlJQUZGRnIzMxEfHy87CKxEunfYFGRWq1GUlISUlJS/KIiublYjOSAKBIfuU5iZqmzVquV3Y6WzFErLy9Hd3e3bBVizInUmzdvRlNTkyz7HRoaQldXF6qqqhAfH4/GxkbMz88jOTlZ0JfoYol6HKq4lT6EFYWv3PqMaT22py1FzHV1dZidncXs7CzOnz8PvV5PRwdpaWmy3ISRm6KVgisqamxspNdKw91XpIjPKsVsNqOxsRFxcXGor6/HsWPHZHMz9Xq96OzsxPj4OGpra5Gamoru7m5ZUoXsidRWqzXk/TKPd/v27UhNTYXdbkdqaiotcpmZmcjKypLlwvHywtXBN4oCPKrQT/m3cz+P3RO+6bFIW2i/kfY52faVl2DCuMW3cXDSloqcuDkAQKe1HMDSnK/8/HwUFhbC4/Fgbm4Os7Oz6O7uhtPpRGpqKn1Rlpo6irbCF51ORzu55uXliVorkoLValXEZ7UxOTmJlpYWn85/udxMHQ4Hmpqa4Ha70dDQgLi4ODriCXX/XBOpQ3UydblcaGpqgsPhQF1dHeLi4uh8/ZYtW0BRFIxGI2ZmZmivkfT0dGRlZeHCdxERzqJ79V8EhNgqBMPs0CIxxu1zrqvVajoCqKiogM1mw8zMDG0ZEBcXRwuRmItytIkPsFztJnatSEpURKrdLgZWvfgwR9ls2rQJubm59N/kEJ+FhQWcO3cOqamp2LZtG73gStZLpO4/0ERq4jgqpXPYYrHQ7q47d+70OUZSWKBSqeiegfXr18NsNmN6ehpf+PdpPP+Q+Avuaol6VjO95Tdgbd/rEX/ddelT6DYurT0eG0jBZWstdK8Ysf1QqVSIiYlBYWEhiouL4Xa76YtyR0cH3G634IsyKTiIJvgmHIRSQceHxWKRba032oka8ZGS5yWNqRaLhXOUTajiQxqo1qxZgzVr1nC6pUqJUJiRCZfFAtNUS0w6jBQWkEZXr9cLj8fDW1gALOe4b/1SH37ww60A5Bmvo+DPSlsrBLJVAEBPNyhNGMOAhXu6COnBI/1z5IcdFWVkZCArKwsURcFisWBmZoa+KCckJNBCxF57XOk1Hy6ERGNyRUXKms8qgMwki4+PR319PWdjaihupt3d3RgeHg7oliol8hEykZqc6GKa20j6rrKyks7Jky9NsC/zdZ9u8vmdVLoJKbMONeqJ1YRv4gGlCt8dNNe6z8WEWq32uUmiKIpuumZHRXFxcSguLqYvykajEbOzs2hpaQFFUbS1dEZGRlSn3cTAFxVNTk4GjIqUNZ8oZ3JyEs3NzSgtLcXatWt5L65SxMflcsFgMMBqtaKuri7giSA28hE6kZr5pQ4GRVHo7Oyk03fp6ekhCU8ohKvMWqfYMkSEwwNrcHlpv+jnkfOVXKBJJEQmjLCjoqysLOTk5ICiKCwuLmJmZgYjIyPo6OigI4KFhQVZm85DQQ4n02BR0ejoKCYmJuBwOEJe85HiYsr3Of/0pz/Fv/3bvwEArrjiChw+fNjn7/v378cLL7wg6TijRnyEnGRirQrEig8zKuGLppgIjXzETqQmn0Uw8XG73bRQ7ty5E/Hx8SsmPBcLExbxU8q5uJC9fPiiImaajmyXkJCAxMRErFmzBk6nE729vTAajWhqaoJKpaIjovT0dFnGbklB7qnWXFHR8PAwDhw4gJaWFvT396O9vR3XXXcdPvShD4lucJXiYjo+Pu7z+9///nfceeeduPnmm30ev+uuu/CDH/yA/j0uTnqLQtSITzCY6zv19fWCQlMx4kOmXYvxyRES+UiZSE3WaALt22q14ty5c4iNjcXOnTt93msw4ZFLdA6NRN5++UJn+NLPw0WtzEW2KCN4CvQj+0/7/P7Wi5cE3J4vKiKixByno9FokJiYCLfbvWTvsbCAmZkZDAwMoL29HcnJyXSDa6jzz4RCjjFcqUASFX3605/GbbfdhpqaGnziE5/AxMQEPv3pT+Oaa67xsbUOBnExPXHiBG0m97vf/Q719fXo6uriNZJjFmoBwCuvvIIrr7wSa9as8Xk8Pj7eb1uprArxWVxcRGNjo+CIhCBEfEKZdh0s8gllInUg8SETFvLz87Fu3TqfL3GwL0kg4anQKsUGYojGdZ9QbRWYvN+fi11rlk3CYrReXPXxBrzzl2P0Y0wxCiZEQOCoyOPxwOFw0OdzcnIyUlNTsXbtWtjtdrrBdXBwEFqt1qfBNVxjf8j3KlKDRW02G6699lpceumldFpSDKG4mBImJyfx+uuv4+mnn/b727PPPotnnnkGOTk5uO666/C9731PkmcZEEXiw3cXQ6yog63vcKHRaOBw8N/Nud1utLa2Ym5uTtK060CRT6gTqfnEh+TG169fj6KiItnTbO9MbsYVee2ijpWPpBhlreZCIyWF/5IRalQ0OjqK0dFRbNiwwa9oQafTIS8vDwUFBfB6vXSDa19fH2w2G1JTU5GZmRlSgysXzGxCJGBWu6lUKtHXJKkupkyefvppJCUl+RnI3XbbbSgrK0Nubi5aW1vxwAMPwGAw4ODBg6KOkRA14sOGoij09PRgaGhIkhU1EDjysVqtaGxshFarlexmyhf5kBLtUCZSs8WHVOCNjIzQhQXkyylEeK697Rz9/yqeKrB3JjfT//9qb3XA/f1f81pkyju5KCTU1MU5Uy5StgqNPVrUVLhFPUdMVDQyMoLu7m5s3bqVrnoLVMqdmpqK9PR0VFRUwGq10lFRX18fYmJifBpcQ4la5LJ4EAIpS+daUgi3iymT//3f/8Vtt93ml6m566676P/fuHEjKioqsH37dpw7dw61tbWC9s0kKsWHVJzZbLagFWeB4BMf0g8Tik032T9bIHp7ezE4OBiwRFsITPEh8+rMZrNPYYHQCcBM4Vk6Tt+Iik+M2JTEjIh4BxcHYlNvK93rEwy+ireK4tD2GygqGhwcRH9/P2pqamiHTK70HBEirgbXgoICOhNAKsk6OzvhcrmQlpaGzMxMpKeni14gJ5VukVhfcjgc8Hg8nGmsSLmYHjlyBF1dXXjxxReDbltbWwudToeenp7VLT7kH3dxcRHnzp1DUlIS6urqQqpw4XIzJQ6elZWVKCoqCumYmZEPmUhNDOuk5kGZ+/Z6vfTAUb1ej7q6OlGFBYC/8HDBFqNg/F/zWlHbK8iL1LlufxjdjduLlkplhdgqrM2z4/3+XGSlLp0fjT1a6HTwW/eRAlOMvvUlC2pra3mLccSWcqenpyMzMxPr1q2DxWLh7K8R6mYqd6VbICwWCwBwllpHysX0ySefxLZt27BlS3ArkLa2NrhcLsmu0FEjPsByuqqsrAzl5eUh320wL9RkKu3U1BS2b9/u40Eeyv6ZAqHT6WQzrFOr1VhcXERraytycnJQWVkpqrAAECY8BGb0EyzlpuBLpIeKhsJTQ5fjjuLDwTcMwMyURaajWeInTyQA6KZ/F7JWxNfgyi7ljouLQ1FREUpKSuB2u+kGV+Jmymxw5fLkimTTq9lshkqlCmnNSoqLKWFhYQF/+ctf8N///d9+++3r68Ozzz6L66+/HpmZmWhvb8f999+Pmpoa7Nq1S9KxRo34uFwu9PT0hJyuYkLEx263o6mpCV6vFw0NDbKNQVer1bBYLDh+/DhycnJQVVUl24nqdrvR29uLyspKFBcX018qoSkAqcKjcGHyh9Flm+Onhi7HretbfP7ONdl6pRCzVhQsKmKWcpOZhqS/ZnFxEbOzsxgbG0NXVxcSExN9xv4Qw8VIRj6JiYkh33SLdTElvPDCC6AoCp/85Cf99qnX6/HOO+/gl7/8JcxmM4qKinDDDTfge9/7nuTPJ2rER6fT4dJLL5U1t6rRaOB0OnH8+HFkZGRgw4YNsp5IZGZVVVUVPZE6VEihhc1mQ3FxMYqLi0VVtAHANZ88S/+/Si2keVdY2m3QUYjTXfL6l7DRqy78CjmpDaZSh4s+0X8VYj+4qVepgFAtqMorUtB6TAePyxXajgQgpYJObINrWVkZnE4nHRUZDAaoVCqkp6dDr9dHbMoCcTEN9fXEupgS7r77btx9992c2xcVFflNNwiVqBEfYHmas1wYjUY4HA6sX78epaWlsp1EZCL17Ows0tLSZBMesm60uLiI1NRU0RMLAF/hAQDKSwkSoB/+qBZDs5IP/aLmhPMS1OlP+z0uxkI70vy5a5Nf9EM4PLAGJVn2gM/f2FCNlvdb6d+97shUG8oRFfE1uGZnZyM3Nxder5ce+zM5OQm73Y6zZ8/SUZEc0QkXFovlorFTAKJMfFQqlSziQ8RhdHQUarUaZWVlMhzdEsyJ1KWlpfQiYaiQdSOtVoudO3eira0N8/PzyMzMRExMjCThAYRFPpEiUR++IaIKkScmPg4Oq+2DG5zl1C0lg8GiEOSMithFC8Q2Oy4uDuPj48jLy6MbXDUajc/YH7kaXImXTzTMs4sEUSU+cuB0OmEwGOBwOFBTU4MzZ85I8sXhgj2RemxsDAsLCyHvd25uDo2NjcjKykJVVRUoikJubi4GBgZw9OhRpKSkICsra8nsjefOKBLCE+6Um1C03uhKzZ0f16CuRL79vd6UjRu2Tsm3wxA52+rCto1LVacuFwWdbum8Ss1Ox/TwBKABPK7lHiAVa91zJcRITFREIiBmVERKuV0uF7RaLXJzc5Gfn+/T4Nrf34+2tjY/B1ep15qLaaI1cIGJD7tMmzniPVTx4ZpILYdZHanwq6ioQHFxMZ0OyM7ORk5ODux2O2ZmZjA9PY2+vj7ExsYiOzsbWVlZSElJgUql4hQeYCnlRggkRCqVGkOzSxU2C4xALpidgoI0JuwZyI2VluPsLb8BMArf/snOnbx/e6ppA+7Y2ubzWFGGA8OzvlVfxYUxALgFJC5x6WbIZl4+cZhCBCDqoyIiRMyoyOv1wul0YmpqCklJST4N3cwGV5vNRje49vf3Q6/X+4z9EbPGbDabZZ3OEO1ElfiEIhCBbBbcbrfk8udAE6mlmsmR/ZKG1K1btyIzM5NzfSc2NhaFhYUoLCyE2+3G7Owspqen0dTUBICUqfrfbfq9ntc3nUnESKl0Axwq6ZN5Vzt8PT6B6O9bgHFqAdkFS9VxxvFpaLQauJ0fXKDJGgvHjdlKCBEgPiryeDxobW2FXq9HRUUFAHB6Fen1euTn59MeWqTBtbu7G06n08dALliD68VkJAdEmfhIIZDNAnOhUQrBJlJLtdH2eDxoaWnB/Pw8du7ciYSEBEGFBVqtFjk5OcjJyfG7s2N+kYMJ0dL2S4UIQivdXj+iwkXi7gsAWHSLuwg8P1iHT5acCNPRRJ73m7XYtXk5gjnb6kJyMnfDtz42Fk67HVr90uWELUIU5fW7+QGiIz0H+IuRy+VCY2MjdDodNm/e7BO9CG1wpSiKHvszPT2Nnp4exMXF+Yz9YbdlKOKziiBjZxYXFzlttIk1AblTEYOQidRSIh+73Y5z585Bo9HQExzETCwA/L88bNhfYi4xiqZChEjhUUXH6f5esx637uD+218bCyJ2HPFxarx6Ngt7tk2Lfm5GbgpmJ5b6RBLTkmA2LT3OFCGXY3ltjnm+cQnR0jYrHxW9/sxWeqLIli1b/ARCTCk3yVoUFxfD7XbDZDJhZmaGNpBjN7hebNVuUZVzEZN2s1gsOHHiBDweD+rr63nH2UhZl5mfn8fx48cRFxeHHTt28Dalio18yH6TkpKwfft2n2MjTXDBCCY8bIQIz747xHcoZ6ZH1alz0dBtDB5+zsQtjY0KtN4jlLV5gUuu7Vbfv+tjY+F2uuF2uqFSqSWndVVqtc9PpLjh00347s/V+Pf/dONanrVUglqthkajgV6vR2xsLGJiYqDVaumWEY/HA7fbDZfLRRvjVVVVYdeuXaitrUVSUhLGx8fxhz/8ATU1NWhtbcXCwoKkm2UmP/7xj9HQ0ID4+PiA7qVMKIrCgw8+iPz8fMTFxeGKK65AW5vveqDD4cBXvvIV2k9pz549GBmRPu8xOm4FRTIzMwODwYD8/HysX78+4FQBseIjZiK1mMiHWEOsXbsWJSUloicWAPIID8B/58nH60eiJ0p64UwpNBr/49F88FY/sVEeO4jVRDgmWxfmaQH4XgQXFpZSb0XFSRgeWvaZsVvtSExLwszoUoWeWqvx6fshAsSXfgtEJCMgLsSuFQkt5U5ISEBCQgJKS0tRWloKrVaLP/3pTzhx4gSys7NxzTXXYN++fdi/f7/oY3Y6nfj4xz+O+vp6PPnkk4Ke89Of/hQ///nP8dRTT2HdunX40Y9+hKuvvhpdXV30jf29996L1157DS+88AIyMjJw//3348Ybb8TZs2clNe+vKvFhLv5XV1ejoCB4ikKo+EiZSC0k8mGa1W3ZsgVZWVmiG0cB8cIDLH1xg901qgWeNNnZ/nOvmKQnhqfJ8MUTedBqhYvfC63Lc+n2b+qU9VjkstBeCewOCrEx3J8jX+qNve7DRWJKAszzFszPzEEXo6dTbWrtB+utH4gQWVskUbcQEVpp4WEjpYKOr8GVWbSQmpqKz3/+83j33Xdx66234sorr8Qbb7yB48ePSxIfYr3w1FNPCdqeoig88sgj+Pa3v017+Dz99NPIycnBc889h3/5l3/B/Pw8nnzySfzpT3/C7t1Lo5qeeeYZFBUV4e2338Y111wj+jhXjfh4PB60tbVhdnZWsB01IEx8pE6kJpEPXyk3qZgxmUyoq6sTXFjARorwEMgXmEuEhApPJHFSehw4mUH/LkZ4mHgp4PnmSgDAJzfLK0J8RKLo4JTBiR1bglduktQbF2JG7PSO+6ace7qMqFi/ZOTEXPeJiY+Fw2qHLmbp2Ngi5HGx1yGDi9BKrQEJhXwvpTi4cnkVnT9/Htu3b8fOnTt9nEjDzfnz5zExMUHPggOAmJgYXH755Th27Bj+5V/+BWfPnoXL5fLZJj8/Hxs3bsSxY8dWv/jwXYzJIr1arRZtRx1MfEKZSM08mdhhp91uR2NjI1QqFerq6qDX60UXFgChCQ+Bb92HXel27OQcNm5M9XksUim3v7y/VLar1cqf3ycidOuW6LEJ//OpPNy6Y3ylD0MwxYUxGBpZmlBx9vgQXWYNLEc/RIAAQBejh8Nio7dhpt4A8anfaBUiIcLDhis99/zzz9PDTSMNcThl+/3k5ORgcHCQ3kav1/u5AeTk5Ah2SGUT9avGJpMJx44dQ3JycsDFfz4CiY/JZMLx48eRmpqK7du3i+4F4ivlXlhYwIkTJ5CQkIDt27f7VLQJLSwAwis8QmgdCn/D21/eT6OFJ9w8d7Y8Iq8TDbzSmB98IwZ/+id3JqG5ec7vseT0REyNmnj35bTZ4bTZOc+zUIoQ6H2sQCECF1KEh41arcaBAwdw33334ZVXXsF9993Hud2DDz5IXzv4fs6cORPSsbCvS0Ka80Np4I+qyIfN8PAwOjs7sW7dOhQXF0t6k3ziQ/a9fv16yYNByd2Lx+OhTe9Is+uaNWtQVlZGh9Wk7FsIcogOgZ12Y18QKmrX+z2HMDho4/1bqLz0Xiz0enkuHhoRuyEC9KltfbK8driIlhE7OXn8d+IDHaMAgKS0RDjtDuhjYxATHwuPywXPB2s97PQae+2H+TexrGQEJIfwAMCrr76KL33pS3j++edx3XXX8W4n1MlUCrm5uQCWohumMdzU1BQdDeXm5sLpdMJkMvlEP1NTU4JM6riIKvEh4uL1etHZ2Ynx8XHU1tYiIyMjyDP5YYsPGTo6NjYW8r5J+oyEzufPn0dfXx82b96M7OzsiK/vBIIUH7C/6Bs2ZyIrLXLVbC+9Fx3z4Z47Wx71AhSN5OQlYXJ8qdItPjke1gUrjJNL836cdgfcH6z1aLQaWoCAJbHhmngglQtBeN544w3ceeed+OMf/4g9e/YE3Faok6kUysrKkJubi4MHD6KmpgbAUsXc4cOH8fDDDwMAtm3bBp1Oh4MHD+LWW28FsFwZ/NOf/lTS60Zd2s3pdOL06dMwmUyor68PSRwAX/FxuVw4e/YsZmdnZdk3sCRALpcLLS0tGBoaws6dO6NOeADu9JtG53vvwV7vCYWkGN/hn68e9kSN8BDCkYZ7frBO9n2Gk/i4pfMiJUWPt08uXdCLMnynj/Ol3gg6RrpaG7P8/xrt8pgdPuGRGvWsVNpNLuE5ePAg7rjjDvz+97/HzTffLMs+CUNDQ2hqasLQ0BA8Hg+amprQ1NQEs9lMb1NZWYkDBw4AWLrpv/fee/GTn/wEBw4cQGtrK+644w7Ex8fjU5/6FAAgJSUFd955J+6//3688847aGxsxKc//Wls2rSJrn4TS1RFPjabDceOHUNqaiq2bdsmy6hyIj7sidRyjUFXq9VobW2FWq32KSygKGpVCQ8bOVNurx4WdscbjmKDYPzhWBk+tUvagunFBCk2YEOiH51eD5dz6YZDG6OnIyDKS0GlUnOOcZIqPEzIuR2JSEgu4Tl06BBuu+02PP7440FTaVL4f//v/+Hpp5+mfyfRzLvvvosrrrgCgL+T6Te/+U3YbDbcc889MJlM2LlzJ9566y2fyt9f/OIX0Gq1uPXWW2Gz2XDVVVfhqaeeWv1OpgAQFxeH9evXIzc3VzZPC41Gg8XFRZw4cQJFRUVYt26dbPteXFyEy+VCUlISamtradtdIPKFBYFgfzE1Op3fF5+r0i0UFh16vHtC/jUjrgZThfBQmKfFyPhSP8qc0YbUdOEDWLUxetgXl6eih1rtFojVJDxHjx7F/v378cgjj+Azn/lMWLx7nnrqqaA9PmzfNJVKhQcffBAPPvgg73NiY2Px2GOP4bHHHpPhKKMs7aZSqZCXlyfbPwhFUVhYWIDRaER1dTXWr18v276npqZw4sQJ6HQ6eqGPOdcpWoSHjUa3PBxybU1F2F6HLTwxMVF1n+PDc+/nSn7u+fHo65X6/VvpmJ4OPBaHj7dPev0sFQgDPTOcj8cnL1VFkvSbw2KDw2LjrXaTk9UkPCdOnMDHP/5x/Od//ifuvPPOi8Y0jo+oEh8gNFsFJl6vl27wTElJ8bFCCAVSWGAwGLBp0ybExsbC7XbTJYdiKtpWUniYHDs55/P7qePS01BvHlrEm4cWg28YZYQiQFyEw0L7mXdTZd+nEJgVbwM9M5gdXyqzZq77EHSsdgWVWuVb2caodmP/TQrhLrt+84Xtsuzn7Nmz+NjHPoYf/OAHuOeeey564QGiUHzkwOFw4NSpUzCbzSgvLxcsCMEggjYwMIAdO3YgJycHOp0Ovb296O/vh9lsFmQDHmnRIZCF30ALwKGwGkUnHAgtOvjzqbzgG8mI3cEwFwxw7Tt1mr+Hhw+n3QHbohm2RTM0Wg1dbEC/ngj7DqnILUAPfNGM9957D21tbZiampI88NNgMGDv3r349re/ja9+9auK8HxA9OZCJDI/P4/GxkakpaVh48aNmJ6eDtltFFiqwmtsbITH40FdXR1iYmLg8XiwceNGGI1GTE9P4/Tp09DpdMjOzkZ2djZSU1P9TrSVEB6uL6U+bim10tM97/O41Kgn2oXH7fa/KXA4fC+Gz72fu6LFB0LtFISO2JGb1IwEzM0uOZbOjpvoaQbuD5xLNTodPC4XvT0ptw50oxOtaz9vvXgJbZlN/HhsNhvS0tKQlZWFzMxMQa6jra2t+OhHP4r77rsP3/jGNxThYRB14qNSqQRFD1yQydHMidRyWF0Te+7k5GRs2rTJp7BAr9cjLy8PeXl58Hg8tBAZDAYAQFZWFrKyspCRkYHrPnUupOOQAp/wlG4oE7yPYENFjxyZRmwcd0qPD7kaTOXmufdz8dGd5uAbSuS95siLhhROnTZhxyXckyfik+JhXbTS43S0Oi2vAAUSl2gWHmDZHC49PR3r16+HxWKhLe27u7sRHx9P999wmcN1dHTgox/9KO655x58+9vfVoSHRdSJjxSYbqbsidRS3UYJREhKSkpQXl7uZxjFRKPR0GJTVVVF3zV1d3fjB48FvoCHC/aXMiZhqWJpqHMIxZW+kx2ERj3Ey+fIkaVJyGKFRyihDBW92JDDViElRY/5eSdS0mIxb7LTlW5C4BIg4mjKLDKI9mq3QMUFxAahpKSEtrSfmZlBS0sLvF4vMjIyMDU1herqalgsFtx4442444476NE4Cr6sevFhTqTmcjPVarWSxIeiKAwODqKnpwcbNmygIxuhjaMqlQppaWnY/8VeACsjPGyI8CRnpiE9J9Xnb9bF5eqonPzAE8NnjF50tM3KfnzRwp/fjcWtV0qrFpOTlR6x090+hXXVga1FmMNECU4b/2dHl1xDnjXHSAkPG6alPamqnZmZwaOPPop3330X6enpWLduHfbv368IDw9Rl/sQ8w9ls9lw8uRJuFwuXjdTKWk3r9eLtrY2nD9/Hpdccolo4SGsVGEBFzEJcbTDpNm0ENK+ggmPPnbV39OEzMtvh8ffKBi/fytd9HPIlAM2iclx6G73Fb/UjCWb5/gk//UOrU7rIzxc1WwU5QVFeWWpdFt6DXmq3UIpp1apVEhJSUF5eTl+9atfoaKiAjU1NcjOzsZll12G4uJivPXWWyEd34VI1ImPUIROpCbiI3Qdyel04syZM5ifn0ddXR2Sk5NXvfBodDo6BUIKDQBAHys+XXbulLyOmaEiZqioGP78bnSNApKC1F4fNk0nBwIOGI2JX/6suISACA1XtZscAsR8bSnI1cczNjaG66+/Hpdeein+8Y9/4MCBA5idncX//u//orq6OvgOLjJW5S2qmInUTNuDYGMgyAiepKQk1NTU+KwXrWbhYcNe6xFKtAmPQmSIT4pH08kB5BXzD7Z02hw+GYalIbZLYhPN1W5yCc/ExAQtPL/5zW/o9eCYmBhcffXVsrzGhUbURT6BLvBerxcdHR3o7u5GbW2tICsEIjjBUm8zMzM4ceIEcnNzsWXLFnpaNXDhCA8z6pkaWe5WH+ufDLifqSlHSMIjZLrBSsx1CwYz+gmXhXake31CYXxoBoNdY5ifmcfM2DSsixYszs5jcXapXJ99cxfJ4Z8rKTxTU1O48cYbsX37djz55JOSZ51dbKyayMflcqGpqQkOhwP19fWCauwBX88dPgYHB9Hd3Y3q6mrk5+fTFW1iRAeILuFRqdU+d5yxiUufF7E3Joz1TyI1i7/AoLd9FLGMtEpcQnQUTyj40z6bD0CeVBupeEtMjoN5gX9Gn0avhce5XBXHXGP1Ek8f1lw34MKJeGZnZ7Fnzx5UV1fj6aeflm1g8cVA9N1ucmA2m3H8+HF6crRQ4QEQsNfH6/Wivb0dfX192L59O/Lz8+HxeODxeERHO9EmPFy4HE4kZaT6RD2BYAvPSrMSQ0VDWftZqaKDQAidcsCGFBjo45bWVpljdDR63wuuRqOhhYcJcTG9UITHZDJh7969KCsrw3PPPUcbSioII+rEh33Bn56exokTJ5CTk4Pa2lpJdxZc4kO8fYxGI+rr65GSkrLqCwsAfvsEZsRjW7BArVYHTLf1to8Ker1w9fhEE9FQfMCe63bK4PTb5tjZ0KOelBThTbBcAuRyOP2iayZyVroB4lN7cgnP/Pw8brrpJuTk5ODPf/4zb8GTAj9RGyNSFIWBgQH09vZiw4YNIQ0GZYuPxWLB2bNnkZCQgJ07d/r8/UIUHkJSRipsC5aA+1icM2NxLnwd/hcDvb0r8/lNjMwhtzA17K+jj9PDafMXGDdjsgEX7Go3Oay0l/cVOV+fxcVF3HzzzUhOTsbLL7+MmBglFS2FqBQf0mczMzODHTt2ICUlcNNjMJjiMjs7i6amJhQWFqKiogJerzcqPXjEIsQwjghPQury4nmg9Z5wEenROkLmuikEh4zVYaLT62E2Lc8HJILCFJNAA0UjmYKTI+qxWCz4+Mc/Dr1ej1deeQVxccI9jhR8ibq0m91upydSk3RYqBDxGRoawrlz57B+/XqsW7cOFEXB6/XSVgirVXiApS8e80ej0/pMsNYGyUfPjM1gZkzYWlAgLtQG09fe8m/MjUYvH6nwNZoCS82mbMjaDwC/CdYAQ4QuIOGx2WzYv38/vF4vXnvtNSQkJIS8z4uZqLtSeL1eJCUlobKyUraSRbVajeHhYSwsLGDbtm1IS0u7INZ3+FBrNT6VbvrY5TULEvVYF5ZSQ8xUSQyruGAliw2kznULFZtV2th8Pl5+24OP7ZZ+Hoc6Ymd62o6sLGn/jqTiLRC2xaXziAiQh1FocCEJj8PhwG233Qar1Yo333yTc5qKgjiiTnwSEhKwYcMG2fbncrlgNpuhVqtRX1+P2NjYC154mDCFB1gWHcC370cIq6HMOpxDRV97awEf/Uiy7Pv986k86HUrm4Q4+s55eD8Qi4qN/NYOJPW2MD0HYKmCjSky0WajIIfwOJ1OfPazn8X09DTefvttWbIxClEoPnJitVpx9uxZqFQqFBYW0sIDrO7CAj6WusoZpbRqlU9ko4tRKnJWkmi0Uzj6znm/x3paR+FxLX1PmJVrfL1ybAGivJTfY8y/yUUkhMflcuFzn/schoaG8M477yAtjdtmQkE8UbfmI9cEWKPRiOPHjyMzMxMZGRmgKOqCKCzgg6vgQB0gbcmOetgpNyEIKbMWMt1gNcG19hMNvPyKeBO8I2/7Co9a5X8OMW9YmGlwdm8PaSR1MxpOSV8PYbUJj9vtxt13343u7m4cPHgQmZn844UUxBOVV4ZQDOUAYGRkBB0dHaisrERhYSE6OzsxMTEBjUaDnJwcwRUqq1l42JVuYqMeses9I31LPUNq1qRPrnU7do+HSq3CxtpCUa8HhG+oKJv5Of4O/0jwelM2AP7eGSZylVtrdBo6+uHdhjHdIFBvD18UFC6+f68bvb29yMzMREpKiqQbWo/Hg3vuuQdNTU04dOiQj0eYgjxEpfhIhaIodHV1YXR0FLW1tUhPT4fH40FJSQni4uIwNTWF3t5eJCYmIicnB9nZ2bwVK6tBePia64QIj8Piu5Dscvj2aNgWfEtqNTotFlhOCuwqJ7bwiKH13Ijvvjn2tXWHeIGKBkItOpAbu4PC6SMDgrfXxehpcWH3zGn0WthZ5ddsiPBwlWFLhXnuM6Og15/ZipmZGczMzKCxsRFqtRqZmZnIyspCenq6oCZ1r9eLr371qzhx4gTeffdd5OWtnvl7q4kLRnzcbjcMBgOsVivq6uoQFxdHFxbo9XoUFRWhqKgILpcL09PTmJqaQn9/P+Li4pCdnY3s7GwkJSXhmk+cWem3Igg+4VGpVT6LvWqNxu+ulF12zS5SCCehdLY3nRrxe2xbXeQFKVyFB3yYTMKiHqEc/2cvtDrur75apaYLD/iiH6YAOSy2JbsEHkHhs1GQMwVHJmiTVBuxtfd6vbSbcE9PD+x2O9LS0mi34dhY/+je6/Xi/vvvx6FDh/Duu++iqKhItuNU8OWCEB+r1Ypz584hJiYGO3fu9HEvZRcW6HQ65OfnIz8/n7bCnZycxJkzZ/Cf/7M6yicDCU8wgvX7ANypMr9tBAiW1FJ5rqiHj7MnlgUpkkIkd4/P2eND2FbPPaW9u2UE6zb5v7dTBid2bIlMEQMz+gGWRIcJW1AiVWZN4FrjUavVSE9PR3p6OtavXw+LxYLp6WlMTEygq6sLiYmJyMzMREJCArKzs6FSqfDAAw/gjTfewKFDh1BaWir7cSosE5XiI2bNx2Qy4dy5c8jLy8P69esBQHBhAdMKdzWk2QBxwhOo4IDeRoiI8NwlRxtEiGp2hl+EWltM2LjJv/KJb7RONFa6iYUpQPHJiT5l+8DyORiozDocvPn8NkHbJSQkICEhAaWlpXC5XJiZmcH09DQeeughvPnmm6isrERnZyfeffddlJeXh/moFaKu2k0Mo6OjOHPmDCoqKlBVVeVT0XahllL7P8Y9pJFMD2b/BENqtCJ1vUdOJ0tC48kR+kfB19H0+D97AQBuF38zLbPqTaPjPh9sZivnUM9g/T1yRz1ChYeNTqdDXl4eNm/ejF//+te4+eabMTY2hsTERNTV1eHGG2/Eyy+/LOuxiuWhhx6CSqXCvffeu6LHES5Wxy0tC4qi0N3djeHhYdTU1CAjI0NS4yiw+oWHc1uOklmyLbPvR8VwaiVo4oNXAgpJuXEeVxiEJhhnjw8DALZcshwNrcRct5kpGzKzuT/bljOhiSS7zDqcA0YdVhu8LAEhay6RaiwlSBUeJhRF4dFHH8WBAwfwz3/+E5s2bUJnZydee+01GI1GGY5SGqdPn8Zvf/tbbN68ecWOIdxEpfgEEg+3243m5maYzWbU1dUhISHhgp5YAPCn2thfaJVaFVB4gu1TrVbBafetglOp1HSqhRjSCSHa3BwNp0d8BIgLMaN1FoxmHDtsRsPl8i1IB1r3kQMS9YiFFB6QAaJqtcpPgFaz8Dz66KM4ePAgfaGvqqpCVVVVyPuXitlsxm233Ybf/e53+NGPfrRixxFuVlXazWaz4eTJk3C73di5cyfi4+MvaOGRYkNMUV6fHzHCEwy72Qq72QrrgtnnJ5QSazZiig3EYjg9AsPp1ZmK626R/7iFpt7mp4w+k6uBpfOFnDOeAFYK0Sw8TzzxBH7605/iH//4B7ZtC32fcvHlL38ZN9xwA3bv3r3ShxJWojLy4cJkMqGxsRE5OTmorKwEgItufSfw9vzrPv7bCtynwCjKMu+/yJ6cHnz+1Uqk4QCg7ewwNmwTFrEEazA9dnhYUPQz3DeJ4T6gpr5U0OsSTr7XH/DvXKZyXExP2+Gw2kVPspifCpx64nIsJUSz8Dz55JP44Q9/iDfeeAM7duyQ4cjk4YUXXsC5c+dw+vTquE6FQlSKD1tIxsbG0NbWhnXr1qGoqAher5euhlOLuEivFuERC5/w8MG+YGj1/qdBKOk7AFgw+t4pp2Sm8h7PStB2dmktSKgIrXbaTopPuZHhoXyQUTrkXGHPd5MbuYTnj3/8I77zne/g1Vdfxa5du2Q4MnkYHh7G1772Nbz11lucPUgXGlGddiOFBe3t7di6dSuKi4vh9Xp9PHiEslqER0yqTQ47YpVaBY/b4/cjNwvGeb+fcMNel+CCiFAoHDsc+j6k0ny8J6Tn86Xe5j6IePhuYpgz3Ahklls0C8/zzz+Pf/u3f8OBAwdwxRVXhH5gMnL27FlMTU1h27Zt0Gq10Gq1OHz4MB599FGf3sULhaiMfIClwoKWlhYsLCzQhQVu99IJL7aiDVhuQosGEWKLCxkPIibVFkx0/CyLBUYyBLYlMteIHrGpwWiAq2O/7eww1lRFxwgVuYoO+CrehKTe5lipNvZsNi7hIYSjx0cO4QGAl156Cffeey/+8pe/4KqrrpJln3Jy1VVXoaWlxeexz33uc6isrMS///u/R10RT6hEpfgQN1ONRoO6ujrodDp4PB5QFCVJeJgwO6FXQoi4S6ZDX98JuL3QRZ4AsO+Q+cazcL4+z/HOz8z5/K5WqZGanSr20GShu3kY6zaHPwXXeHxA9LoPgW/SgXF8Ful5GbzPE5NyM00uudmyzxkiQKtVeF555RV86UtfwvPPP4/rrrtOln3KTVJSEjZu3OjzWEJCAjIyMvwevxCISvGxWCxISUkJqbBACJEUIrmiBDmFh+39w/+a/vtgipGcPkFzU3P0/0dKiMh7ESpAC0b/Aotjh4eRXSDe6yXUHh8psKMft8sNrU5LCw+wFDmzz53VKjyvv/46vvCFL+CPf/wj9uzZI8s+FUInKsUnMzMTycnJdLQDiCsskAJ7NpQUMSIXabbXyGpIT/EJkZBjZw8uZYqRULHk8pJZCSEKRwQ0/IHdRDTDFB4CU4ACWSZEs/C89dZb+NznPocnn3wSN998syz7jCSHDh1a6UMIG1EpPkajETExS2Znckc7QhEbFTEv0uEUG67G0oDbC1j7CfQalNcjeuo180KlVquglSEyEitEQooN+JAqQH2twyjfGLnqOWaxAVfqbWJkDuN9I35W6lx4vRRnr9dqTrW9++67+PSnP43HH38c+/fvl2WfCvKhokJxbQsTn/zkJ3H48GF89KMfxU033YSGhgZBPhyRgi1GKxXZhFzpJmEtSIwQcV3M+ISIK/IRCpcYcYkPnzkaV8WXy+lE4dp8zu250m4AsGgyc4oPV+RD1n340m7b6ot5e3zIug+70o1r3SeQ+JDU2/yMiX6M/W9GbiS4zpVoFp4jR47glltuwSOPPILPf/7zK3IDqxCYqMwHPf300/j9738Pt9uNz3zmM1i7di3+9V//Fe+88w5cAbqpI8Xzj6/BA18044mfpK/YMcjRoMmchCD0NSmv1+eHD76JCW6H0+cHCE14gKWoiBkZiSFQl/9I75jo/fW1rlzZNZvxviVhY49MYsIUHsBXtJkRLPs8kSI8wc4buYTnxIkTuPXWW/Hwww8rwhPFRE84wUCv1+P666/H9ddfD5fLhcOHD+P//u//cNddd8HlcuGGG27Avn37cOWVV9LpuUgxOjqKzs5OVFVVIT8/H2+96Dt6/ZpPng37MYRjMgDtNskjBHyvybyQSIkA3YwLnJD0UCCYApScGXzCghBGesd4I6DVhNNu9/t8TRMznJGs10vxjsyhKK+kPh624LDPm78/WyN6n1ycOXMGH/vYx/CDH/wAX/rSlxThiWKiMu3Gh8fjwdGjR/F///d/OHDgAMxmM6677jrs27cPu3fvRlxc8InMUqEoCn19fRgeHsaWLVuQnh486pFbiMI9jkas8ARC6uRrJlLFyMu4Q0/NWq5A40q78aXc2DAFiCvttmjyfYyZfuMrOKipLw1Y7cYXlfGl3QDf1BuJfAjsz5P48bAFiHlDwP63l0N42HzrSxYAoO2uMzIyJKXZm5qacMMNN+Bb3/oWvvGNbyjCE+WsKvFh4vF4cOLECbz00ks4cOAAZmZmcM0112Dv3r245pprkJiYKOtrtbe3Y35+Hlu3bpW071CFaKWER67jCFWMxAiRlyONyLxoMufOCRUfYEmAAq33MBEiPsF6pQKJz4k3GxGf7H8eEvFhCw+BfI5sIzgiQG6eqraAVtkBmqSDCc9bL14CiqIwPz+PqakpTE9Pw263Iz09nba7FpLdaG1txfXXX4+vf/3r+Na3vqUIzypg1YoPE6/Xi7Nnz9IR0cjICK6++mrs3bsX1113HVJSpKdgnE4nDAYDKIrC1q1bodfL09MiRowiOYBTbBHCSkRFgYSIS3gA7jv2+KQEzm35xMflcCGDp5mTLT7AkgAFKrNemJ5Den4m599mR6eQks0fXRvHpgHAT4Ckig+Bby1HrMAQj59AcFlfA6DtrqenpzE/P4+kpCRkZ2cjKysLCQkJfsLS0dGB6667Dl/84hfx/e9/XxGeVcIFIT5MvF4vmpubaSHq6+vDhz/8Yezduxc33HAD0tLSBJ+cFosFjY2NdOdxuMZbBBKilZr8DKxOIQoW9fg+vrRtQkqSz+OBxAcApwCFQ3wA8AoQn/gAoO0PuJp/nTYHAG6L9WDCw0aIwPDBJzxsnE4nLUSzs7OIiYlBVlYWnYUYGBjAddddh9tvvx0/+clPwt4PqCAfF5z4MKEoCh0dHfi///s/vPzyy2hvb8fll1+Offv24cYbb0RmZiavEJlMJhgMBuTn56OioiJid1NMIVpJ4WGzWoRIivgQElKSggoPgSlAXMJDH1Mcf6RMpkZzCVAg8SHCQ2ALENN7hylARHgITAESKzyhIFR42Hg8HhiNRkxOTuKWW27B3NwckpOTsXXrVjz77LMhZTgUIs8FLT5MKIpCb28vLURNTU3YtWsX9u3bhz179iAnJ4cWmPb2doyPj9MWDivJtbedW9HX5yJahYiUCevjfNcIuMSH76LKfi6BLT7AsgAFEh/boplTQJh2BWzxIcJDYD+fLT7AsgCxTd+AJQFiC48Qokl42PT19eGzn/0sdDodzGYzneH40Y9+FHFjuIceeggvv/wyOjs7ERcXh4aGBjz88MNYv359RI9jtXHRiA8TiqIwMDCAl156CS+//DJOnTqFuro67NmzB319ffjrX/+K999/H/n50VViqwgRP3wTDfjmzvFdWEkEwLQM5xIeQkZeBq/42BaXH2cLCNsrhylAbPFhPp9LeIDA4gMIi/6Y6znRLDyjo6O45pprsHv3bvzmN7+BWq1GT08PXn31VezZswcVFRWyvI5Qrr32WnziE5/AJZdcArfbjW9/+9toaWlBe3s7EhK41xUVLlLxYUJRFEZGRvDnP/8ZP/vZz7C4uIjt27fTlXOlpaVRuYAZjUIEhF+M+IQo2DgdpggFEx4msYnxvOLDnnfGXjsSIz7AsgBJER+Py837WTLNA4UITChrOXzIJTwTExO49tpr0dDQgCeffDIqbQamp6eRnZ2Nw4cP47LLLlvpw4laLvrVOZVKhaSkJLzxxhvIz8/HkSNHsH//frzzzjvYunUrLr30UvzsZz9Dd3c3okmn//FsLf0TTYidmkB5KfpHCFyGd0LmuLkcTvpHDHazVfC2lvlF3r8x7agDOYRO9HNXqU0PjmN6cBwejvJr8hjX58h2rRUynSJahWdqago33HADLrnkEvz+97+PSuEBgPn5pehTSC/gxcxFH/kAwD/+8Q888cQTePbZZ+keHoqiMDMzg1deeQUvvfQS/vnPf2LdunXYu3cv9u3bh6qqqqiJiEgacWBgAA//PjoXXcMVEZGLrdCZc8wLKzMa4ltwZ4qcPm65si6QiCWkJPlEPUzI87jKxZljcAIVCgCA5oMeIS4xIoQjdSYWuYRnZmYGN9xwAyorK/Hcc89Bp9PJsl+5oSgKe/fuhclkwpEjR1b6cKIaRXw+gKIoXjGhKApzc3N49dVX8dJLL+HgwYMoKSnB3r17cdNNN2Hjxo0rVuJJURQ6OzsxNTWF2tpaJCUtp34uhtQcV8TEJ0R8F2NdjF6Q+ADLAsQnPmQsDddaE/s5TAHimr9G9hEtxQJikUt4TCYTbrzxRhQXF+Mvf/mLbL124eDLX/4yXn/9dRw9ehSFhf7GfwrLKOIjgYWFBfztb3/DSy+9hH/84x/Izc2lI6La2tqICZHH40FrayssFgtqamoCjhe6EIVISKqOKUR8F2SyH63ed+IAW3h8j4P7uNkz0YiA8ImVPjY24OBP5vH5Px65YgGxyCU88/Pz2LNnD7KysnDgwIGIz3IUw1e+8hX89a9/xXvvvYeysrKVPpyoRxGfEDGbzfj73/+Ol156CW+88QbS0tKwZ88e7Nu3Dzt27AhbXtrlcqGpqQkAsHXrVtFpiGgUIznsvnn3zTsYlfv0DxRtMaMkDeNz5xvGqYvRBzZjc3N7JrHXa6JNYPiQS3gWFxdx0003ISEhAa+++mpYZzeGAkVR+MpXvoIDBw7g0KFDEa+2W60o4iMjNpsNb775Jl5++WW89tpriI+Px0c/+lHs27dPVk8im82GxsZGxMfHY9OmTSEL3GoTIq5iBjHCRYQlUOREXoM9CYAvPafR6fgnQX/wOlyCxhYYIkLsx1cLcgmPxWLBzTffDLVajddffz2qS5bvuecePPfcc3jllVd8entSUlKiVjCjAUV8woTdbsc777yDl19+Ga+88go0Gg1uvPFG3HTTTbj00kslL5guLi6isbERWVlZqKyslL3oIdqFKFgVnRzRE9drqDWagB42lJfiFBcugSPbrVaB4UMu4bHZbPj4xz8Op9OJv//97z7rmNEI33fwD3/4A+64447IHswqQhGfCMD0JPrrX/8Kl8uFG2+8EXv37hXlSWQ0GmEwGFBaWhqR/qNoFCIxiImeyLZ84hYoegkkMAGjqyhOnYlFLuGx2+345Cc/ifn5ebz55pvKyJwLGEV8Iozb7aY9if7617/CbDbj+uuvx759+3DVVVfxhukTExNoa2ujTewizYUkRIEERujakBBx4eNCEh1APuFxOp349Kc/jfHxcbz99ttIS0sL/iSFVYsiPisI8SQiQjQzM4Nrr72W9iQiee4DBw4gJSUFmzdvRmYm9xTkSLLahYgLKdGLGJsBso0iPNy4XC7cfvvtOH/+PP75z38iI4PbukLhwkERnyjB6/XizJkztBXE2NgYdu/eDbfbjffffx9Hjx7FmjVrVvow/bgQhEiu6OVCFRg+5BIet9uNL3zhC2hra8O7776L7OxsWfarEN0o4hOFeL1enD59Gp///OcxODiI5ORk1NTU0J5EqampUTNdgclqFCIlbSYNuYTH4/HgS1/6Es6cOYNDhw4hNzdXlv0qRD+K+EQh8/PzuOmmm7C4uIi//e1vmJmZoa0gOjo6cMUVV9CeRBkZGYoQBUFMJVrwfSnC8+DXXLSzaHJysuTzz+Px4Ktf/SqOHj2KQ4cOoaCgQOYjVYhmFPGJQsbGxvC9730Pv/jFL+hZc8BSM1tPTw8tRAaDAR/60Iewd+9eP0+ilcZms+HcuXNISkrCNx/mnz8mF2JLnRXhkcYbz9ZgdnYWU1NTmJmZgUajoYUoLS1N8HQPr9eL++67DwcPHsShQ4dQUlIS5iNXiDYU8VmlUBSF8+fP46WXXsKBAwdw6tQp1NfXY8+ePdi7dy8KCgpWTIjMZjPOnTvH2YsUjohIToHhfw1FeNipNq/XC5PJhKmpKUxNTcHr9SIrKwtZWVnIyMjgbar2er144IEH8Morr+Ddd99FeXl5JA5fIcpQxOcCgKIoDA8P4+WXX8aBAwfw/vvvY/v27di7dy/27t2LkpKSiAnR/Pw8GhsbUVhYiPLy8oCvK1SIpPbRyIUiPMHXeCiKwsLCAi1Edrsd6enpdFREhoF6vV78v//3//DCCy/g0KFDWLfu/7d3p0FNnW8bwC+KtQjiiqSlFmWpA24g0aIgoq0Li0CijuBWsZUZlCqo4zYu1drKuFVb/wVHqUtFLQIhCMoSFFErnaKAK2ItIi2QxDgaFFkkOe8Hh7xSlbIcchJy/2b8wAEOl9jm4jzn8NyDtBGf6CAqn06GYRhUVlYiKSkJIpEIFy5cwPDhwzVFZG9v32FF9OjRI1y7dg329vawtrZu1ee+rYi0US7NoeJp28MF1dXVkMvlePjwIaqqqhATEwNHR0fU1NQgLi4O2dnZGDx4cAekJfqCyqcTa5xJJBaLNTOJHBwcNEXE5kwimUyGmzdvsvJLsI1FxHXxvMxg2OXDxlNtNTU12Lt3LxITE3Hr1i0MGjQIQUFBEAgEcHJy4vQ+ZVRUFHbs2IHKykoMGTIEe/bsgYeHB2d5DInBTzLtzIyMjNCvXz+EhIQgLS0NUqkUy5cvR2FhIcaOHYuRI0fim2++wfXr16Fux4tseXk5bt26hWHDhrGy+0LjhNaME/x2n6ut/mvapyFg63FqExMTdO3aFeXl5cjOzsbGjRtx+/ZteHh44OjRo6x8jbaIi4tDREQE1q1bh4KCAnh4eMDb2xtlZWWcZTIkdOVjoJRKJVJTUyESiZCeno4PPvgA/v7+EAqFGDFiRIufWiotLcX9+/fh5OTU4WODp8y62qHnb2TopQOwVzwMwyAqKgpbt25FRkYGPvnkE837amtrwTAMZzs/u7q6wsXFBdHR0Zpjjo6OEAgEiIyM5CSTIaHyIXj27BnOnDkDkUiEM2fOoE+fPvDz84NQKMSoUaPeOLKBYRjcu3cP5eXlcHFxQY8ePbSauaOKiIqH3eKJiYnBxo0bkZaWBjc3N1bOy4b6+nqYmpoiPj4eQqFQczw8PByFhYXIycnhMJ1hYGfADNFr3bt3x8yZMzFz5kw8f/4cmZmZSExMxPTp02FmZqaZSTRmzBh06dIFDQ0NEIvF4PF4GDVqFCezVl5dkmOriKh42C2eI0eOYMOGDUhJSdGp4gEAhUIBlUoFHo/X5DiPx4NUKuUolWGh8iFNmJqaQiAQQCAQoLa2FllZWRCJRJg9eza6dOkCHx8fFBcXQy6XIzc3VyeGfLFRRFQ87GEYBsePH8fq1auRnJwMT09PriO91b8fdmAYRmd+UbuzM9gHDurq6uDs7AwjIyPNOGrSlImJCaZOnYqDBw9CKpXiwIEDkEgkKCoqQm1tLVauXImMjAzU1dVxHVUj4wRf86elqHheYuuqJyEhAcuWLUN8fDw+/fRTVs7JNgsLCxgbG792lSOXy1+7GiIdw2DLZ9WqVZzMxdFXVVVV+O677+Dg4ICSkhLExsbCzMwMX331FWxsbBASEoLU1FTU1NRwHVWjJUVExfMSW8UjFouxePFinDhxAl5eXqycsyN07doVfD4fEomkyXGJRKJzS4SdlUGWT1paGjIzM7Fz506uo+iN0tJSODg44PTp0+jduzfGjx+Pn376CWVlZUhNTYWlpSVWrVoFGxsbzJ8/H0lJSaiuruY6tkbGCT7Sj7vgpy3/PxmTiucltoonNTUVISEhOHr0KPz8/Fg5Z0davnw5YmJicPDgQRQVFWHZsmUoKytDaGgo19EMgsE97SaTycDn8yEWi2FhYQEbGxsUFBTA2dmZ62h6r3EURON+cxUVFZg0aRIEAgG8vb1hbm7OWTaGYXD37l3Nv7+ZmRkmB+ZxlkdXsFU8GRkZmDdvHn7++WcEBgayck5tiIqKwvbt21FZWYmhQ4di9+7dGDduHNexDIJBlQ/DMPDx8YG7uzvWr1+P0tJSKp8Oolarce3aNc0O3KWlpfjss880M4l69uyptRu7DMOgqKgIjx49Ap/Ph6mp6WsfY4hFxFbxZGdnIzAwENHR0Zg7dy7dsCct0inKZ9OmTdi8eXOzH5OXl4fLly8jLi4OFy5cgLGxMZWPljAMg1u3bmmK6M6dO5gwYQIEAgF8fX07dCaRWq3G7du3oVQqwefzYWJi8p+fYwhFxFbxXLx4ETNmzMCePXvwxRdfUPGQFusU5aNQKKBQKJr9mIEDByIoKAgpKSlN/gdRqVQwNjbGnDlzcOTIkY6OavAal78SExM1M4k8PDw0M4ksLS1ZewFTq9W4ceMGqqurwefz8d5777X6HJ2xiNgqntzcXAiFQmzbtg2hoaFUPKRVOkX5tFRZWRmqqqo0b1dUVGDKlClISEiAq6sr+vfvz2E6w8MwDEpKSjT3iPLy8uDm5qaZSWRlZdWuKZnXr19HXV0dXFxcNFv6t0dnKCK2iufKlSvw9/fHN998gyVLllDxkFYzqPL5N1p20x2vziQSiUS4fPmyZiaRQCCAtbV1i1/gVCoVCgsLoVKpMGLECLz77rus59XHImKreAoLC+Hr64t169ZhxYoVVDykTah8qHx0zqsziRITE3Hx4kUMHz4cAoEAAQEBzQ6pa2hoQEFBAYyMjODs7PzWaZps0ociYqt4bt68CW9vb6xYsQJr166l4iFtZtDlQ3Rf40yixiLKzs6Gg4ODpoheHdP9+PFjFBcXo2vXrnBycnrjhqgdTReL6H9beoLH46FHjx7tKouioiJ4e3tj0aJF2LRpExUPaRcqH6I3GIbB48ePkZycDJFIBIlEAltbWwQEBGDs2LFYunQpZs2ahTVr1rR4JERH0oUiit07QDNR1NjYGJaWlrC0tESvXr1a9T26e/cuvL29ERwcjK1bt1LxkHaj8uFQaWkptmzZgnPnzkEqlcLKygpz587FunXrWLlB3tkplUqkpKTg2LFjyM7Ohq2tLby8vDBt2jQ4OzvrRAE14qKIXl1qU6vVePz4MeRyOeRyORiGQb9+/WBpaYm+ffs2+70qKSmBl5cXZs6ciZ07d+rU95XoLyofDqWnpyMuLg6zZs2Cvb09bt68iZCQEMybN4+2/mmhxl9edXd3h4+PD5KSknDmzBlYWFg0mUmkSy+Y2iii5u7xMAwDpVIJmUwGuVyOFy9ewMLCAjweD3379m1yn+zBgwfw8vKCn58ffvzxR536PhL9RuWjY3bs2IHo6GiUlJRwHUUv+Pv7Y8CAAfjhhx80L4zPnz9HRkYGEhMTkZqaiu7du8Pf318zk4iLe0Fv0xFF1JqHCxiGwdOnTzVXRDU1Nfjzzz/R0NCAcePGYfbs2Zg0aRKio6OpeAir6L8mHaNUKjt8HHVncuzYsdd+Ijc1NYVQKERsbCykUin27duHmpoazJo1Cx9//DHCw8Nx/vx5vHjxgsPkL2XGjUL8/kFYG/qMtfO1hpGREXr06AF7e3u4ublh9OjRUKvViImJgbu7O2prazF8+HDIZDJW8rGptLQUX375JWxsbNCtWzfY2dnh66+/Rn19PdfRSAvQlY8O+euvv+Di4oJdu3Zh4cKFXMfpdF68eIHs7GwkJCQgOTkZKpUKU6dOhUAgwPjx4zm5z6ZUKpGfnw9bW1sMGDBAc7wtV0RsPU4tl8vh7e0NR0dHuLu7Izk5GZcvX0ZQUBBiY2NZ+RpsoGVr/Ubl0wFautfcyJEjNW9XVFTA09MTnp6eiImJ6eiIBq+hoQEXL15EQkICxGIxnj9/Dl9fX/j7+2PixIkt2gOuvd5WPP/WkiJiq3gUCgV8fX3h4OCA48ePa35BVyaT4f79+xg9ejQrX6ej0LK1/qDy6QAt3Wuu8QWuoqICEyZMgKurKw4fPkxr61qmUqlw+fJlzTY/T548wZQpUyAQCDB58uQ37oLdXo3FY2dnB2tr6xZ/3puKiK3iefz4MaZOnYoBAwbg5MmTevnE5fr165Geno4rV65wHYX8ByofjpWXl2PChAng8/mIjY3VqZvhhqhxJlFCQgKSkpIglUo1M4m8vLxYmUnU1uLpSEqlEn5+fuDxeBCJRG3ahJVrtGytX6h8ONS41GZtbY1ffvmlSfG8//77HCYjwMsiKiws1BRRaWkpJk6ciICAAPj4+LRpJtGTJ09QUFCgU8Xz9OlTCAQCmJub49SpU1pZcmwOLVsbBiofDh0+fBgLFix44/von0W3MAyDmzdvaoqouLi4yUyiPn36/GcR6WLxVFdXY/r06TA2NkZqairMzMy4jkTL1gaCyoeQVmIYBsXFxZqZRNevX8e4ceMQEBAAPz+/N84k0sXiqampwYwZM9DQ0IAzZ85wOua8rWjZWn9R+RDSDq/OJBKJRLh69SrGjBmjGY5nZWWFs2fPIjc3F8HBwfjoo4+4jgwAqK2tRVBQEKqqqpCRkYGePXtyHanVaNlav1H5EMIShmFQVlammUmUm5sLZ2dnFBUVYdGiRdi8ebNObMhZX1+PuXPnQiqVQiKRoHfv3lxHahNattZvVD4EUVFR2LFjByorKzFkyBDs2bMHHh4eXMfSawzD4NSpUwgKCoKjoyNu3LgBJycnzSgIW1tbToroxYsXmD9/PkpLS3H27Fn07dtX6xkIAWh7HYMXFxeHiIgIrFu3DgUFBfDw8IC3tzfKysq4jqbXcnNz8fnnn+P777/H1atX8c8//yAkJAQXL14En8+Hm5sbtm3bhjt37mjtp/SGhgYsXLgQ9+7dQ2ZmJhUP4RRd+Rg4V1dXuLi4IDo6WnPM0dERAoEAkZGRHCbTb/n5+cjPz3/t901enUmUmJiIrKws2NnZwd/fH0KhEIMHD+6Qp7VUKhVCQ0ORn5+P7OxsuidCOEflY8Dq6+thamqK+Ph4CIVCzfHw8HAUFhYiJyeHw3SG4cmTJ0hJSYFIJEJGRgb69++PgIAACAQCODk5sVJEKpUKS5cuxaVLl3D+/Hl8+OGHLCQnpH1o2c2AKRQKqFQq8Hi8Jsd5PB6kUilHqQxLr169MG/ePCQlJUEmk2HLli2aGTrDhg3D2rVr8ccff0CtVrfp/Gq1GitWrEBOTg6ysrKoeIjOoPIhr934ZhhGJ57KMjTm5uYIDAzEyZMnIZPJsGvXLigUCggEAjg6OmLlypX47bffoFKpWnQ+tVqNNWvWID09HVlZWc1uXkqItlH5GDALCwsYGxu/dpUjl8tfuxoi2mVqaopp06bh2LFjkEqliIqKwvPnzxEYGIhBgwYhIiKi2ZlEarUaGzZsQFJSErKysmBra6vlvwEhzaPyMWBdu3YFn8+HRCJpclwikcDNzY2jVOTfTExM4Ofnh0OHDkEqleLw4cMAgODgYNjb22Px4sWQSCSaIWoMw+Dbb7/FiRMnkJWVhUGDBnGYnpA3owcODFxcXBzmzZuHffv2YcyYMdi/fz8OHDiAW7du0TKNjmucSRQfHw+xWIyamhr4+vpCpVIhKysL586dw7Bhw7iOScgbUfkQREVFYfv27aisrMTQoUOxe/dujBs3jutYpBUaZxIdPXoUR44cQWZmJjw9PbmORchbUfkQ0smoVCraYJPoPLrnQ0gnow/FU1dXB2dnZxgZGaGwsJDrOIQDVD5E50RGRmLUqFEwNzeHpaUlBAIBiouLuY5FWLRq1SpYWVlxHYNwiMqH6JycnByEhYXh999/h0QiQUNDAyZPnozq6mquoxEWpKWlITMzEzt37uQ6CuEQ3fMhOu/hw4ewtLRETk4OPQih52QyGfh8PsRiMSwsLGBjY4OCggI4OztzHY1oGV35EJ2nVCoBAH369OE4CWkPhmEQHByM0NBQjBw5kus4hGNUPkSnMQyD5cuXY+zYsRg6dCjXccgbbNq0CUZGRs3+uXLlCvbu3YuqqiqsXbuW68hEB9CyG9FpYWFhOH36NC5duoT+/ftzHYe8gUKhgEKhaPZjBg4ciKCgIKSkpDTZN7DxsfA5c+bgyJEjHR2V6BAqH6KzlixZArFYjAsXLsDGxobrOKSdysrKUFVVpXm7oqICU6ZMQUJCAlxdXemHCwPThesAhPwbwzBYsmQJkpKScP78eSqeTsLa2rrJ2927dwcA2NnZUfEYICofonPCwsJw/PhxJCcnw9zcXLPrds+ePdGtWzeO0xFC2EDLbkTnvG2W0KFDhxAcHKzdMISQDkFXPkTn0M9DhHR+9Kg1IYQQraPyIYQQonVUPoS0UWRkJIyMjBAREcF1FEL0DpUPIW2Ql5eH/fv3Y/jw4VxHIUQvUfkQ0krPnj3DnDlzcODAAfTu3ZvrOIToJSofQlopLCwMvr6+mDhxItdRCNFb9Kg1Ia3w66+/Ij8/H3l5eVxHIUSvUfkQ0kJ///03wsPDkZmZCRMTE67jEKLXaIcDQlpILBZDKBTC2NhYc0ylUsHIyAjvvPMO6urqmryPEPJ2VD6EtNDTp0/x4MGDJscWLFgABwcHrF69muYNEdIKtOxGSAuZm5u/VjBmZmbo27cvFQ8hrURPuxFCCNE6WnYjhBCidXTlQwghROuofAghhGgdlQ8hhBCto/IhhBCidVQ+hBBCtI7KhxBCiNZR+RBCCNE6Kh9CCCFaR+VDCCFE66h8CCGEaB2VDyGEEK37PxXJEtlVZGDDAAAAAElFTkSuQmCC" - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": "<Figure size 640x480 with 1 Axes>", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiIAAAGdCAYAAAAvwBgXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAjj0lEQVR4nO3df2yV9d3/8dehZgWk7S0ScKRFGrLEH8QRi99miyJkpsz4xzDLonfI7kDAjKwQGMmtKf4BGHP3D5ySsYAyM9ExkWxubCa6W5JNcDHmxgpxcdNEDWu1Q+1yrwecOcRyvn/se/o9LefH9ePzuT6f67qej6R/tJz2XHp6Tp/n/fmc6xTK5XJZAAAADsxwfQAAACC/CBEAAOAMIQIAAJwhRAAAgDOECAAAcIYQAQAAzhAiAADAGUIEAAA4c4XrA2jk0qVLGh0dVVtbmwqFguvDAQAAAZTLZZ0/f14LFy7UjBmNZx5eh8jo6Ki6urpcHwYAAIhgZGREnZ2dDS/jdYi0tbVJkva99D+adeUcx0cDAACC+PyzC9py5/+Z/DveiNchUlmOmXXlHM2e0/w/BgAA+CPItgo2qwIAAGcIEQAA4AwhAgAAnCFEAACAM4QIAABwhhABAADOECIAAMAZQgQAADhDiAAAAGcIEQAA4AwhAgAAnCFEAACAM4QIAABwhhABAADOECIAAMAZQgQAADhDiAAAAGcIEQAA4AwhAgAAnCFEAACAM4QIAABwJrEQGRwcVKFQ0LZt25K6SgAA4LlEQuTUqVM6ePCgbrrppiSuDgAApIT1ELlw4YLWrl2rn/zkJ7rqqqtsXx0AAEgR6yHS39+vu+66S3fccUfTy5ZKJRWLxSkfAAAgu66w+cOfe+45vfnmmzp16lSgyw8ODmr37t02DwkAAHjE2kRkZGREW7du1eHDhzVz5sxA3zMwMKDx8fHJj5GREVuHBwAAPGBtIjI0NKRPPvlEPT09k1+bmJjQyZMn9eMf/1ilUkktLS1Tvqe1tVWtra22DgkAAHjGWoh84xvf0J/+9KcpX1u/fr2uu+46PfDAA5dFCAAAyB9rIdLW1qalS5dO+dqVV16pq6+++rKvAwCAfOLMqgAAwBmrr5qZ7pVXXkny6gAAgOeYiAAAAGcIEQAA4AwhAgAAnCFEAACAM4QIAABwhhABAADOECIAAMAZQgQAADiT6AnNAMCVOVcdsX4dF/73361fB5A1hAiAzEgiNqJeP5EC1EaIAEgt1+ERRvWxEiXA/0eIAEiNNIVHI9P/OwgT5BkhAsArWYmNMGr9NxMnyAtCBIAzSUbH7hPPWL+Onbf/h7GfxdQEeUGIAEiEzehIIjKCaHYccUKFPSbIKkIEgDU24sOX6Ihi+rFHDROiBFlCiAAwzlSAmIyOYzv+Yexn1bLmv/4t9PeYCBOiBGlHiAAwxkSARI0P26ER5/qDRkr1f3ucKCFIkCaECIBY8hwfQVUfZxJRMueqI8QIUoMQARCJiwCJEx4Lf/7tyN/bzOja5wNfNk6UhAkSpiNIi0K5XC67Poh6isWiOjo69OTJP2v2nDbXhwNA8QPEdnzYDI4ogkZKmD0mUZZtCBIk6Z8Xzmvjihs0Pj6u9vb2hpclRAAEEidAbMaHb+HRiOkoCRskxAiSQogAMCLJ6UfQ+IgbHkPnvxfr+6v1tD0R+XuDRAlBgrQiRADEktT0w2Z8mAyOMKLESbMosREkxAhsIkQAhJbU9CNIfIQND1fREVSYOGkUJUGChOkIfECIAAgsiemHjcmH7/FRT9AoiRMkxAhcI0QANOVLgASND1PhcWjPH438HEla95+3xvr+IFGSVJAQIzCJEAFQlw8BEiQ+4oSHydiIIkqgNIuSqEFCjMAFQgRATVEjJKkACRsfroMjjDBxUi9K4mxqJUiQJEIEwBQuA8Tk9CNN4dFI0CiJEiTECHxAiACQZH8ZJk6ABIkP0+GxuvNRYz/rvz/cbuTnBIkSk0FCjCAJhAgAq1OQNAWIyfiox0SUNAsSV9MRYgRRECJAjuU9QJIIj0biREnUGJHqBwkxAhcIESCHfN0HkkSAuI6PeqJGienpCDGCpBEiQM74OAWxHSC+xkctNoKEGIHPCBEgJ/IWIGmKj3rCRkmU6QgxAtcIESAHXERI3H0gUQIkC/FRi8kgIUbgG0IEyLA0TkEIkPrCBImppRpiBLYRIkBG+TYFMR0geYmPeoJECTGCNCBEgAyKEiFpCJC8x0ctNoKEGEGSCBEgQ2xNQaIswxAgySFGkGZhQmRGQscEIIKoU5CoEbLw59+OFCGH9vyRCDEsyP+jRv/Pa91etW7bRkEa9BwzQByECOApX5Zihs5/r2mEBLW681EiJIQg/7/C7sMJGyNBxHlPI4ClGcAzSS/FRN0LwgQkWVGXakxsYGWJBmGxNAOklIulmFpMT0EQX9T/j/VuxzCTEZZoYBMhAnjCp6WYesLsBWEZxryoyzRJxQhLNIjiCtcHAOSdL0sxpl4RQ3zYtbrz0YbLNIf2/LHmEs3Q+e81fPdeU+ZcdYQlGoTCHhHAobRMQYIgQJLVbM9IvZf2xn1ZL/tFEAR7RIAU8CFCTO0FIUKSZ3KZhv0icImJCJCwNCzFECDp4fNkhKlIfjERATzl06ti6gkSIWxE9YfPkxE2ryIIQgRIiA9LMZKZCIFfTL+0F0gSSzNAAmxECEsxmK7RMg1LNEgSSzOAR7IQISzFpEOj24glGviKEAEsykqEIBvixkg9vJIGcRAigAVzrjoSOkJ83ZSKdDH9JnnV4rw5HlMR1EOIAIYluSl14c+/Hfr8IEFO085STLbVuv1ZooErhAhgkK2lmHoRUgtTEAS5DYkR+IIQAQxhPwh8EjVGgiJGYAohAsQUZT+I5FeEsBSTTVFu07gbV4GwCBEgBh/OlGpiPwjyiyUauEaIABH5sim1HpZiINk/6yqvpEFchAgQAZtSkSamXtLLuUVgAyEChMR+EKRRlBiJu0QTBFMRECJACD5HCPtB0IypzatBMRVBEFZDZHBwULfccova2to0f/58rVmzRu+++67NqwSs8T1CmiFCIEV7P5rpmIrAJKshcuLECfX39+v111/X8ePH9cUXX6ivr0+fffaZzasFjCNCkCVhfx/iLNEwFUEzVkPkd7/7ndatW6cbb7xRX/3qV/XUU09peHhYQ0NDNq8WMIoIQRbV+70I88Z4QfFyXjSS6B6R8fFxSdLcuXNr/nupVFKxWJzyAbiUVITUe3muRITAX6aXaJBPiYVIuVzW9u3bdeutt2rp0qU1LzM4OKiOjo7Jj66urqQOD7hMkhFSS7MTlTVDhKAZE1ORoC/pZSqCehILkc2bN+utt97SkSP1f9EGBgY0Pj4++TEyMpLU4QFTmI6QJM8RwstzYUKcJZo4Z1xF/iQSIlu2bNFvf/tb/eEPf1BnZ2fdy7W2tqq9vX3KB5A0GxFSi60IAcKw9SqaqJiK5I/VECmXy9q8ebN+9atf6fe//726u7ttXh0Qm68RwjlC4BOmIjDJaoj09/fr8OHDevbZZ9XW1qZz587p3Llz+vzzz21eLRCJzxHSDBGCOJKeijSLEaYi+WI1RA4cOKDx8XGtXLlSX/7ylyc/jh49avNqgdCIEKC+oKd/n45X0CAI60sztT7WrVtn82qBUIgQwMwb44V5OS9TEVTwXjPINSIEiC7Oy3klYgT/Qoggt4gQYCpT79A7He9Dg0YIEeQSEQJEk/QSDbKPEEHuECFAfVF+x+K8D00QTEWyjRBBrhAhQHMmfteYiiAoQgS54WOEBD1RGRECn0TdKxIHU5HsIkSQCy4jpN6b1zEFgc+SnooEQYxkEyGCzHMdIbUQIUiDsGdcDfpy3lr3IZZn8osQQaYRIUC2MBXJHkIEmUWEAPExFYFthAgyiQgBzPHt95KpSLYQIsgckxFybMc/iBBA9X8/mYogLkIEmWJ6ElIPEQI0Fvd9aJphKpIdhAgyg+UYwK4wU5GgmIqAEEEmECGAf5iKIAhCBKlHhADJCTsVifruvExF8oMQQaqFjZDdJ54hQgBLgi7RBJ2KECP5QIggtaJESCMmIqQZ3jcGeRR1KhIEyzPpR4gglXyNkEbPCAkQZEXYk5zVwlQEFYQIUocIAdInzlSEGMk2QgSpQoQAfkhyKtIMyzPpRoggNYgQID2CnnE1KKYi2UWIIBWIEMA/Jn7Hp9/n2LSaP4QIvGf6ASaJCAEQfSoSda8IMZJOhAgyx/Z5QqTmEcI0BHnR7Hc9SLCbPNsq0ocQgddMLsmEiZBGiBAgHqYiqEaIwFsuIyTqWVOJEOSRjb0icRAj6UKIwEtECJAdUfdTcV6RfCBE4B0iBEifsPeBoO/My6toso8QgVeIECCbTL/KLMhUhBhJB0IE3iBCgHTzbSqCdCBE4AXTJyyrhQgB3HJx7h2mIv4jROBcEmdNJUKAZCQ5FWHTajYQIsgUIgRwz0SMmMRUxG+ECJxKYl9ILUQI4Eacd+ZlKpJNhAiccbU5lQgB7PNtKgJ/ESJwIm0RAiC8ejESZyoSFcsz/iJEkLg0RgjTECCaMPedqO9Bw/JMuhEiSC0Tb2JHhABuMBVBBSGCRJmahpg4YRkRAviHqUj+ECJIDBEC5JNPe0XgH0IEibAdIfVE3YlPhADu2JyKsDzjH0IE3jF1+nZepgv4z/RUhCWa9CFEYF2YZyCuXyFDhAB2cF4R1EOIwCoiBEAzUc/jU+9xodlUhOUZvxAisIYIARDH9Pszm1aziRCBFaYipB4iBEifRve1pKci8AchAuNMjj2Dvpsur44B0i9IjJiairA84w9CBEbZPn172AehRg9sRAjgvzibVpmKpAMhAmOSeA+ZWngjOyA9mj0BiDIVCXt+oQqmIn4gROAEm1MBBMVUJNsIERjB5lQAQZmYikzHVCS9CBF4Je7mVCIEyIcw+8WYiviNEEFsNs8XEubBhggBsivq+88EwVTELUIEXoi7OZWNqUC62Ni0inQiRBCLiWmIic2pzTANAdIvzlSE0777ixBBZDY3qLIvBMg+E/dNpiLpR4jAKfaFAKjH9Gnfm2Eq4gYhgkhsLcnUi5AoSzJECOC/sPfToI8FtR5fgkxmiZHkESKwKuy+kFqiLMkQIUB+sDyTboQIQgv6jMHVvhAiBEiXsO/KG+dMq/APIYLE2T5pGYD8qfUYwvJMOhAiCCXuNCTqJrIKNqcC2cR9N78IEQTmw5JMIzyQAelW7z4cdHmGvSLpRIjAKBOncGdfCABTeCmv/xIJkf3796u7u1szZ85UT0+PXn311SSuFga5vlOyLwTIhzBPKqJuWuVN8PxiPUSOHj2qbdu26cEHH9Tp06d122236c4779Tw8LDtq4YhJpZk4kxD2BcCIOiTEZZn0sd6iDz66KPasGGDNm7cqOuvv1579+5VV1eXDhw4YPuqYYDvEQIge0xPRVie8ZvVELl48aKGhobU19c35et9fX167bXXLrt8qVRSsVic8oF0C/oAwJvZATAlyFSE5Rl/WA2RsbExTUxMaMGCBVO+vmDBAp07d+6yyw8ODqqjo2Pyo6ury+bhoYkkXyVTC0syAKoxIc2mRDarFgqFKZ+Xy+XLviZJAwMDGh8fn/wYGRlJ4vBgCUsyAJJgc3kG9lkNkXnz5qmlpeWy6ccnn3xy2ZREklpbW9Xe3j7lA6iFaQiQbWHOKVKLqeUZ9onYZzVEvvSlL6mnp0fHjx+f8vXjx4/r61//us2rRkw2zqDKq2QA2ML7z6SX9aWZ7du368knn9RPf/pT/eUvf9EPfvADDQ8Pa9OmTbavGo4wAgWQFmxadc96iNxzzz3au3evHnroIS1btkwnT57Uiy++qGuvvdb2VSMiG5tUmYYAMCXqHjKeJPkpkc2q3//+93X27FmVSiUNDQ1pxYoVSVwtHIh7GvdGiBAAjUx/XDF1cjP2idjFe83AG7yXDIBqSd3vWZ5xixDBFHGWZWxOQwCgGssz2UGIwAtMQwDYEHR5hqmIO4QIQktyGkKEAAiDiWv6ECKY5GpDVr1pCBECoBHOwJwNhAhCCTq+ZG8IABN4QpJ9hAhiC7L5q16EMA0BkLR6j1mNnmjxEl57CBEEFmcaAgA2BFme4THJb4QIrAs7DQGAOFgGThdCBJKijx1tvCafZRkAYfCkJt0IEQRielmGBw4AQSX55ITziSSPEIFVjEgB+IgzrPqDEEFT9Z4hRL0jcxZVAEljw6q/CBEYwx0dABAWIQJrai3LsDcEQBJYFk4PQgQAkHo8yUkvQgQNmdwf0uyBgv0hAJA/hAiMYH8IAJt4opJdhAisYH0WgO/CvucM7zdjByGCRLB+CwCohRCBF5XP2BUA8okQQV2mT2QGAMB0hAhiY6MqACAqQgTGsVEVABAUIQJd+N9/t/rz2agKIAtsP1bmFSECAACcIUTgHK+YAYD8IkQAAIAzhAgAAFV23v4frg8hVwgRAADgDCECAACcIUQAAIAzhAgAwHv//eF24z9zzX/9m/GfifAIEQAA4AwhgrrYOQ4gLdb9561TPu9pe8LRkSAsQgTO2Ri5AkC10bXPT/m83rJMvSdgnN7dHkIEAAA4Q4ggtOnPJKY/0wAAk5iaZhshAklmx46szQIAgiJEYN30TWQAYJKpxxg26LtBiAAAMmX6VDboRlW4QYigIZ4hAHCp2f6QJCauvGLGLkIEkZh+RsFmNAAu8aTLHUIETQW5g04ffU4fjbJPBIBptR5XoizLECFuESKY5Hr8yFQEQDUfHhNcPy7mASGCQEw8Y2AqAsCUKNOQWpiGuEeIILJmJzbjfCIAfMKrZfxEiCBRzaYiPoxiAbjX6LEgqfOGsCyTDEIEUzS645kaYbJEAyCqeo8fnDskvQgRxML7zgBII6Yh/iBEEErYqUi9fSKNpiIszwD5Vu8xIOo0BH4jRHCZsM8EmIoAMCXsE5Egm+KnP0YxDfELIYLQmIoAsMHEBtW4e0OIkOQRIqjJ9FSEGAHQSJQIMX0WVSLEDUIEkUR5BU3UGCFIgGyLch83vS+ECHGHEEFdPu0VIUaAbIry7rpR9oVInEXVV4QIGop7XhETSzQVxAiQLaYihCWZdCNEYFSQjWHECABT92WWZNKPEEFTce+otR4o4rwPDTECpFuQ+3DUN7ULMw0hQvxAiCCWWnfwoC+XqxUjQV+iR4wA6RT1vht1X0g9RIg/CBEEEnavSNCNq3FjhCAB0iPoJGT6Y0DUfSESG1TTgBBBYKY3rlbEiRGJIAF8F/Q+anJzqsSSTFoQIgglzB241oND2BgJGyQA/BL0fkmE5BchAmOC7hcZXft8qA2sTEeAdIq6FCMRIXliLUTOnj2rDRs2qLu7W7NmzdKSJUu0c+dOXbx40dZVIiFx94tUhI0RggRIhzhLMRJ7QvLGWoi88847unTpkp544gm9/fbbeuyxx/T4449rx44dtq4SCXIRI1K46YhEkABJCnN/SzpCmIb4q1Aul8tJXdmePXt04MABffDBB4EuXywW1dHRoSdP/lmz57RZPjqENeeqIw3/ffeJZy772rEd/6h52YU///ZlXxs6/72GP//Qnj82/PdaVnc+Gvp7ADQXJvjDnLY97rlCJCLEhX9eOK+NK27Q+Pi42tvbG1420T0i4+Pjmjt3bpJXCYui3LnDTkZMTkckJiSAaWGnIElHCPyXWIi8//772rdvnzZt2lT3MqVSScViccoH0qveg0OYGJGaL9UQJIAbcacgkv0IYRriv9BLM7t27dLu3bsbXubUqVNavnz55Oejo6O6/fbbdfvtt+vJJ58M/bNZmvFblCUaKdwyTYWN5ZpqLN0AzYWN+LhTEIkISZswSzOhQ2RsbExjY2MNL7N48WLNnDlT0r8iZNWqVert7dWhQ4c0Y0b9IUypVFKpVJr8vFgsqqurixBJgagxItUOkkYxIjUPEil+lFQQJ4CZ+KhIYimGCHHLaoiE8dFHH2nVqlXq6enR4cOH1dLSEur72ayaLqZjRDITJJK5KKlGoCDLoi5dmgoQiSlImnkRIpXlmEWLFumZZ56ZEiHXXHNNoJ9BiKRPsxiRwi/VSOaCRLITJdUIFKRVnH1TzfZqESH54kWIHDp0SOvXr6/5b0GvkhBJJ1sxIpkNEsl+lFQjUOAbExu2kwgQiQhJGy9CxARCJL3ixIiUfJBIyUZJNQIFSUoiPqTwASKxHyRLCBF4wXaMSM2DRIoWJZK7MKkgUGCKqZeqxwkQyfwURCJCfEWIwBtBYkTyO0gk91EyHZGCRkyfI8dWgEhESFYRIvCKiRiRzAWJFC9KJP/CpIJAyS8X8SE1DhAp/BREIkKygBCBd0zFiGQ2SKT4USL5GybViJRssXVmYAIEJhAi8FbSQSKFixLJTJhUS0OkxEHgxOfy7QbCvkVC1ACR4i3DSERImhAi8JrJGJGCB4kUPkok82EyXdZDxTe1wilP7zsU5b2Z4sSHFH8KIhEhaUOIwHtBY0SyEyRStCipsB0ntRAsCCNKcFRrFh8SAYL6CBGkhg9BIsWLkulcREothEs+xA2OiiDhUREnQCQiJA8IEaRKmBiRggdJRZQwkczGieRPoDRDwPjFVGjUYjI+JHMBIhEhaUeIIJVsB4kUPUok82EyXVpCpRlCJjibkTFdmOioMBEfFUxB8oUQQWqFjZGKpKOkmu1AqZaVWInC18BJMiaCihIdFUHiQyJA0BghglSLGiNStCCpZipOpGQDZbo8B0seJREekvn4kAiQrCJEkAlxgqQibphIZuOkwmWkVCNY0iVOcFSzER9SuACRiJAsI0SQKSaCRDITJZKdMKnmS6TUQ7zYZSo2KsJER4XN+JAIkDwgRJBJvgVJNdtxUs33UIkqa4FjOiiCihIeFWECRGICgvoIEWSWqRipsBElFUnGST1ZjRbEC45qYeNDYgqC5ggRZJ7pIKmwGSYVPgRKI8SLP0zFRrUo4SFFiw+JAMkrQgS5YStIKpIIk+l8D5UwiJrL2YiLeqJGR0XU+JAIkLwjRJA7toOkwkWY1JOlYIEZccNDIj5gRpgQuSKhYwKsqjwA2g6SRg/SSUdKlD86xEu6mQiN6eKERwUBgjgIEWTK9AfEpCYlkl+RUo+pP2QEjVk2AqMeE+FRQYDABEIEmVbrgTLJOKkI+uDvS7A0k+QfTp+jJ8n/D2GZDI5qxAdMI0SQOy6nJs1E+eORlniJyuc/9q7Zio1qhAdsI0SQez6HSRBB/hhlPVbyIInoqEaAICmECDBN9QNw2qKknqwtDeVB0uFRQYAgaYQI0EAWo6SRqH/8CJhwXEVGIwQIXCFEgIB82fjqI9N/WNMSNj4GRRjEB3xAiAAx1HsgJ1DiSfsfeF8RHvARIQJYQKDAB4QH0oAQARIU5A8DsYIoiA6kFSECeCbqHxQCJtsIDWQVIQJkhOk/VISNXYQF8C+ECICabP2hzFrgEBRAPIQIgETxhxtAtRmuDwAAAOQXIQIAAJwhRAAAgDOECAAAcIYQAQAAzhAiAADAGUIEAAA4Q4gAAABnCBEAAOAMIQIAAJwhRAAAgDOECAAAcIYQAQAAzhAiAADAGUIEAAA4Q4gAAABnCBEAAOAMIQIAAJwhRAAAgDOECAAAcIYQAQAAzhAiAADAGUIEAAA4Q4gAAABnCBEAAOAMIQIAAJwhRAAAgDOECAAAcIYQAQAAzhAiAADAGUIEAAA4k0iIlEolLVu2TIVCQWfOnEniKgEAQAokEiL333+/Fi5cmMRVAQCAFLEeIi+99JJefvllPfLII7avCgAApMwVNn/4xx9/rPvuu0/Hjh3T7Nmzm16+VCqpVCpNfl4sFm0eHgAAcMzaRKRcLmvdunXatGmTli9fHuh7BgcH1dHRMfnR1dVl6/AAAIAHQofIrl27VCgUGn688cYb2rdvn4rFogYGBgL/7IGBAY2Pj09+jIyMhD08AACQIoVyuVwO8w1jY2MaGxtreJnFixfr3nvv1QsvvKBCoTD59YmJCbW0tGjt2rV6+umnm15XsVhUR0eHnjz5Z82e0xbmMAEAgCP/vHBeG1fcoPHxcbW3tze8bOg9IvPmzdO8efOaXu5HP/qRHn744cnPR0dHtXr1ah09elS9vb1hrxYAAGSQtc2qixYtmvL5nDlzJElLlixRZ2enrasFAAApwplVAQCAM1Zfvltt8eLFCrkdBQAAZBwTEQAA4AwhAgAAnCFEAACAM4QIAABwhhABAADOECIAAMAZQgQAADhDiAAAAGcIEQAA4AwhAgAAnCFEAACAM4QIAABwhhABAADOECIAAMAZQgQAADhDiAAAAGcIEQAA4AwhAgAAnCFEAACAM4QIAABwhhABAADOECIAAMAZQgQAADhDiAAAAGcIEQAA4AwhAgAAnCFEAACAM4QIAABwhhABAADOECIAAMCZK1wfQCPlclmS9PlnFxwfCQAACKryd7vyd7yRQjnIpRz58MMP1dXV5fowAABABCMjI+rs7Gx4Ga9D5NKlSxodHVVbW5sKhYLrw4mlWCyqq6tLIyMjam9vd304+H+4XfzE7eIfbhM/+Xq7lMtlnT9/XgsXLtSMGY13gXi9NDNjxoymJZU27e3tXv2y4F+4XfzE7eIfbhM/+Xi7dHR0BLocm1UBAIAzhAgAAHCGEElIa2urdu7cqdbWVteHgircLn7idvEPt4mfsnC7eL1ZFQAAZBsTEQAA4AwhAgAAnCFEAACAM4QIAABwhhBxqFQqadmyZSoUCjpz5ozrw8m1s2fPasOGDeru7tasWbO0ZMkS7dy5UxcvXnR9aLmzf/9+dXd3a+bMmerp6dGrr77q+pBybXBwULfccova2to0f/58rVmzRu+++67rw0KVwcFBFQoFbdu2zfWhREKIOHT//fdr4cKFrg8Dkt555x1dunRJTzzxhN5++2099thjevzxx7Vjxw7Xh5YrR48e1bZt2/Tggw/q9OnTuu2223TnnXdqeHjY9aHl1okTJ9Tf36/XX39dx48f1xdffKG+vj599tlnrg8Nkk6dOqWDBw/qpptucn0okfHyXUdeeuklbd++Xc8//7xuvPFGnT59WsuWLXN9WKiyZ88eHThwQB988IHrQ8mN3t5e3XzzzTpw4MDk166//nqtWbNGg4ODDo8MFZ9++qnmz5+vEydOaMWKFa4PJ9cuXLigm2++Wfv379fDDz+sZcuWae/eva4PKzQmIg58/PHHuu+++/Szn/1Ms2fPdn04qGN8fFxz5851fRi5cfHiRQ0NDamvr2/K1/v6+vTaa685OipMNz4+LkncNzzQ39+vu+66S3fccYfrQ4nF6ze9y6Jyuax169Zp06ZNWr58uc6ePev6kFDD+++/r3379umHP/yh60PJjbGxMU1MTGjBggVTvr5gwQKdO3fO0VGhWrlc1vbt23Xrrbdq6dKlrg8n15577jm9+eabOnXqlOtDiY2JiCG7du1SoVBo+PHGG29o3759KhaLGhgYcH3IuRD0dqk2Ojqqb37zm/rOd76jjRs3Ojry/CoUClM+L5fLl30NbmzevFlvvfWWjhw54vpQcm1kZERbt27V4cOHNXPmTNeHExt7RAwZGxvT2NhYw8ssXrxY9957r1544YUpD6wTExNqaWnR2rVr9fTTT9s+1FwJertU7syjo6NatWqVent7dejQIc2YQasn5eLFi5o9e7Z+8Ytf6O677578+tatW3XmzBmdOHHC4dFhy5YtOnbsmE6ePKnu7m7Xh5Nrx44d0913362WlpbJr01MTKhQKGjGjBkqlUpT/s13hEjChoeHVSwWJz8fHR3V6tWr9ctf/lK9vb3q7Ox0eHT59tFHH2nVqlXq6enR4cOHU3VHzore3l719PRo//79k1+74YYb9K1vfYvNqo6Uy2Vt2bJFv/71r/XKK6/oK1/5iutDyr3z58/rr3/965SvrV+/Xtddd50eeOCB1C2bsUckYYsWLZry+Zw5cyRJS5YsIUIcGh0d1cqVK7Vo0SI98sgj+vTTTyf/7ZprrnF4ZPmyfft2ffe739Xy5cv1ta99TQcPHtTw8LA2bdrk+tByq7+/X88++6x+85vfqK2tbXK/TkdHh2bNmuX46PKpra3tsti48sordfXVV6cuQiRCBJAkvfzyy3rvvff03nvvXRaEDA2Tc8899+jvf/+7HnroIf3tb3/T0qVL9eKLL+raa691fWi5VXkp9cqVK6d8/amnntK6deuSPyBkDkszAADAGXbiAQAAZwgRAADgDCECAACcIUQAAIAzhAgAAHCGEAEAAM4QIgAAwBlCBAAAOEOIAAAAZwgRAADgDCECAACcIUQAAIAz/xewTKnkN1LA0wAAAABJRU5ErkJggg==" - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "from matplotlib import cm\n", - "from matplotlib.ticker import LinearLocator\n", - "\n", - "fig, ax = plt.subplots(subplot_kw={\"projection\": \"3d\"})\n", - "\n", - "# Make data.\n", - "X = np.arange(-5, 5, 0.25)\n", - "Y = np.arange(-5, 5, 0.25)\n", - "X, Y = np.meshgrid(X, Y)\n", - "\n", - "f_hat = Model()\n", - "Z = f_hat.predict_mgrid(X, Y)\n", - "\n", - "# Plot the surface.\n", - "surf = ax.plot_surface(X, Y, Z, cmap=cm.coolwarm,\n", - " linewidth=0, antialiased=False)\n", - "\n", - "# Customize the z axis.\n", - "ax.set_zlim(-1.0, 1.0)\n", - "\n", - "plt.show()\n", - "\n", - "plt.contourf(X, Y, Z, cmap='Paired')\n", - "plt.show()" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-10-27T20:23:10.999680Z", - "start_time": "2023-10-27T20:23:10.873572Z" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false - } - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/Notebooks/friedrich/p3/GINN_P3.ipynb b/Notebooks/friedrich/p3/GINN_P3.ipynb index 47f1646..84fe462 100644 --- a/Notebooks/friedrich/p3/GINN_P3.ipynb +++ b/Notebooks/friedrich/p3/GINN_P3.ipynb @@ -1,611 +1,817 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [], - "gpuType": "T4", - "toc_visible": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - }, - "accelerator": "GPU" + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "gpuType": "T4", + "toc_visible": true }, - "cells": [ - { - "cell_type": "markdown", - "source": [ - "#0. Imports und Helper" - ], - "metadata": { - "id": "194EMZeTSLIk" - } - }, - { - "cell_type": "code", - "source": [ - "!pip install torchinfo" - ], - "metadata": { - "id": "6IoTrfAlzktH", - "colab": { - "base_uri": "https://localhost:8080/" - }, - "outputId": "f9bdc25e-4895-436b-91ab-00d3a38af883" - }, - "execution_count": 1, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Collecting torchinfo\n", - " Downloading torchinfo-1.8.0-py3-none-any.whl (23 kB)\n", - "Installing collected packages: torchinfo\n", - "Successfully installed torchinfo-1.8.0\n" - ] - } - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "dPt0DPgfLjEQ" - }, - "outputs": [], - "source": [ - "# Imports\n", - "import copy\n", - "\n", - "import ipywidgets as widgets\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "import time\n", - "import torch\n", - "import torchvision\n", - "#import torchvision.datasets as datasets\n", - "import torch.nn.functional as F\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "import tqdm\n", - "\n", - "import random\n", - "import keras.datasets.imdb\n", - "\n", - "from torch.autograd import Variable\n", - "from tqdm.auto import tqdm as tqdmauto" - ] - }, - { - "cell_type": "code", - "source": [ - "def set_seed(seed=None, seed_torch=True):\n", - " \"\"\"\n", - " Handles variability by controlling sources of randomness\n", - " through set seed values\n", - "\n", - " Args:\n", - " seed: Integer\n", - " Set the seed value to given integer.\n", - " If no seed, set seed value to random integer in the range 2^32\n", - " seed_torch: Bool\n", - " Seeds the random number generator for all devices to\n", - " offer some guarantees on reproducibility\n", - "\n", - " Returns:\n", - " Nothing\n", - " \"\"\"\n", - " if seed is None:\n", - " seed = np.random.choice(2 ** 32)\n", - " random.seed(seed)\n", - " np.random.seed(seed)\n", - " if seed_torch:\n", - " torch.manual_seed(seed)\n", - " torch.cuda.manual_seed_all(seed)\n", - " torch.cuda.manual_seed(seed)\n", - " torch.backends.cudnn.benchmark = False\n", - " torch.backends.cudnn.deterministic = True\n", - " print(f'Random seed {seed} has been set.')\n", - "SEED = 2021\n", - "set_seed(seed=SEED)\n", - "DEVICE = \"cuda\"\n", - "\n", - "def zero_grad(params):\n", - " \"\"\"\n", - " Clear gradients as they accumulate on successive backward calls\n", - "\n", - " Args:\n", - " params: an iterator over tensors\n", - " i.e., updating the Weights and biases\n", - "\n", - " Returns:\n", - " Nothing\n", - " \"\"\"\n", - " for par in params:\n", - " if not(par.grad is None):\n", - " par.grad.data.zero_()\n", - "\n", - "\n", - "def print_params(model):\n", - " \"\"\"\n", - " Lists the name and current value of the model's\n", - " named parameters\n", - "\n", - " Args:\n", - " model: an nn.Module inherited model\n", - " Represents the ML/DL model\n", - "\n", - " Returns:\n", - " Nothing\n", - " \"\"\"\n", - " for name, param in model.named_parameters():\n", - " if param.requires_grad:\n", - " print(name, param.data)\n", - "\n", - "def sample_minibatch(input_data, target_data, num_points=100):\n", - " \"\"\"\n", - " Sample a minibatch of size num_point from the provided input-target data\n", - "\n", - " Args:\n", - " input_data: Tensor\n", - " Multi-dimensional tensor containing the input data\n", - " target_data: Tensor\n", - " 1D tensor containing the class labels\n", - " num_points: Integer\n", - " Number of elements to be included in minibatch with default=100\n", - "\n", - " Returns:\n", - " batch_inputs: Tensor\n", - " Minibatch inputs\n", - " batch_targets: Tensor\n", - " Minibatch targets\n", - " \"\"\"\n", - " # Sample a collection of IID indices from the existing data\n", - " batch_indices = np.random.choice(len(input_data), num_points)\n", - " # Use batch_indices to extract entries from the input and target data tensors\n", - " batch_inputs = input_data[batch_indices, :]\n", - " batch_targets = target_data[batch_indices]\n", - "\n", - " return batch_inputs, batch_targets\n", - "\n", - "\n", - "def gradient_update(loss, params, lr=1e-3):\n", - " \"\"\"\n", - " Perform a gradient descent update on a given loss over a collection of parameters\n", - "\n", - " Args:\n", - " loss: Tensor\n", - " A scalar tensor containing the loss through which the gradient will be computed\n", - " params: List of iterables\n", - " Collection of parameters with respect to which we compute gradients\n", - " lr: Float\n", - " Scalar specifying the learning rate or step-size for the update\n", - "\n", - " Returns:\n", - " Nothing\n", - " \"\"\"\n", - " # Clear up gradients as Pytorch automatically accumulates gradients from\n", - " # successive backward calls\n", - " zero_grad(params)\n", - "\n", - " # Compute gradients on given objective\n", - " loss.backward()\n", - "\n", - " with torch.no_grad():\n", - " for par in params:\n", - " # Here we work with the 'data' attribute of the parameter rather than the\n", - " # parameter itself.\n", - " # Hence - use the learning rate and the parameter's .grad.data attribute to perform an update\n", - " par.data -= lr * par.grad.data" - ], - "metadata": { - "id": "U6niQp1RNHxp", - "colab": { - "base_uri": "https://localhost:8080/" - }, - "outputId": "f07ba931-b6c5-4157-df21-91a429ff70a1" - }, - "execution_count": 3, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Random seed 2021 has been set.\n" - ] - } - ] - }, - { - "cell_type": "code", - "source": [ - "(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=10000, maxlen=250,)\n", - "def vectorize_sequences(sequences, dimension=10000):\n", - " # all zero matrix of shape (len(sequences), dimension)\n", - " result = np.zeros((len(sequences), dimension))\n", - " for i,sequence in enumerate(sequences):\n", - " result[i, sequence] = 1\n", - " return result\n", - "\n", - "\n", - "x_train = vectorize_sequences(x_train)\n", - "x_test = vectorize_sequences(x_test)\n", - "#x_train = np.expand_dims(x_train, -1)\n", - "#x_test = np.expand_dims(x_test, -1)\n", - "\n", - "x_train = Variable(torch.from_numpy(x_train)).float().to(DEVICE)\n", - "y_train = Variable(torch.from_numpy(y_train)).long().to(DEVICE)\n", - "x_test = Variable(torch.from_numpy(x_test)).float().to(DEVICE)\n", - "y_test = Variable(torch.from_numpy(y_test)).long().to(DEVICE)\n", - "\n", - "print(\"x_train shape:\", x_train.shape)\n", - "print(\"y_train shape:\", y_train.shape)\n", - "print(x_train.shape[0], \"train samples\")\n", - "print(x_test.shape[0], \"test samples\")" - ], - "metadata": { - "id": "tNDNF10dyqUm", - "colab": { - "base_uri": "https://localhost:8080/" - }, - "outputId": "f654597c-411a-45ac-f694-aba79e025aa8" - }, - "execution_count": 4, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/imdb.npz\n", - "17464789/17464789 [==============================] - 0s 0us/step\n", - "x_train shape: torch.Size([17121, 10000])\n", - "y_train shape: torch.Size([17121])\n", - "17121 train samples\n", - "17588 test samples\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "#1. Softmax Implementieren\n", - "Implementieren Sie die Softmax Funktion mit Numpy und stellen Sie zunächst sicher, dass diese die selben Ergebnisse liefert wie die Pytorch-Funktion im Beispiel. Vergleichen Sie dann Ihre Implementierungen mit anderen Gruppen und diskutieren Sie auch über Performance und numerische Stabilität. Erstellen Sie ein kleines Benchmark, um Performance und numerische Stabilität zu testen." - ], - "metadata": { - "id": "tKJZz5YsSSyT" - } - }, - { - "cell_type": "code", - "source": [ - "#Ihr Code hier" - ], - "metadata": { - "id": "_80I03V8ogds" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "#%%timeit\n", - "#Ihre Benchmarks hier\n" - ], - "metadata": { - "id": "x0VacAxQu5JS" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "#2. Regularisierung Implementieren\n", - "\n", - "Unten finden Sie einen Pytorch-SGD Schritt mit eingebauter L2-Regularisierung und ohne. Interpretieren Sie die unterschiedlichen Ausgaben. Modifizieren Sie den ersten Codabschnitt mit einer eigenen L2-Regularisierung so, dass identische Ergebnisse erzeugt werden. Sie können dazu die noch nicht verwendete und noch falsch definierte Variable \"regtermwrong\" umdefinieren und zu einem späteren Zeitpunkt im Code darauf zurückgreifen. ACHTUNG: weight_decay*2=lambda." - ], - "metadata": { - "id": "c0u-OqU8U-sL" - } - }, - { - "cell_type": "code", - "source": [ - "#Datendefinition\n", - "np.random.seed(123)\n", - "np.set_printoptions(8, suppress=True)\n", - "\n", - "x_numpy = np.random.random((3, 4)).astype(np.double)\n", - "w_numpy = np.random.random((4, 5)).astype(np.double)\n", - "w_numpy[0,0] =9.9\n", - "x_torch = torch.tensor(x_numpy, requires_grad=True)\n" - ], - "metadata": { - "id": "O0Hn585ETRWD" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "# ohne Regularisierung\n", - "w_torch = torch.tensor(w_numpy, requires_grad=True)\n", - "print('Original weights', w_torch)\n", - "\n", - "lr = 0.1\n", - "sgd = torch.optim.SGD([w_torch], lr=lr, weight_decay=0)\n", - "regtermwrong = max(p.max() for p in w_torch)\n", - "y_torch = torch.matmul(x_torch, w_torch)\n", - "loss = y_torch.sum() +regtermwrong #\n", - "\n", - "\n", - "sgd.zero_grad()\n", - "loss.backward()\n", - "sgd.step()\n", - "\n", - "w_grad = w_torch.grad.data.numpy()\n", - "print('0 weight decay', w_torch)\n" - ], - "metadata": { - "id": "S5XEpjWFTTzi" - }, - "execution_count": null, - "outputs": [] + "kernelspec": { + "name": "python3", + "language": "python", + "display_name": "Python 3 (ipykernel)" + }, + "language_info": { + "name": "python" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# 0. Imports und Helper" + ], + "metadata": { + "id": "194EMZeTSLIk" + } + }, + { + "cell_type": "code", + "source": [ + "!pip install torchinfo" + ], + "metadata": { + "id": "6IoTrfAlzktH", + "colab": { + "base_uri": "https://localhost:8080/" }, + "outputId": "f9bdc25e-4895-436b-91ab-00d3a38af883", + "ExecuteTime": { + "end_time": "2023-11-20T22:19:19.732145Z", + "start_time": "2023-11-20T22:19:18.131897Z" + } + }, + "execution_count": 1, + "outputs": [ { - "cell_type": "code", - "source": [ - "#mit Regularisierung\n", - "\n", - "\n", - "w_torch = torch.tensor(w_numpy, requires_grad=True)\n", - "\n", - "print('Reset Original weights', w_torch)\n", - "\n", - "sgd = torch.optim.SGD([w_torch], lr=lr, weight_decay=2)\n", - "\n", - "y_torch = torch.matmul(x_torch, w_torch)\n", - "loss = y_torch.sum()\n", - "\n", - "sgd.zero_grad()\n", - "loss.backward()\n", - "sgd.step()\n", - "\n", - "w_grad = w_torch.grad.data.numpy()\n", - "print('1 weight decay', w_torch)" - ], - "metadata": { - "id": "WiQW-Y4VkH7v" - }, - "execution_count": null, - "outputs": [] + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting torchinfo\r\n", + " Downloading torchinfo-1.8.0-py3-none-any.whl.metadata (21 kB)\r\n", + "Downloading torchinfo-1.8.0-py3-none-any.whl (23 kB)\r\n", + "Installing collected packages: torchinfo\r\n", + "Successfully installed torchinfo-1.8.0\r\n" + ] + } + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "dPt0DPgfLjEQ", + "ExecuteTime": { + "end_time": "2023-11-20T22:29:17.347227Z", + "start_time": "2023-11-20T22:29:15.452701Z" + } + }, + "outputs": [], + "source": [ + "# Imports\n", + "import copy\n", + "\n", + "import ipywidgets as widgets\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "import time\n", + "import torch\n", + "import torchvision\n", + "#import torchvision.datasets as datasets\n", + "import torch.nn.functional as F\n", + "import torch.nn as nn\n", + "import torch.optim as optim\n", + "import tqdm\n", + "\n", + "import random\n", + "import keras.datasets.imdb\n", + "\n", + "from torch.autograd import Variable\n", + "from tqdm.auto import tqdm as tqdmauto" + ] + }, + { + "cell_type": "code", + "source": [ + "def set_seed(seed=None, seed_torch=True):\n", + " \"\"\"\n", + " Handles variability by controlling sources of randomness\n", + " through set seed values\n", + "\n", + " Args:\n", + " seed: Integer\n", + " Set the seed value to given integer.\n", + " If no seed, set seed value to random integer in the range 2^32\n", + " seed_torch: Bool\n", + " Seeds the random number generator for all devices to\n", + " offer some guarantees on reproducibility\n", + "\n", + " Returns:\n", + " Nothing\n", + " \"\"\"\n", + " if seed is None:\n", + " seed = np.random.choice(2 ** 32)\n", + " random.seed(seed)\n", + " np.random.seed(seed)\n", + " if seed_torch:\n", + " torch.manual_seed(seed)\n", + " torch.cuda.manual_seed_all(seed)\n", + " torch.cuda.manual_seed(seed)\n", + " torch.backends.cudnn.benchmark = False\n", + " torch.backends.cudnn.deterministic = True\n", + " print(f'Random seed {seed} has been set.')\n", + "SEED = 2021\n", + "set_seed(seed=SEED)\n", + "DEVICE = \"mps\"\n", + "\n", + "def zero_grad(params):\n", + " \"\"\"\n", + " Clear gradients as they accumulate on successive backward calls\n", + "\n", + " Args:\n", + " params: an iterator over tensors\n", + " i.e., updating the Weights and biases\n", + "\n", + " Returns:\n", + " Nothing\n", + " \"\"\"\n", + " for par in params:\n", + " if not(par.grad is None):\n", + " par.grad.data.zero_()\n", + "\n", + "\n", + "def print_params(model):\n", + " \"\"\"\n", + " Lists the name and current value of the model's\n", + " named parameters\n", + "\n", + " Args:\n", + " model: an nn.Module inherited model\n", + " Represents the ML/DL model\n", + "\n", + " Returns:\n", + " Nothing\n", + " \"\"\"\n", + " for name, param in model.named_parameters():\n", + " if param.requires_grad:\n", + " print(name, param.data)\n", + "\n", + "def sample_minibatch(input_data, target_data, num_points=100):\n", + " \"\"\"\n", + " Sample a minibatch of size num_point from the provided input-target data\n", + "\n", + " Args:\n", + " input_data: Tensor\n", + " Multi-dimensional tensor containing the input data\n", + " target_data: Tensor\n", + " 1D tensor containing the class labels\n", + " num_points: Integer\n", + " Number of elements to be included in minibatch with default=100\n", + "\n", + " Returns:\n", + " batch_inputs: Tensor\n", + " Minibatch inputs\n", + " batch_targets: Tensor\n", + " Minibatch targets\n", + " \"\"\"\n", + " # Sample a collection of IID indices from the existing data\n", + " batch_indices = np.random.choice(len(input_data), num_points)\n", + " # Use batch_indices to extract entries from the input and target data tensors\n", + " batch_inputs = input_data[batch_indices, :]\n", + " batch_targets = target_data[batch_indices]\n", + "\n", + " return batch_inputs, batch_targets\n", + "\n", + "\n", + "def gradient_update(loss, params, lr=1e-3):\n", + " \"\"\"\n", + " Perform a gradient descent update on a given loss over a collection of parameters\n", + "\n", + " Args:\n", + " loss: Tensor\n", + " A scalar tensor containing the loss through which the gradient will be computed\n", + " params: List of iterables\n", + " Collection of parameters with respect to which we compute gradients\n", + " lr: Float\n", + " Scalar specifying the learning rate or step-size for the update\n", + "\n", + " Returns:\n", + " Nothing\n", + " \"\"\"\n", + " # Clear up gradients as Pytorch automatically accumulates gradients from\n", + " # successive backward calls\n", + " zero_grad(params)\n", + "\n", + " # Compute gradients on given objective\n", + " loss.backward()\n", + "\n", + " with torch.no_grad():\n", + " for par in params:\n", + " # Here we work with the 'data' attribute of the parameter rather than the\n", + " # parameter itself.\n", + " # Hence - use the learning rate and the parameter's .grad.data attribute to perform an update\n", + " par.data -= lr * par.grad.data" + ], + "metadata": { + "id": "U6niQp1RNHxp", + "colab": { + "base_uri": "https://localhost:8080/" }, + "outputId": "f07ba931-b6c5-4157-df21-91a429ff70a1", + "ExecuteTime": { + "end_time": "2023-11-20T22:30:27.238079Z", + "start_time": "2023-11-20T22:30:27.232238Z" + } + }, + "execution_count": 7, + "outputs": [ { - "cell_type": "markdown", - "source": [ - "#3. Einfaches MLP in Pytorch\n", - "Machen Sie sich ein wenig mit dem IMDB Datensatz und den für Sie erstellten Datenstrukturen in x/y_train/test vertraut." - ], - "metadata": { - "id": "sQk-ciLBYGnu" - } + "name": "stdout", + "output_type": "stream", + "text": [ + "Random seed 2021 has been set.\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=10000, maxlen=250,)\n", + "def vectorize_sequences(sequences, dimension=10000):\n", + " # all zero matrix of shape (len(sequences), dimension)\n", + " result = np.zeros((len(sequences), dimension))\n", + " for i,sequence in enumerate(sequences):\n", + " result[i, sequence] = 1\n", + " return result\n", + "\n", + "\n", + "x_train = vectorize_sequences(x_train)\n", + "x_test = vectorize_sequences(x_test)\n", + "#x_train = np.expand_dims(x_train, -1)\n", + "#x_test = np.expand_dims(x_test, -1)\n", + "\n", + "x_train = Variable(torch.from_numpy(x_train)).float().to(DEVICE)\n", + "y_train = Variable(torch.from_numpy(y_train)).long().to(DEVICE)\n", + "x_test = Variable(torch.from_numpy(x_test)).float().to(DEVICE)\n", + "y_test = Variable(torch.from_numpy(y_test)).long().to(DEVICE)\n", + "\n", + "print(\"x_train shape:\", x_train.shape)\n", + "print(\"y_train shape:\", y_train.shape)\n", + "print(x_train.shape[0], \"train samples\")\n", + "print(x_test.shape[0], \"test samples\")" + ], + "metadata": { + "id": "tNDNF10dyqUm", + "colab": { + "base_uri": "https://localhost:8080/" }, + "outputId": "f654597c-411a-45ac-f694-aba79e025aa8", + "ExecuteTime": { + "end_time": "2023-11-20T22:30:31.684372Z", + "start_time": "2023-11-20T22:30:29.176089Z" + } + }, + "execution_count": 8, + "outputs": [ { - "cell_type": "markdown", - "source": [ - "## 3.1 Modell erstellen und Angaben zur Modellgröße verstehen\n", - "Definieren Sie ein Pytorch Multilayer Perzeptron mit der Größe des IMDB-Dictionaries für one-hot-encodierte Wörte als Eingabe (Sigmoid Aktivierung), 50 Neuronen im Hidden Layer und 2 Ausgabeneuronen. Layer 1 und 2 Ihres Netzes verwendet die Sigmoid-Aktivierungsfunktion, Layer 3 die Softmax-Aktivierungsfunktion.\n", - "\n", - "Generieren Sie Modell-Summary mit torchinfo und erklären Sie, was die ausgegebenen Werte bedeuten und wie diese zustande kommen." - ], - "metadata": { - "id": "mLeOKvUxMunF" - } - }, + "name": "stdout", + "output_type": "stream", + "text": [ + "x_train shape: torch.Size([17121, 10000])\n", + "y_train shape: torch.Size([17121])\n", + "17121 train samples\n", + "17588 test samples\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "# 1. Softmax Implementieren\n", + "Implementieren Sie die Softmax Funktion mit Numpy und stellen Sie zunächst sicher, dass diese die selben Ergebnisse liefert wie die Pytorch-Funktion im Beispiel. Vergleichen Sie dann Ihre Implementierungen mit anderen Gruppen und diskutieren Sie auch über Performance und numerische Stabilität. Erstellen Sie ein kleines Benchmark, um Performance und numerische Stabilität zu testen." + ], + "metadata": { + "id": "tKJZz5YsSSyT" + } + }, + { + "cell_type": "markdown", + "source": [], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "source": [ + "# If values get too large this function becomes numerically unstable\n", + "def softmax(x):\n", + " return np.exp(x)/np.exp(x).sum()\n", + "\n", + "def softmax_stable(x):\n", + " return np.exp(x - np.max(x)) / np.exp(x - np.max(x)).sum()" + ], + "metadata": { + "id": "_80I03V8ogds", + "ExecuteTime": { + "end_time": "2023-11-20T22:37:21.783849Z", + "start_time": "2023-11-20T22:37:21.779265Z" + } + }, + "execution_count": 9, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "array_small = np.array([2, 11, 7])\n", + "array_large = np.array([555, 999, 111])\n", + "\n", + "print(softmax(array_small).round(3))\n", + "print(softmax(array_large))\n", + "print(softmax_stable(array_large).round(2))" + ], + "metadata": { + "id": "x0VacAxQu5JS", + "ExecuteTime": { + "end_time": "2023-11-20T22:38:06.327953Z", + "start_time": "2023-11-20T22:38:06.325348Z" + } + }, + "execution_count": 15, + "outputs": [ { - "cell_type": "code", - "source": [ - "#Ihr Code hier\n", - "class Model(nn.Module):\n", - " pass\n" - ], - "metadata": { - "id": "TikQht7LmNnc" - }, - "execution_count": null, - "outputs": [] + "name": "stdout", + "output_type": "stream", + "text": [ + "[0. 0.982 0.018]\n", + "[ 0. nan 0.]\n", + "[0. 1. 0.]\n" + ] }, { - "cell_type": "markdown", - "source": [ - "## 3.2 Modell trainieren und Performancekurven interpretieren\n", - "Nutzen Sie den untenstehenedn Code um Ihr Modell zu trainieren. Interpretieren und diskutieren Sie die entstehenden Performancekurven. Falls Sie einen unerwarteten Anstieg Ihres Losses beobachten, recherchieren Sie wie Sie diese mit dem Einbau einer einzelnen Verbesserung innerhalb des gegebenen SGD Lernverfahrens beheben können. ACHTUNG: Wenn Sie Ihr Modell nicht oben neu initialisieren, optimieren Sie weiter auf den schon veränderten Parametern." - ], - "metadata": { - "id": "GUoAJo8WMzfb" - } - }, + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/_v/gtvs0k6x72bf23jbq0yd1wh40000gn/T/ipykernel_78955/2076069751.py:3: RuntimeWarning: overflow encountered in exp\n", + " return np.exp(x)/np.exp(x).sum()\n", + "/var/folders/_v/gtvs0k6x72bf23jbq0yd1wh40000gn/T/ipykernel_78955/2076069751.py:3: RuntimeWarning: invalid value encountered in divide\n", + " return np.exp(x)/np.exp(x).sum()\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "# 2. Regularisierung Implementieren\n", + "\n", + "Unten finden Sie einen Pytorch-SGD Schritt mit eingebauter L2-Regularisierung und ohne. Interpretieren Sie die unterschiedlichen Ausgaben. Modifizieren Sie den ersten Codabschnitt mit einer eigenen L2-Regularisierung so, dass identische Ergebnisse erzeugt werden. Sie können dazu die noch nicht verwendete und noch falsch definierte Variable \"regtermwrong\" umdefinieren und zu einem späteren Zeitpunkt im Code darauf zurückgreifen. ACHTUNG: weight_decay*2=lambda." + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "source": [ + "#Datendefinition\n", + "np.random.seed(123)\n", + "np.set_printoptions(8, suppress=True)\n", + "\n", + "x_numpy = np.random.random((3, 4)).astype(np.double)\n", + "w_numpy = np.random.random((4, 5)).astype(np.double)\n", + "w_numpy[0,0] =9.9\n", + "x_torch = torch.tensor(x_numpy, requires_grad=True)\n" + ], + "metadata": { + "id": "S5XEpjWFTTzi", + "ExecuteTime": { + "end_time": "2023-11-20T22:39:36.082809Z", + "start_time": "2023-11-20T22:39:36.080252Z" + } + }, + "execution_count": 16, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# mit Regularisierung\n", + "w_torch = torch.tensor(w_numpy, requires_grad=True)\n", + "print('Original weights', w_torch)\n", + "\n", + "lr = 0.1\n", + "sgd = torch.optim.SGD([w_torch], lr=lr, weight_decay=0)\n", + "#regtermwrong = max(p.max() for p in w_torch)\n", + "regterm = sum(sum(p) for p in w_torch)\n", + "y_torch = torch.matmul(x_torch, w_torch)\n", + "loss = y_torch.sum() + 4 * regterm\n", + "\n", + "\n", + "sgd.zero_grad()\n", + "loss.backward()\n", + "sgd.step()\n", + "\n", + "w_grad = w_torch.grad.data.numpy()\n", + "print('0 weight decay', w_torch)\n" + ], + "metadata": { + "id": "WiQW-Y4VkH7v", + "ExecuteTime": { + "end_time": "2023-11-20T22:39:38.992708Z", + "start_time": "2023-11-20T22:39:38.699008Z" + } + }, + "execution_count": 17, + "outputs": [ { - "cell_type": "code", - "source": [ - "EPOCHS = 291 #@param {type:\"slider\", min:2, max:1000, step:1}\n", - "RATE = 0.902 #@param {type:\"slider\", min:0.001, max:2, step:0.001}\n", - "optimizer = torch.optim.SGD(model.parameters(), lr=RATE, weight_decay=0)\n", - "loss_fn = nn.CrossEntropyLoss()\n", - "loss_list = np.zeros((EPOCHS,))\n", - "accuracy_list = np.zeros((EPOCHS,))\n", - "accuracy_list_test = np.zeros((EPOCHS,))\n", - "\n", - "\n", - "for epoch in tqdm.trange(EPOCHS):\n", - " y_pred = model(x_train)\n", - " #loss = loss_fn(y_pred, y_train)\n", - " loss_list[epoch] = loss.item()\n", - " loss = loss_fn(y_pred, y_train)# + 0.01 *l2_reg(model)\n", - "\n", - " # Zero gradients\n", - " optimizer.zero_grad()\n", - "\n", - " loss.backward()\n", - " #torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)#, args.clip)\n", - " optimizer.step()\n", - "\n", - " with torch.no_grad():\n", - " y_pred = model(x_train)\n", - " correct = (torch.argmax(y_pred, dim=1) == y_train).type(torch.FloatTensor)\n", - " accuracy_list[epoch] = correct.mean()\n", - " y_pred = model(x_test)\n", - " correct = (torch.argmax(y_pred, dim=1) == y_test).type(torch.FloatTensor)\n", - " accuracy_list_test[epoch] = correct.mean()\n", - "\n", - "\n", - "\n", - "\n", - "fig, (ax1, ax2, ax3) = plt.subplots(3, figsize=(12, 6), sharex=True)\n", - "\n", - "ax1.plot(accuracy_list)\n", - "ax1.set_ylabel(\"train accuracy\")\n", - "ax2.plot(loss_list)\n", - "ax2.set_ylabel(\"train loss\")\n", - "ax3.plot(accuracy_list_test)\n", - "ax3.set_ylabel(\"test acc\")\n", - "ax3.set_xlabel(\"epochs\");" - ], - "metadata": { - "id": "JD2AkGHGrpGV" - }, - "execution_count": null, - "outputs": [] + "name": "stdout", + "output_type": "stream", + "text": [ + "Original weights tensor([[9.9000, 0.0597, 0.3980, 0.7380, 0.1825],\n", + " [0.1755, 0.5316, 0.5318, 0.6344, 0.8494],\n", + " [0.7245, 0.6110, 0.7224, 0.3230, 0.3618],\n", + " [0.2283, 0.2937, 0.6310, 0.0921, 0.4337]], dtype=torch.float64,\n", + " requires_grad=True)\n" + ] }, { - "cell_type": "markdown", - "source": [ - "##3.3. Momentum Implementieren\n", - "Vervollständigen Sie Methode momentum_update. Überlegen Sie sich, wie Sie die Korrektheit mit einem Durchlauf inkl. Momentum Update auf Ihrem oben definierten Modell prüfen können" - ], - "metadata": { - "id": "F3kyvbJaTjZ7" - } - }, + "ename": "TypeError", + "evalue": "cannot inherit non-frozen dataclass from a frozen one", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mTypeError\u001B[0m Traceback (most recent call last)", + "Cell \u001B[0;32mIn[17], line 6\u001B[0m\n\u001B[1;32m 3\u001B[0m \u001B[38;5;28mprint\u001B[39m(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mOriginal weights\u001B[39m\u001B[38;5;124m'\u001B[39m, w_torch)\n\u001B[1;32m 5\u001B[0m lr \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m0.1\u001B[39m\n\u001B[0;32m----> 6\u001B[0m sgd \u001B[38;5;241m=\u001B[39m \u001B[43mtorch\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43moptim\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mSGD\u001B[49m\u001B[43m(\u001B[49m\u001B[43m[\u001B[49m\u001B[43mw_torch\u001B[49m\u001B[43m]\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mlr\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mlr\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mweight_decay\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;241;43m0\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[1;32m 7\u001B[0m regtermwrong \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mmax\u001B[39m(p\u001B[38;5;241m.\u001B[39mmax() \u001B[38;5;28;01mfor\u001B[39;00m p \u001B[38;5;129;01min\u001B[39;00m w_torch)\n\u001B[1;32m 8\u001B[0m y_torch \u001B[38;5;241m=\u001B[39m torch\u001B[38;5;241m.\u001B[39mmatmul(x_torch, w_torch)\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/optim/sgd.py:26\u001B[0m, in \u001B[0;36m__init__\u001B[0;34m(self, params, lr, momentum, dampening, weight_decay, nesterov, maximize, foreach, differentiable)\u001B[0m\n\u001B[1;32m 21\u001B[0m defaults \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mdict\u001B[39m(lr\u001B[38;5;241m=\u001B[39mlr, momentum\u001B[38;5;241m=\u001B[39mmomentum, dampening\u001B[38;5;241m=\u001B[39mdampening,\n\u001B[1;32m 22\u001B[0m weight_decay\u001B[38;5;241m=\u001B[39mweight_decay, nesterov\u001B[38;5;241m=\u001B[39mnesterov,\n\u001B[1;32m 23\u001B[0m maximize\u001B[38;5;241m=\u001B[39mmaximize, foreach\u001B[38;5;241m=\u001B[39mforeach,\n\u001B[1;32m 24\u001B[0m differentiable\u001B[38;5;241m=\u001B[39mdifferentiable)\n\u001B[1;32m 25\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m nesterov \u001B[38;5;129;01mand\u001B[39;00m (momentum \u001B[38;5;241m<\u001B[39m\u001B[38;5;241m=\u001B[39m \u001B[38;5;241m0\u001B[39m \u001B[38;5;129;01mor\u001B[39;00m dampening \u001B[38;5;241m!=\u001B[39m \u001B[38;5;241m0\u001B[39m):\n\u001B[0;32m---> 26\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mNesterov momentum requires a momentum and zero dampening\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m 27\u001B[0m \u001B[38;5;28msuper\u001B[39m()\u001B[38;5;241m.\u001B[39m\u001B[38;5;21m__init__\u001B[39m(params, defaults)\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/optim/optimizer.py:266\u001B[0m, in \u001B[0;36m__init__\u001B[0;34m(self, params, defaults)\u001B[0m\n\u001B[1;32m 262\u001B[0m \u001B[38;5;129m@staticmethod\u001B[39m\n\u001B[1;32m 263\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mprofile_hook_step\u001B[39m(func):\n\u001B[1;32m 265\u001B[0m \u001B[38;5;129m@functools\u001B[39m\u001B[38;5;241m.\u001B[39mwraps(func)\n\u001B[0;32m--> 266\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mwrapper\u001B[39m(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs):\n\u001B[1;32m 267\u001B[0m \u001B[38;5;28mself\u001B[39m, \u001B[38;5;241m*\u001B[39m_ \u001B[38;5;241m=\u001B[39m args\n\u001B[1;32m 268\u001B[0m profile_name \u001B[38;5;241m=\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mOptimizer.step#\u001B[39m\u001B[38;5;132;01m{}\u001B[39;00m\u001B[38;5;124m.step\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;241m.\u001B[39mformat(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m\u001B[38;5;18m__class__\u001B[39m\u001B[38;5;241m.\u001B[39m\u001B[38;5;18m__name__\u001B[39m)\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_compile.py:22\u001B[0m, in \u001B[0;36minner\u001B[0;34m(*args, **kwargs)\u001B[0m\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_dynamo/__init__.py:1\u001B[0m\n\u001B[0;32m----> 1\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m allowed_functions, convert_frame, eval_frame, resume_execution\n\u001B[1;32m 2\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mbackends\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mregistry\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m list_backends, register_backend\n\u001B[1;32m 3\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mconvert_frame\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m replay\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py:29\u001B[0m\n\u001B[1;32m 27\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mguards\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m CheckFunctionManager, GuardedCode\n\u001B[1;32m 28\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mhooks\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Hooks\n\u001B[0;32m---> 29\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01moutput_graph\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m OutputGraph\n\u001B[1;32m 30\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mreplay_record\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m ExecutionRecord\n\u001B[1;32m 31\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01msymbolic_convert\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m InstructionTranslator\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_dynamo/output_graph.py:23\u001B[0m\n\u001B[1;32m 14\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtorch\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m_guards\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m (\n\u001B[1;32m 15\u001B[0m Checkpointable,\n\u001B[1;32m 16\u001B[0m Guard,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 19\u001B[0m TracingContext,\n\u001B[1;32m 20\u001B[0m )\n\u001B[1;32m 21\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtorch\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mfx\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mexperimental\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01msymbolic_shapes\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m ShapeEnv\n\u001B[0;32m---> 23\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m config, logging \u001B[38;5;28;01mas\u001B[39;00m torchdynamo_logging, variables\n\u001B[1;32m 24\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mbackends\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mregistry\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m CompiledFn, CompilerFn\n\u001B[1;32m 25\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mbytecode_transformation\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m create_instruction, Instruction, unique_id\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_dynamo/variables/__init__.py:1\u001B[0m\n\u001B[0;32m----> 1\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mbase\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m VariableTracker\n\u001B[1;32m 2\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mbuiltin\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m BuiltinVariable\n\u001B[1;32m 3\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mconstant\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m ConstantVariable, EnumVariable\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_dynamo/variables/base.py:6\u001B[0m\n\u001B[1;32m 4\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m variables\n\u001B[1;32m 5\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mexc\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m unimplemented\n\u001B[0;32m----> 6\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01msource\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m AttrSource, Source\n\u001B[1;32m 7\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mutils\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m dict_values, identity, istype, odict_values\n\u001B[1;32m 10\u001B[0m \u001B[38;5;28;01mclass\u001B[39;00m \u001B[38;5;21;01mMutableLocal\u001B[39;00m:\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_dynamo/source.py:49\u001B[0m\n\u001B[1;32m 39\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mis_input_source\u001B[39m(source):\n\u001B[1;32m 40\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m source\u001B[38;5;241m.\u001B[39mguard_source() \u001B[38;5;129;01min\u001B[39;00m [\n\u001B[1;32m 41\u001B[0m GuardSource\u001B[38;5;241m.\u001B[39mLOCAL,\n\u001B[1;32m 42\u001B[0m GuardSource\u001B[38;5;241m.\u001B[39mGLOBAL,\n\u001B[1;32m 43\u001B[0m GuardSource\u001B[38;5;241m.\u001B[39mLOCAL_NN_MODULE,\n\u001B[1;32m 44\u001B[0m GuardSource\u001B[38;5;241m.\u001B[39mGLOBAL_NN_MODULE,\n\u001B[1;32m 45\u001B[0m ]\n\u001B[1;32m 48\u001B[0m \u001B[38;5;129;43m@dataclasses\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdataclass\u001B[49m\n\u001B[0;32m---> 49\u001B[0m \u001B[38;5;28;43;01mclass\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[38;5;21;43;01mLocalSource\u001B[39;49;00m\u001B[43m(\u001B[49m\u001B[43mSource\u001B[49m\u001B[43m)\u001B[49m\u001B[43m:\u001B[49m\n\u001B[1;32m 50\u001B[0m \u001B[43m \u001B[49m\u001B[43mlocal_name\u001B[49m\u001B[43m:\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mstr\u001B[39;49m\n\u001B[1;32m 52\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;28;43;01mdef\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[38;5;21;43mreconstruct\u001B[39;49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcodegen\u001B[49m\u001B[43m)\u001B[49m\u001B[43m:\u001B[49m\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/dataclasses.py:1184\u001B[0m, in \u001B[0;36mdataclass\u001B[0;34m(cls, init, repr, eq, order, unsafe_hash, frozen, match_args, kw_only, slots)\u001B[0m\n\u001B[1;32m 1181\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m wrap\n\u001B[1;32m 1183\u001B[0m \u001B[38;5;66;03m# We're called as @dataclass without parens.\u001B[39;00m\n\u001B[0;32m-> 1184\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mwrap\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mcls\u001B[39;49m\u001B[43m)\u001B[49m\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/dataclasses.py:1175\u001B[0m, in \u001B[0;36mdataclass.<locals>.wrap\u001B[0;34m(cls)\u001B[0m\n\u001B[1;32m 1174\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mwrap\u001B[39m(\u001B[38;5;28mcls\u001B[39m):\n\u001B[0;32m-> 1175\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43m_process_class\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mcls\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43minit\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mrepr\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43meq\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43morder\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43munsafe_hash\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 1176\u001B[0m \u001B[43m \u001B[49m\u001B[43mfrozen\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmatch_args\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mkw_only\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mslots\u001B[49m\u001B[43m)\u001B[49m\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/dataclasses.py:985\u001B[0m, in \u001B[0;36m_process_class\u001B[0;34m(cls, init, repr, eq, order, unsafe_hash, frozen, match_args, kw_only, slots)\u001B[0m\n\u001B[1;32m 982\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m has_dataclass_bases:\n\u001B[1;32m 983\u001B[0m \u001B[38;5;66;03m# Raise an exception if any of our bases are frozen, but we're not.\u001B[39;00m\n\u001B[1;32m 984\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m any_frozen_base \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m frozen:\n\u001B[0;32m--> 985\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mTypeError\u001B[39;00m(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mcannot inherit non-frozen dataclass from a \u001B[39m\u001B[38;5;124m'\u001B[39m\n\u001B[1;32m 986\u001B[0m \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mfrozen one\u001B[39m\u001B[38;5;124m'\u001B[39m)\n\u001B[1;32m 988\u001B[0m \u001B[38;5;66;03m# Raise an exception if we're frozen, but none of our bases are.\u001B[39;00m\n\u001B[1;32m 989\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m any_frozen_base \u001B[38;5;129;01mand\u001B[39;00m frozen:\n", + "\u001B[0;31mTypeError\u001B[0m: cannot inherit non-frozen dataclass from a frozen one" + ] + } + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "outputs": [ { - "cell_type": "code", - "source": [ - "def momentum_update(loss, params, grad_vel, lr=1e-3, beta=0.8):\n", - " # Clear up gradients as Pytorch automatically accumulates gradients from\n", - " # successive backward calls\n", - " zero_grad(params)\n", - " # Compute gradients on given objective\n", - " loss.backward()\n", - "\n", - " with torch.no_grad():\n", - " for (par, vel) in zip(params, grad_vel):\n", - " print(\"Ihr Code hier\")\n" - ], - "metadata": { - "id": "36whMhqnMKuc" - }, - "execution_count": null, - "outputs": [] + "name": "stdout", + "output_type": "stream", + "text": [ + "Reset Original weights tensor([[9.9000, 0.0597, 0.3980, 0.7380, 0.1825],\n", + " [0.1755, 0.5316, 0.5318, 0.6344, 0.8494],\n", + " [0.7245, 0.6110, 0.7224, 0.3230, 0.3618],\n", + " [0.2283, 0.2937, 0.6310, 0.0921, 0.4337]], dtype=torch.float64,\n", + " requires_grad=True)\n" + ] }, { - "cell_type": "markdown", - "source": [ - "##3.4 Experimente zum Lernverhalten mit Momentum und Batch Size\n", - "Im folgenden können Sie für ein festgelegtes Zeitbudget schauen, wie sich der Loss Ihres neuronalen Netzes innerhalb dieser Zeit entwickelt.\n", - "Experimentieren Sie zunächst mit den Voreinstellungen mit und ohne Momentum, probieren Sie dann eigene Einstellungen aus. Diskutieren Sie das visualisierte Lernverhalten insbesondere bzgl. unterschiedlicher Batch Sizes." - ], - "metadata": { - "id": "OPLRI_RKn2Mc" - } - }, + "ename": "TypeError", + "evalue": "cannot inherit non-frozen dataclass from a frozen one", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mTypeError\u001B[0m Traceback (most recent call last)", + "Cell \u001B[0;32mIn[18], line 8\u001B[0m\n\u001B[1;32m 4\u001B[0m w_torch \u001B[38;5;241m=\u001B[39m torch\u001B[38;5;241m.\u001B[39mtensor(w_numpy, requires_grad\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m)\n\u001B[1;32m 6\u001B[0m \u001B[38;5;28mprint\u001B[39m(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mReset Original weights\u001B[39m\u001B[38;5;124m'\u001B[39m, w_torch)\n\u001B[0;32m----> 8\u001B[0m sgd \u001B[38;5;241m=\u001B[39m \u001B[43mtorch\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43moptim\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mSGD\u001B[49m\u001B[43m(\u001B[49m\u001B[43m[\u001B[49m\u001B[43mw_torch\u001B[49m\u001B[43m]\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mlr\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mlr\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mweight_decay\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;241;43m2\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[1;32m 10\u001B[0m y_torch \u001B[38;5;241m=\u001B[39m torch\u001B[38;5;241m.\u001B[39mmatmul(x_torch, w_torch)\n\u001B[1;32m 11\u001B[0m loss \u001B[38;5;241m=\u001B[39m y_torch\u001B[38;5;241m.\u001B[39msum()\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/optim/sgd.py:26\u001B[0m, in \u001B[0;36m__init__\u001B[0;34m(self, params, lr, momentum, dampening, weight_decay, nesterov, maximize, foreach, differentiable)\u001B[0m\n\u001B[1;32m 21\u001B[0m defaults \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mdict\u001B[39m(lr\u001B[38;5;241m=\u001B[39mlr, momentum\u001B[38;5;241m=\u001B[39mmomentum, dampening\u001B[38;5;241m=\u001B[39mdampening,\n\u001B[1;32m 22\u001B[0m weight_decay\u001B[38;5;241m=\u001B[39mweight_decay, nesterov\u001B[38;5;241m=\u001B[39mnesterov,\n\u001B[1;32m 23\u001B[0m maximize\u001B[38;5;241m=\u001B[39mmaximize, foreach\u001B[38;5;241m=\u001B[39mforeach,\n\u001B[1;32m 24\u001B[0m differentiable\u001B[38;5;241m=\u001B[39mdifferentiable)\n\u001B[1;32m 25\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m nesterov \u001B[38;5;129;01mand\u001B[39;00m (momentum \u001B[38;5;241m<\u001B[39m\u001B[38;5;241m=\u001B[39m \u001B[38;5;241m0\u001B[39m \u001B[38;5;129;01mor\u001B[39;00m dampening \u001B[38;5;241m!=\u001B[39m \u001B[38;5;241m0\u001B[39m):\n\u001B[0;32m---> 26\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mNesterov momentum requires a momentum and zero dampening\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m 27\u001B[0m \u001B[38;5;28msuper\u001B[39m()\u001B[38;5;241m.\u001B[39m\u001B[38;5;21m__init__\u001B[39m(params, defaults)\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/optim/optimizer.py:266\u001B[0m, in \u001B[0;36m__init__\u001B[0;34m(self, params, defaults)\u001B[0m\n\u001B[1;32m 262\u001B[0m \u001B[38;5;129m@staticmethod\u001B[39m\n\u001B[1;32m 263\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mprofile_hook_step\u001B[39m(func):\n\u001B[1;32m 265\u001B[0m \u001B[38;5;129m@functools\u001B[39m\u001B[38;5;241m.\u001B[39mwraps(func)\n\u001B[0;32m--> 266\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mwrapper\u001B[39m(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs):\n\u001B[1;32m 267\u001B[0m \u001B[38;5;28mself\u001B[39m, \u001B[38;5;241m*\u001B[39m_ \u001B[38;5;241m=\u001B[39m args\n\u001B[1;32m 268\u001B[0m profile_name \u001B[38;5;241m=\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mOptimizer.step#\u001B[39m\u001B[38;5;132;01m{}\u001B[39;00m\u001B[38;5;124m.step\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;241m.\u001B[39mformat(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m\u001B[38;5;18m__class__\u001B[39m\u001B[38;5;241m.\u001B[39m\u001B[38;5;18m__name__\u001B[39m)\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_compile.py:22\u001B[0m, in \u001B[0;36minner\u001B[0;34m(*args, **kwargs)\u001B[0m\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_dynamo/__init__.py:1\u001B[0m\n\u001B[0;32m----> 1\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m allowed_functions, convert_frame, eval_frame, resume_execution\n\u001B[1;32m 2\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mbackends\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mregistry\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m list_backends, register_backend\n\u001B[1;32m 3\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mconvert_frame\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m replay\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py:29\u001B[0m\n\u001B[1;32m 27\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mguards\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m CheckFunctionManager, GuardedCode\n\u001B[1;32m 28\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mhooks\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Hooks\n\u001B[0;32m---> 29\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01moutput_graph\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m OutputGraph\n\u001B[1;32m 30\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mreplay_record\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m ExecutionRecord\n\u001B[1;32m 31\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01msymbolic_convert\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m InstructionTranslator\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_dynamo/output_graph.py:23\u001B[0m\n\u001B[1;32m 14\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtorch\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m_guards\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m (\n\u001B[1;32m 15\u001B[0m Checkpointable,\n\u001B[1;32m 16\u001B[0m Guard,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 19\u001B[0m TracingContext,\n\u001B[1;32m 20\u001B[0m )\n\u001B[1;32m 21\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtorch\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mfx\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mexperimental\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01msymbolic_shapes\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m ShapeEnv\n\u001B[0;32m---> 23\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m config, logging \u001B[38;5;28;01mas\u001B[39;00m torchdynamo_logging, variables\n\u001B[1;32m 24\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mbackends\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mregistry\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m CompiledFn, CompilerFn\n\u001B[1;32m 25\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mbytecode_transformation\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m create_instruction, Instruction, unique_id\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_dynamo/variables/__init__.py:1\u001B[0m\n\u001B[0;32m----> 1\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mbase\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m VariableTracker\n\u001B[1;32m 2\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mbuiltin\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m BuiltinVariable\n\u001B[1;32m 3\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mconstant\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m ConstantVariable, EnumVariable\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_dynamo/variables/base.py:6\u001B[0m\n\u001B[1;32m 4\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m variables\n\u001B[1;32m 5\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mexc\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m unimplemented\n\u001B[0;32m----> 6\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01msource\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m AttrSource, Source\n\u001B[1;32m 7\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mutils\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m dict_values, identity, istype, odict_values\n\u001B[1;32m 10\u001B[0m \u001B[38;5;28;01mclass\u001B[39;00m \u001B[38;5;21;01mMutableLocal\u001B[39;00m:\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_dynamo/source.py:49\u001B[0m\n\u001B[1;32m 39\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mis_input_source\u001B[39m(source):\n\u001B[1;32m 40\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m source\u001B[38;5;241m.\u001B[39mguard_source() \u001B[38;5;129;01min\u001B[39;00m [\n\u001B[1;32m 41\u001B[0m GuardSource\u001B[38;5;241m.\u001B[39mLOCAL,\n\u001B[1;32m 42\u001B[0m GuardSource\u001B[38;5;241m.\u001B[39mGLOBAL,\n\u001B[1;32m 43\u001B[0m GuardSource\u001B[38;5;241m.\u001B[39mLOCAL_NN_MODULE,\n\u001B[1;32m 44\u001B[0m GuardSource\u001B[38;5;241m.\u001B[39mGLOBAL_NN_MODULE,\n\u001B[1;32m 45\u001B[0m ]\n\u001B[1;32m 48\u001B[0m \u001B[38;5;129;43m@dataclasses\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdataclass\u001B[49m\n\u001B[0;32m---> 49\u001B[0m \u001B[38;5;28;43;01mclass\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[38;5;21;43;01mLocalSource\u001B[39;49;00m\u001B[43m(\u001B[49m\u001B[43mSource\u001B[49m\u001B[43m)\u001B[49m\u001B[43m:\u001B[49m\n\u001B[1;32m 50\u001B[0m \u001B[43m \u001B[49m\u001B[43mlocal_name\u001B[49m\u001B[43m:\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mstr\u001B[39;49m\n\u001B[1;32m 52\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;28;43;01mdef\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[38;5;21;43mreconstruct\u001B[39;49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcodegen\u001B[49m\u001B[43m)\u001B[49m\u001B[43m:\u001B[49m\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/dataclasses.py:1184\u001B[0m, in \u001B[0;36mdataclass\u001B[0;34m(cls, init, repr, eq, order, unsafe_hash, frozen, match_args, kw_only, slots)\u001B[0m\n\u001B[1;32m 1181\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m wrap\n\u001B[1;32m 1183\u001B[0m \u001B[38;5;66;03m# We're called as @dataclass without parens.\u001B[39;00m\n\u001B[0;32m-> 1184\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mwrap\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mcls\u001B[39;49m\u001B[43m)\u001B[49m\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/dataclasses.py:1175\u001B[0m, in \u001B[0;36mdataclass.<locals>.wrap\u001B[0;34m(cls)\u001B[0m\n\u001B[1;32m 1174\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mwrap\u001B[39m(\u001B[38;5;28mcls\u001B[39m):\n\u001B[0;32m-> 1175\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43m_process_class\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mcls\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43minit\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mrepr\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43meq\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43morder\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43munsafe_hash\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 1176\u001B[0m \u001B[43m \u001B[49m\u001B[43mfrozen\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmatch_args\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mkw_only\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mslots\u001B[49m\u001B[43m)\u001B[49m\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/dataclasses.py:985\u001B[0m, in \u001B[0;36m_process_class\u001B[0;34m(cls, init, repr, eq, order, unsafe_hash, frozen, match_args, kw_only, slots)\u001B[0m\n\u001B[1;32m 982\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m has_dataclass_bases:\n\u001B[1;32m 983\u001B[0m \u001B[38;5;66;03m# Raise an exception if any of our bases are frozen, but we're not.\u001B[39;00m\n\u001B[1;32m 984\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m any_frozen_base \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m frozen:\n\u001B[0;32m--> 985\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mTypeError\u001B[39;00m(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mcannot inherit non-frozen dataclass from a \u001B[39m\u001B[38;5;124m'\u001B[39m\n\u001B[1;32m 986\u001B[0m \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mfrozen one\u001B[39m\u001B[38;5;124m'\u001B[39m)\n\u001B[1;32m 988\u001B[0m \u001B[38;5;66;03m# Raise an exception if we're frozen, but none of our bases are.\u001B[39;00m\n\u001B[1;32m 989\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m any_frozen_base \u001B[38;5;129;01mand\u001B[39;00m frozen:\n", + "\u001B[0;31mTypeError\u001B[0m: cannot inherit non-frozen dataclass from a frozen one" + ] + } + ], + "source": [ + "#mit Regularisierung\n", + "\n", + "w_torch = torch.tensor(w_numpy, requires_grad=True)\n", + "\n", + "print('Reset Original weights', w_torch)\n", + "\n", + "sgd = torch.optim.SGD([w_torch], lr=lr, weight_decay=2)\n", + "\n", + "y_torch = torch.matmul(x_torch, w_torch)\n", + "loss = y_torch.sum()\n", + "\n", + "sgd.zero_grad()\n", + "loss.backward()\n", + "sgd.step()\n", + "\n", + "w_grad = w_torch.grad.data.numpy()\n", + "print('1 weight decay', w_torch)" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-20T22:39:58.623547Z", + "start_time": "2023-11-20T22:39:58.475069Z" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "# 3. Einfaches MLP in Pytorch\n", + "Machen Sie sich ein wenig mit dem `IMDB` Datensatz und den für Sie erstellten Datenstrukturen in x/y_train/test vertraut." + ], + "metadata": { + "id": "mLeOKvUxMunF" + } + }, + { + "cell_type": "markdown", + "source": [ + "## 3.1 Modell erstellen und Angaben zur Modellgröße verstehen\n", + "Definieren Sie ein Pytorch Multilayer Perzeptron mit der Größe des IMDB-Dictionaries für one-hot-encodierte Wörte als Eingabe (Sigmoid Aktivierung), 50 Neuronen im Hidden Layer und 2 Ausgabeneuronen. Layer 1 und 2 Ihres Netzes verwendet die Sigmoid-Aktivierungsfunktion, Layer 3 die Softmax-Aktivierungsfunktion.\n", + "\n", + "Generieren Sie Modell-Summary mit torchinfo und erklären Sie, was die ausgegebenen Werte bedeuten und wie diese zustande kommen." + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 24, + "outputs": [ { - "cell_type": "code", - "source": [ - "@widgets.interact_manual\n", - "def minibatch_experiment(batch_sizes='1, 20, 500, 17000',\n", - " lrs='0.9, 0.9, 0.9, 0.9',\n", - " time_budget=widgets.Dropdown(options=[\"0.05\", \"0.5\", \"2.0\", \"5.0\", \"7.5\"],\n", - " value=\"5.0\"),\n", - " use_momentum = widgets.ToggleButton(value=True)):\n", - "\n", - " \"\"\"\n", - " Demonstration of minibatch experiment\n", - "\n", - " Args:\n", - " batch_sizes: String\n", - " Size of minibatches\n", - " lrs: String\n", - " Different learning rates\n", - " time_budget: widget dropdown instance\n", - " Different time budgets with default=2.5s\n", - "\n", - " Returns:\n", - " Nothing\n", - " \"\"\"\n", - " batch_sizes = [int(s) for s in batch_sizes.split(',')]\n", - " lrs = [float(s) for s in lrs.split(',')]\n", - "\n", - " LOSS_HIST = {_:[] for _ in batch_sizes}\n", - "\n", - " #X, y = train_set.data, train_set.targets\n", - " base_model = Model(x_train.shape[1]).to(DEVICE)\n", - " #base_model = MLP(in_dim=784, out_dim=10, hidden_dims=[100, 100])\n", - "\n", - " for id, batch_size in enumerate(tqdm.auto.tqdm(batch_sizes)):\n", - " start_time = time.time()\n", - " # Create a new copy of the model for each batch size\n", - " model = copy.deepcopy(base_model)\n", - " params = list(model.parameters())\n", - " lr = lrs[id]\n", - " # Fixed budget per choice of batch size\n", - " #initial_vel = [torch.randn_like(p) for p in model.parameters()]\n", - " aux_tensors = [torch.zeros_like(_) for _ in params]\n", - " while (time.time() - start_time) < float(time_budget):\n", - " data, labels = sample_minibatch(x_train, y_train, batch_size)\n", - " loss = loss_fn(model(data), labels)\n", - " if use_momentum:\n", - " momentum_update(loss, params, grad_vel=aux_tensors, lr=lr, beta=0.5)\n", - " else:\n", - " gradient_update(loss, params, lr=lr)\n", - " LOSS_HIST[batch_size].append([time.time() - start_time,\n", - " loss.item()])\n", - "\n", - " fig, axs = plt.subplots(1, len(batch_sizes), figsize=(10, 3))\n", - " for ax, batch_size in zip(axs, batch_sizes):\n", - " plot_data = np.array(LOSS_HIST[batch_size])\n", - " ax.plot(plot_data[:, 0], plot_data[:, 1], label=batch_size,\n", - " alpha=0.8)\n", - " #ax.set_title('Batch size: ' + str(batch_size) + ' #: ' + str(batch_size*len(LOSS_HIST[batch_size])))\n", - " ax.set_title(' #: ' + str(batch_size*len(LOSS_HIST[batch_size])))\n", - " ax.set_xlabel('Seconds')\n", - " ax.set_ylabel('Loss')\n", - " plt.show()\n", - " #return(LOSS_HIST)\n" - ], - "metadata": { - "id": "ebevIBNQt4-l" - }, - "execution_count": null, - "outputs": [] - }, + "data": { + "text/plain": "=================================================================\nLayer (type:depth-idx) Param #\n=================================================================\nModel --\n├─Linear: 1-1 500,050\n├─Linear: 1-2 102\n├─Sigmoid: 1-3 --\n├─Softmax: 1-4 --\n=================================================================\nTotal params: 500,152\nTrainable params: 500,152\nNon-trainable params: 0\n=================================================================" + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "class Model(nn.Module):\n", + "\n", + " def __init__(self,\n", + " n_input: int = 10000,\n", + " n_hidden: int = 50,\n", + " n_out: int = 2):\n", + " super(Model, self).__init__()\n", + " self.hidden = nn.Linear(n_input, n_hidden)\n", + " self.out = nn.Linear(n_hidden, n_out)\n", + " self.sigmoid = nn.Sigmoid()\n", + " self.softmax = nn.Softmax()\n", + "\n", + "\n", + " def forward(self, x):\n", + " x = self.hidden(x)\n", + " x = self.sigmoid(x)\n", + " x = self.out(x)\n", + " x = self.softmax(x)\n", + " return x\n", + "\n", + "model = Model()\n", + "summary(m)" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-20T23:09:52.169718Z", + "start_time": "2023-11-20T23:09:52.161860Z" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## 3.2 Modell trainieren und Performancekurven interpretieren\n", + "Nutzen Sie den untenstehenedn Code um Ihr Modell zu trainieren. Interpretieren und diskutieren Sie die entstehenden Performancekurven. Falls Sie einen unerwarteten Anstieg Ihres Losses beobachten, recherchieren Sie wie Sie diese mit dem Einbau einer einzelnen Verbesserung innerhalb des gegebenen SGD Lernverfahrens beheben können. ACHTUNG: Wenn Sie Ihr Modell nicht oben neu initialisieren, optimieren Sie weiter auf den schon veränderten Parametern." + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 25, + "outputs": [ { - "cell_type": "markdown", - "source": [ - "" - ], - "metadata": { - "id": "e9yCfSXR1The" - } + "ename": "TypeError", + "evalue": "cannot inherit non-frozen dataclass from a frozen one", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mTypeError\u001B[0m Traceback (most recent call last)", + "Cell \u001B[0;32mIn[25], line 3\u001B[0m\n\u001B[1;32m 1\u001B[0m EPOCHS \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m291\u001B[39m \u001B[38;5;66;03m#@param {type:\"slider\", min:2, max:1000, step:1}\u001B[39;00m\n\u001B[1;32m 2\u001B[0m RATE \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m0.902\u001B[39m \u001B[38;5;66;03m#@param {type:\"slider\", min:0.001, max:2, step:0.001}\u001B[39;00m\n\u001B[0;32m----> 3\u001B[0m optimizer \u001B[38;5;241m=\u001B[39m \u001B[43mtorch\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43moptim\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mSGD\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmodel\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mparameters\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mlr\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mRATE\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mweight_decay\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;241;43m0\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[1;32m 4\u001B[0m loss_fn \u001B[38;5;241m=\u001B[39m nn\u001B[38;5;241m.\u001B[39mCrossEntropyLoss()\n\u001B[1;32m 5\u001B[0m loss_list \u001B[38;5;241m=\u001B[39m np\u001B[38;5;241m.\u001B[39mzeros((EPOCHS,))\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/optim/sgd.py:26\u001B[0m, in \u001B[0;36m__init__\u001B[0;34m(self, params, lr, momentum, dampening, weight_decay, nesterov, maximize, foreach, differentiable)\u001B[0m\n\u001B[1;32m 21\u001B[0m defaults \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mdict\u001B[39m(lr\u001B[38;5;241m=\u001B[39mlr, momentum\u001B[38;5;241m=\u001B[39mmomentum, dampening\u001B[38;5;241m=\u001B[39mdampening,\n\u001B[1;32m 22\u001B[0m weight_decay\u001B[38;5;241m=\u001B[39mweight_decay, nesterov\u001B[38;5;241m=\u001B[39mnesterov,\n\u001B[1;32m 23\u001B[0m maximize\u001B[38;5;241m=\u001B[39mmaximize, foreach\u001B[38;5;241m=\u001B[39mforeach,\n\u001B[1;32m 24\u001B[0m differentiable\u001B[38;5;241m=\u001B[39mdifferentiable)\n\u001B[1;32m 25\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m nesterov \u001B[38;5;129;01mand\u001B[39;00m (momentum \u001B[38;5;241m<\u001B[39m\u001B[38;5;241m=\u001B[39m \u001B[38;5;241m0\u001B[39m \u001B[38;5;129;01mor\u001B[39;00m dampening \u001B[38;5;241m!=\u001B[39m \u001B[38;5;241m0\u001B[39m):\n\u001B[0;32m---> 26\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mNesterov momentum requires a momentum and zero dampening\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m 27\u001B[0m \u001B[38;5;28msuper\u001B[39m()\u001B[38;5;241m.\u001B[39m\u001B[38;5;21m__init__\u001B[39m(params, defaults)\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/optim/optimizer.py:266\u001B[0m, in \u001B[0;36m__init__\u001B[0;34m(self, params, defaults)\u001B[0m\n\u001B[1;32m 262\u001B[0m \u001B[38;5;129m@staticmethod\u001B[39m\n\u001B[1;32m 263\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mprofile_hook_step\u001B[39m(func):\n\u001B[1;32m 265\u001B[0m \u001B[38;5;129m@functools\u001B[39m\u001B[38;5;241m.\u001B[39mwraps(func)\n\u001B[0;32m--> 266\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mwrapper\u001B[39m(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs):\n\u001B[1;32m 267\u001B[0m \u001B[38;5;28mself\u001B[39m, \u001B[38;5;241m*\u001B[39m_ \u001B[38;5;241m=\u001B[39m args\n\u001B[1;32m 268\u001B[0m profile_name \u001B[38;5;241m=\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mOptimizer.step#\u001B[39m\u001B[38;5;132;01m{}\u001B[39;00m\u001B[38;5;124m.step\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;241m.\u001B[39mformat(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m\u001B[38;5;18m__class__\u001B[39m\u001B[38;5;241m.\u001B[39m\u001B[38;5;18m__name__\u001B[39m)\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_compile.py:22\u001B[0m, in \u001B[0;36minner\u001B[0;34m(*args, **kwargs)\u001B[0m\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_dynamo/__init__.py:1\u001B[0m\n\u001B[0;32m----> 1\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m allowed_functions, convert_frame, eval_frame, resume_execution\n\u001B[1;32m 2\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mbackends\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mregistry\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m list_backends, register_backend\n\u001B[1;32m 3\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mconvert_frame\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m replay\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py:29\u001B[0m\n\u001B[1;32m 27\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mguards\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m CheckFunctionManager, GuardedCode\n\u001B[1;32m 28\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mhooks\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Hooks\n\u001B[0;32m---> 29\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01moutput_graph\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m OutputGraph\n\u001B[1;32m 30\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mreplay_record\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m ExecutionRecord\n\u001B[1;32m 31\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01msymbolic_convert\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m InstructionTranslator\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_dynamo/output_graph.py:23\u001B[0m\n\u001B[1;32m 14\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtorch\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m_guards\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m (\n\u001B[1;32m 15\u001B[0m Checkpointable,\n\u001B[1;32m 16\u001B[0m Guard,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 19\u001B[0m TracingContext,\n\u001B[1;32m 20\u001B[0m )\n\u001B[1;32m 21\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtorch\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mfx\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mexperimental\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01msymbolic_shapes\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m ShapeEnv\n\u001B[0;32m---> 23\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m config, logging \u001B[38;5;28;01mas\u001B[39;00m torchdynamo_logging, variables\n\u001B[1;32m 24\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mbackends\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mregistry\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m CompiledFn, CompilerFn\n\u001B[1;32m 25\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mbytecode_transformation\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m create_instruction, Instruction, unique_id\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_dynamo/variables/__init__.py:1\u001B[0m\n\u001B[0;32m----> 1\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mbase\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m VariableTracker\n\u001B[1;32m 2\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mbuiltin\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m BuiltinVariable\n\u001B[1;32m 3\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mconstant\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m ConstantVariable, EnumVariable\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_dynamo/variables/base.py:6\u001B[0m\n\u001B[1;32m 4\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m variables\n\u001B[1;32m 5\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mexc\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m unimplemented\n\u001B[0;32m----> 6\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01msource\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m AttrSource, Source\n\u001B[1;32m 7\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mutils\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m dict_values, identity, istype, odict_values\n\u001B[1;32m 10\u001B[0m \u001B[38;5;28;01mclass\u001B[39;00m \u001B[38;5;21;01mMutableLocal\u001B[39;00m:\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/site-packages/torch/_dynamo/source.py:49\u001B[0m\n\u001B[1;32m 39\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mis_input_source\u001B[39m(source):\n\u001B[1;32m 40\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m source\u001B[38;5;241m.\u001B[39mguard_source() \u001B[38;5;129;01min\u001B[39;00m [\n\u001B[1;32m 41\u001B[0m GuardSource\u001B[38;5;241m.\u001B[39mLOCAL,\n\u001B[1;32m 42\u001B[0m GuardSource\u001B[38;5;241m.\u001B[39mGLOBAL,\n\u001B[1;32m 43\u001B[0m GuardSource\u001B[38;5;241m.\u001B[39mLOCAL_NN_MODULE,\n\u001B[1;32m 44\u001B[0m GuardSource\u001B[38;5;241m.\u001B[39mGLOBAL_NN_MODULE,\n\u001B[1;32m 45\u001B[0m ]\n\u001B[1;32m 48\u001B[0m \u001B[38;5;129;43m@dataclasses\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdataclass\u001B[49m\n\u001B[0;32m---> 49\u001B[0m \u001B[38;5;28;43;01mclass\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[38;5;21;43;01mLocalSource\u001B[39;49;00m\u001B[43m(\u001B[49m\u001B[43mSource\u001B[49m\u001B[43m)\u001B[49m\u001B[43m:\u001B[49m\n\u001B[1;32m 50\u001B[0m \u001B[43m \u001B[49m\u001B[43mlocal_name\u001B[49m\u001B[43m:\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mstr\u001B[39;49m\n\u001B[1;32m 52\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;28;43;01mdef\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[38;5;21;43mreconstruct\u001B[39;49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcodegen\u001B[49m\u001B[43m)\u001B[49m\u001B[43m:\u001B[49m\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/dataclasses.py:1184\u001B[0m, in \u001B[0;36mdataclass\u001B[0;34m(cls, init, repr, eq, order, unsafe_hash, frozen, match_args, kw_only, slots)\u001B[0m\n\u001B[1;32m 1181\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m wrap\n\u001B[1;32m 1183\u001B[0m \u001B[38;5;66;03m# We're called as @dataclass without parens.\u001B[39;00m\n\u001B[0;32m-> 1184\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mwrap\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mcls\u001B[39;49m\u001B[43m)\u001B[49m\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/dataclasses.py:1175\u001B[0m, in \u001B[0;36mdataclass.<locals>.wrap\u001B[0;34m(cls)\u001B[0m\n\u001B[1;32m 1174\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mwrap\u001B[39m(\u001B[38;5;28mcls\u001B[39m):\n\u001B[0;32m-> 1175\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43m_process_class\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mcls\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43minit\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mrepr\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43meq\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43morder\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43munsafe_hash\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 1176\u001B[0m \u001B[43m \u001B[49m\u001B[43mfrozen\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmatch_args\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mkw_only\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mslots\u001B[49m\u001B[43m)\u001B[49m\n", + "File \u001B[0;32m~/miniconda3/envs/ginn/lib/python3.10/dataclasses.py:985\u001B[0m, in \u001B[0;36m_process_class\u001B[0;34m(cls, init, repr, eq, order, unsafe_hash, frozen, match_args, kw_only, slots)\u001B[0m\n\u001B[1;32m 982\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m has_dataclass_bases:\n\u001B[1;32m 983\u001B[0m \u001B[38;5;66;03m# Raise an exception if any of our bases are frozen, but we're not.\u001B[39;00m\n\u001B[1;32m 984\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m any_frozen_base \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m frozen:\n\u001B[0;32m--> 985\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mTypeError\u001B[39;00m(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mcannot inherit non-frozen dataclass from a \u001B[39m\u001B[38;5;124m'\u001B[39m\n\u001B[1;32m 986\u001B[0m \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mfrozen one\u001B[39m\u001B[38;5;124m'\u001B[39m)\n\u001B[1;32m 988\u001B[0m \u001B[38;5;66;03m# Raise an exception if we're frozen, but none of our bases are.\u001B[39;00m\n\u001B[1;32m 989\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m any_frozen_base \u001B[38;5;129;01mand\u001B[39;00m frozen:\n", + "\u001B[0;31mTypeError\u001B[0m: cannot inherit non-frozen dataclass from a frozen one" + ] } - ] -} \ No newline at end of file + ], + "source": [ + "EPOCHS = 291 #@param {type:\"slider\", min:2, max:1000, step:1}\n", + "RATE = 0.902 #@param {type:\"slider\", min:0.001, max:2, step:0.001}\n", + "optimizer = torch.optim.SGD(model.parameters(), lr=RATE, weight_decay=0)\n", + "loss_fn = nn.CrossEntropyLoss()\n", + "loss_list = np.zeros((EPOCHS,))\n", + "accuracy_list = np.zeros((EPOCHS,))\n", + "accuracy_list_test = np.zeros((EPOCHS,))\n", + "\n", + "\n", + "for epoch in tqdm.trange(EPOCHS):\n", + " y_pred = model(x_train)\n", + " #loss = loss_fn(y_pred, y_train)\n", + " loss_list[epoch] = loss.item()\n", + " loss = loss_fn(y_pred, y_train)# + 0.01 *l2_reg(model)\n", + "\n", + " # Zero gradients\n", + " optimizer.zero_grad()\n", + "\n", + " loss.backward()\n", + " #torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)#, args.clip)\n", + " optimizer.step()\n", + "\n", + " with torch.no_grad():\n", + " y_pred = model(x_train)\n", + " correct = (torch.argmax(y_pred, dim=1) == y_train).type(torch.FloatTensor)\n", + " accuracy_list[epoch] = correct.mean()\n", + " y_pred = model(x_test)\n", + " correct = (torch.argmax(y_pred, dim=1) == y_test).type(torch.FloatTensor)\n", + " accuracy_list_test[epoch] = correct.mean()\n", + "\n", + "\n", + "\n", + "\n", + "fig, (ax1, ax2, ax3) = plt.subplots(3, figsize=(12, 6), sharex=True)\n", + "\n", + "ax1.plot(accuracy_list)\n", + "ax1.set_ylabel(\"train accuracy\")\n", + "ax2.plot(loss_list)\n", + "ax2.set_ylabel(\"train loss\")\n", + "ax3.plot(accuracy_list_test)\n", + "ax3.set_ylabel(\"test acc\")\n", + "ax3.set_xlabel(\"epochs\");" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-20T23:10:04.077035Z", + "start_time": "2023-11-20T23:10:03.912932Z" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## 3.3. Momentum Implementieren\n", + "Vervollständigen Sie Methode momentum_update. Überlegen Sie sich, wie Sie die Korrektheit mit einem Durchlauf inkl. Momentum Update auf Ihrem oben definierten Modell prüfen können" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "def momentum_update(loss, params, grad_vel, lr=1e-3, beta=0.8):\n", + " # Clear up gradients as Pytorch automatically accumulates gradients from\n", + " # successive backward calls\n", + " zero_grad(params)\n", + " # Compute gradients on given objective\n", + " loss.backward()\n", + "\n", + " with torch.no_grad():\n", + " for (par, vel) in zip(params, grad_vel):\n", + " w_u = lr * par.grad + beta * grad_vel\n", + " grad_vel = w_u\n", + " par -= w_u" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "## 3.4 Experimente zum Lernverhalten mit Momentum und Batch Size\n", + "Im folgenden können Sie für ein festgelegtes Zeitbudget schauen, wie sich der Loss Ihres neuronalen Netzes innerhalb dieser Zeit entwickelt.\n", + "Experimentieren Sie zunächst mit den Voreinstellungen mit und ohne Momentum, probieren Sie dann eigene Einstellungen aus. Diskutieren Sie das visualisierte Lernverhalten insbesondere bzgl. unterschiedlicher Batch Sizes." + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "@widgets.interact_manual\n", + "def minibatch_experiment(batch_sizes='1, 20, 500, 17000',\n", + " lrs='0.9, 0.9, 0.9, 0.9',\n", + " time_budget=widgets.Dropdown(options=[\"0.05\", \"0.5\", \"2.0\", \"5.0\", \"7.5\"],\n", + " value=\"5.0\"),\n", + " use_momentum = widgets.ToggleButton(value=True)):\n", + "\n", + " \"\"\"\n", + " Demonstration of minibatch experiment\n", + "\n", + " Args:\n", + " batch_sizes: String\n", + " Size of minibatches\n", + " lrs: String\n", + " Different learning rates\n", + " time_budget: widget dropdown instance\n", + " Different time budgets with default=2.5s\n", + "\n", + " Returns:\n", + " Nothing\n", + " \"\"\"\n", + " batch_sizes = [int(s) for s in batch_sizes.split(',')]\n", + " lrs = [float(s) for s in lrs.split(',')]\n", + "\n", + " LOSS_HIST = {_:[] for _ in batch_sizes}\n", + "\n", + " #X, y = train_set.data, train_set.targets\n", + " base_model = Model(x_train.shape[1]).to(DEVICE)\n", + " #base_model = MLP(in_dim=784, out_dim=10, hidden_dims=[100, 100])\n", + "\n", + " for id, batch_size in enumerate(tqdm.auto.tqdm(batch_sizes)):\n", + " start_time = time.time()\n", + " # Create a new copy of the model for each batch size\n", + " model = copy.deepcopy(base_model)\n", + " params = list(model.parameters())\n", + " lr = lrs[id]\n", + " # Fixed budget per choice of batch size\n", + " #initial_vel = [torch.randn_like(p) for p in model.parameters()]\n", + " aux_tensors = [torch.zeros_like(_) for _ in params]\n", + " while (time.time() - start_time) < float(time_budget):\n", + " data, labels = sample_minibatch(x_train, y_train, batch_size)\n", + " loss = loss_fn(model(data), labels)\n", + " if use_momentum:\n", + " momentum_update(loss, params, grad_vel=aux_tensors, lr=lr, beta=0.5)\n", + " else:\n", + " gradient_update(loss, params, lr=lr)\n", + " LOSS_HIST[batch_size].append([time.time() - start_time,\n", + " loss.item()])\n", + "\n", + " fig, axs = plt.subplots(1, len(batch_sizes), figsize=(10, 3))\n", + " for ax, batch_size in zip(axs, batch_sizes):\n", + " plot_data = np.array(LOSS_HIST[batch_size])\n", + " ax.plot(plot_data[:, 0], plot_data[:, 1], label=batch_size,\n", + " alpha=0.8)\n", + " #ax.set_title('Batch size: ' + str(batch_size) + ' #: ' + str(batch_size*len(LOSS_HIST[batch_size])))\n", + " ax.set_title(' #: ' + str(batch_size*len(LOSS_HIST[batch_size])))\n", + " ax.set_xlabel('Seconds')\n", + " ax.set_ylabel('Loss')\n", + " plt.show()\n", + " #return(LOSS_HIST)\n" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "" + ], + "metadata": { + "collapsed": false + } + } + ] +} -- GitLab