reformer_fastai/docs/experiment.enwik8-n_hashes.html

---

title: LSH attention


keywords: fastai
sidebar: home_sidebar

summary: "model performance as function of number of hashing rounds"
description: "model performance as function of number of hashing rounds"
nb_path: "nbs/13_experiment.enwik8-n_hashes.ipynb"
---
<!--

#################################################
### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ###
#################################################
# file to edit: nbs/13_experiment.enwik8-n_hashes.ipynb
# command to build the docs after a change: nbdev_build_docs

-->

<div class="container" id="notebook-container">
        
    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span> 
</pre></div>

    </div>
</div>
</div>

</div>
    {% endraw %}

<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h2 id="Experiment-Tracking">Experiment Tracking<a class="anchor-link" href="#Experiment-Tracking"> </a></h2>
</div>
</div>
</div>
<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<p>Make sure you have wandb and are logged in:</p>

</div>
</div>
</div>
<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<p>Load Experiment Tracking with Weights &amp; Biases:</p>

</div>
</div>
</div>
    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">import</span> <span class="nn">wandb</span>

<span class="n">WANDB_NAME</span> <span class="o">=</span> <span class="s1">&#39;test_n_hashes_enwik8&#39;</span>
<span class="n">GROUP</span> <span class="o">=</span> <span class="s1">&#39;TEST&#39;</span>
<span class="n">NOTES</span> <span class="o">=</span> <span class="s1">&#39;LSHLM on enwik8 sl 4096&#39;</span>
<span class="n">CONFIG</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">TAGS</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;lm&#39;</span><span class="p">,</span><span class="s1">&#39;lsh&#39;</span><span class="p">,</span><span class="s1">&#39;enwik8&#39;</span><span class="p">,</span> <span class="s1">&#39;test&#39;</span><span class="p">]</span>
</pre></div>

    </div>
</div>
</div>

</div>
    {% endraw %}

<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h2 id="Download-and-Unpack-enwik8-Data">Download and Unpack enwik8 Data<a class="anchor-link" href="#Download-and-Unpack-enwik8-Data"> </a></h2><p>Download and unzip enwik8 data</p>

</div>
</div>
</div>
    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">path</span> <span class="o">=</span> <span class="n">untar_data</span><span class="p">(</span><span class="s1">&#39;http://mattmahoney.net/dc/enwik8.zip&#39;</span><span class="p">,</span> <span class="n">dest</span><span class="o">=</span><span class="s1">&#39;/data&#39;</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">


<div class="output_html rendered_html output_subarea ">

</div>

</div>

</div>
</div>

</div>
    {% endraw %}

<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h2 id="Prepare-Data">Prepare Data<a class="anchor-link" href="#Prepare-Data"> </a></h2>
</div>
</div>
</div>
    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;text&#39;</span><span class="p">:</span><span class="n">read_lines</span><span class="p">(</span><span class="n">path</span><span class="p">)})</span>
<span class="n">df</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">


<div class="output_html rendered_html output_subarea output_execute_result">
<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>text</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>&lt;mediawiki xmlns="http://www.mediawiki.org/xml/export-0.3/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.3/ http://www.mediawiki.org/xml/export-0.3.xsd" version="0.3" xml:lang="en"&gt;\n</td>
    </tr>
    <tr>
      <th>1</th>
      <td>&lt;siteinfo&gt;\n</td>
    </tr>
    <tr>
      <th>2</th>
      <td>&lt;sitename&gt;Wikipedia&lt;/sitename&gt;\n</td>
    </tr>
    <tr>
      <th>3</th>
      <td>&lt;base&gt;http://en.wikipedia.org/wiki/Main_Page&lt;/base&gt;\n</td>
    </tr>
    <tr>
      <th>4</th>
      <td>&lt;generator&gt;MediaWiki 1.6alpha&lt;/generator&gt;\n</td>
    </tr>
  </tbody>
</table>
</div>
</div>

</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">btt</span> <span class="o">=</span> <span class="n">ByteTextTokenizer</span><span class="p">(</span><span class="n">is_lm</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">add_bos</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">add_eos</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="o">%%time</span>
<span class="n">df</span><span class="p">[</span><span class="s1">&#39;toks&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="s1">&#39;text&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="n">btt</span><span class="p">)</span>
<span class="n">df</span><span class="p">[</span><span class="s1">&#39;lens&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="s1">&#39;toks&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="nb">len</span><span class="p">)</span>
<span class="n">df</span><span class="p">[</span><span class="s1">&#39;lens_cum_sum&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">lens</span><span class="o">.</span><span class="n">cumsum</span><span class="p">()</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">

<div class="output_subarea output_stream output_stdout output_text">
<pre>CPU times: user 1min 56s, sys: 2.56 s, total: 1min 58s
Wall time: 1min 55s
</pre>
</div>
</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">train_cutoff</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">lens</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> <span class="o">-</span> <span class="mi">10_000_000</span>  <span class="c1"># keep all but 10M characters for val and test</span>
<span class="n">train_idxs</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">df</span><span class="p">[</span><span class="s1">&#39;lens_cum_sum&#39;</span><span class="p">]</span> <span class="o">&lt;</span> <span class="n">train_cutoff</span><span class="p">]</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">values</span>
<span class="n">train_idxs</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">max</span><span class="p">(</span><span class="n">train_idxs</span><span class="p">)))</span>

<span class="n">remaining_idxs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="o">-</span> <span class="nb">max</span><span class="p">(</span><span class="n">train_idxs</span><span class="p">)</span>
<span class="n">validation_idxs</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="nb">max</span><span class="p">(</span><span class="n">train_idxs</span><span class="p">),</span> <span class="nb">max</span><span class="p">(</span><span class="n">train_idxs</span><span class="p">)</span> <span class="o">+</span> <span class="nb">int</span><span class="p">(</span><span class="n">remaining_idxs</span><span class="o">/</span><span class="mi">2</span><span class="p">)))</span>
<span class="n">test_idxs</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="nb">max</span><span class="p">(</span><span class="n">validation_idxs</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="n">df</span><span class="p">)))</span>

<span class="n">splits</span> <span class="o">=</span> <span class="p">[</span><span class="n">train_idxs</span><span class="p">,</span> <span class="n">validation_idxs</span><span class="p">]</span>
</pre></div>

    </div>
</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">tfms</span> <span class="o">=</span> <span class="p">[</span><span class="n">attrgetter</span><span class="p">(</span><span class="s2">&quot;text&quot;</span><span class="p">),</span> <span class="n">btt</span><span class="p">]</span>
<span class="n">dsets</span> <span class="o">=</span> <span class="n">Datasets</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="p">[</span><span class="n">tfms</span><span class="p">],</span> <span class="n">splits</span><span class="o">=</span><span class="n">splits</span><span class="p">,</span> <span class="n">dl_type</span><span class="o">=</span><span class="n">LMDataLoader</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="o">%%time</span>
<span class="n">bs</span><span class="p">,</span> <span class="n">sl</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">4096</span>
<span class="c1"># pad_seq2seq = partial(pad_input, pad_idx=bte.pad_token_id, pad_fields=[0,1])</span>
<span class="n">dl_kwargs</span> <span class="o">=</span> <span class="p">[{</span><span class="s1">&#39;lens&#39;</span><span class="p">:</span><span class="n">df</span><span class="p">[</span><span class="s1">&#39;lens&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">train_idxs</span><span class="p">]},</span>
             <span class="p">{</span><span class="s1">&#39;val_lens&#39;</span><span class="p">:</span><span class="n">df</span><span class="p">[</span><span class="s1">&#39;lens&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">validation_idxs</span><span class="p">]}]</span>
<span class="n">dls</span> <span class="o">=</span> <span class="n">dsets</span><span class="o">.</span><span class="n">dataloaders</span><span class="p">(</span><span class="n">bs</span><span class="o">=</span><span class="n">bs</span><span class="p">,</span> <span class="n">seq_len</span><span class="o">=</span><span class="n">sl</span><span class="p">,</span> <span class="n">dl_kwargs</span><span class="o">=</span><span class="n">dl_kwargs</span><span class="p">,</span> <span class="n">shuffle_train</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">n_workers</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">

<div class="output_subarea output_stream output_stdout output_text">
<pre>CPU times: user 28.5 s, sys: 2.43 s, total: 30.9 s
Wall time: 36.4 s
</pre>
</div>
</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">dls</span><span class="o">.</span><span class="n">show_batch</span><span class="p">(</span><span class="n">max_n</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>
<details class="description">
      <summary data-open="Hide Output" data-close="Show Output"></summary>
        <summary></summary>
        
<div class="output_wrapper">
<div class="output">

<div class="output_area">


<div class="output_html rendered_html output_subarea ">
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>text</th>
      <th>text_</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>*[[Shot (film)|Shot]]\n* [[Computer network]]\n== Grammar ==\nThe 46 city divisions are combined into 16 area districts or ''[[Ortsbezirke]]'', which each have a district committee and chairperson.\n*The [[Kennedy School of Government|John F. Kennedy School of Government]] (1936)\n*[[swallow]]\n\n\n[[ro:Alfred Nobel]]\nPerhaps the most notorious incident of these centuries was the massacre of Christians in [[1860]], when fighting between [[Druze]] and [[Maronites]] in [[Mount Lebanon]] spilled over into the city. Some thousands of Christians were killed, with many more being saved through the intervention of the Algerian exile [[Abd al-Qadir]] and his soldiers, who brought them to safety in Abd al-Qadir's residence and the citadel. The Christian quarter of the old city, including a number of churches, was burnt down. The Christian inhabitants of the notoriously poor and refractory Midan district outside the walls were, however, protected by their Muslim neighbours.\n\n      &lt;comment&gt;/* Marriage */&lt;/comment&gt;\n[[br:Albert Speer]]\n* [[Transportation in Luxembourg|Luxembourg]] &amp;amp;mdash; yes\n[[fr:Alaric II]]\nThe</td>
      <td>[[Shot (film)|Shot]]\n* [[Computer network]]\n== Grammar ==\nThe 46 city divisions are combined into 16 area districts or ''[[Ortsbezirke]]'', which each have a district committee and chairperson.\n*The [[Kennedy School of Government|John F. Kennedy School of Government]] (1936)\n*[[swallow]]\n\n\n[[ro:Alfred Nobel]]\nPerhaps the most notorious incident of these centuries was the massacre of Christians in [[1860]], when fighting between [[Druze]] and [[Maronites]] in [[Mount Lebanon]] spilled over into the city. Some thousands of Christians were killed, with many more being saved through the intervention of the Algerian exile [[Abd al-Qadir]] and his soldiers, who brought them to safety in Abd al-Qadir's residence and the citadel. The Christian quarter of the old city, including a number of churches, was burnt down. The Christian inhabitants of the notoriously poor and refractory Midan district outside the walls were, however, protected by their Muslim neighbours.\n\n      &lt;comment&gt;/* Marriage */&lt;/comment&gt;\n[[br:Albert Speer]]\n* [[Transportation in Luxembourg|Luxembourg]] &amp;amp;mdash; yes\n[[fr:Alaric II]]\nThe</td>
    </tr>
    <tr>
      <th>1</th>
      <td>ca's revolt.\n== W ==\n\n[[Image:Stairs_of_Finnish_Parliament.jpg|thumb|[[Finnish Parliament]] House in [[Helsinki]]]]\n[[Category:Christian music]]\n| 1986&amp;amp;ndash;1996 || Frank Shrontz \n*[[Kleemann]]\n      &lt;text xml:space="preserve"&gt;#REDIRECT [[March 1]]&lt;/text&gt;\n[[fr:Bataille de Blenheim]]\nBDC&amp;lt;br&amp;gt;\n      &lt;/contributor&gt;\n[[es:Congreso Nacional Africano]]\n*''[[Blackmail (1929 film)|Blackmail]]'' (1929), the first British [[talkie]]\n{{expand list}}\n# [[Diapason (movie)|Diapason]] (Italy, 2001) [http://www.imdb.com/title/tt0274460/ (IMDb entry)]\n      &lt;minor /&gt;\n\n[[Image:Ethnic Composition of BiH in 2005.GIF|thumb|150px|right|Ethnic composition of Bosnia &amp;amp; Herzegovina in 2005. &amp;lt;br&amp;gt; '''Green:''' Predominantly ethnic Bosniaks &amp;lt;br&amp;gt; '''Red:''' Predominantly ethnic Serbs &amp;lt;br&amp;gt; '''Blue:''' Predominantly ethnic Croats]]\n | url = http://patft.uspto.gov/netacgi/nph-Parser?Sect1=PTO2&amp;amp;Sect2=HITOFF&amp;amp;u=%2Fnetahtml%2Fsearch-adv.htm&amp;amp;r=0&amp;amp;p=1&amp;amp;f=S&amp;amp;l=50&amp;amp;Query=an%2F%22Cessna+Aircraft+Company%22&amp;amp;d=ptxt\n\n* [[Panama Canal]]\n*[http://www.liverpoolmuseums.org.uk/walker/exhibitions/heathrobinson Heath Robinson online exhibition]\n\n* Doyle, J. &amp;amp; Carlson, J. M. (2000) ''Phys. Rev. Lett.'' '''84,''' 5656–5659.\n  &lt;page&gt;\n\n[[lt:Žemynai]]\n{{coor dm|15|00|N|86|30|W|type:country}}\n    &lt;revision&gt;\n***[[Newberry Caldera]] ([[Oregon]], US)\nThe programme makers emphasised that it was to be about 'everyday life' in the inner city 'today' and regard it as a 'slice of life'. Creator/ producer Julia Smith declared that 'we don't make life, we reflect</td>
      <td>a's revolt.\n== W ==\n\n[[Image:Stairs_of_Finnish_Parliament.jpg|thumb|[[Finnish Parliament]] House in [[Helsinki]]]]\n[[Category:Christian music]]\n| 1986&amp;amp;ndash;1996 || Frank Shrontz \n*[[Kleemann]]\n      &lt;text xml:space="preserve"&gt;#REDIRECT [[March 1]]&lt;/text&gt;\n[[fr:Bataille de Blenheim]]\nBDC&amp;lt;br&amp;gt;\n      &lt;/contributor&gt;\n[[es:Congreso Nacional Africano]]\n*''[[Blackmail (1929 film)|Blackmail]]'' (1929), the first British [[talkie]]\n{{expand list}}\n# [[Diapason (movie)|Diapason]] (Italy, 2001) [http://www.imdb.com/title/tt0274460/ (IMDb entry)]\n      &lt;minor /&gt;\n\n[[Image:Ethnic Composition of BiH in 2005.GIF|thumb|150px|right|Ethnic composition of Bosnia &amp;amp; Herzegovina in 2005. &amp;lt;br&amp;gt; '''Green:''' Predominantly ethnic Bosniaks &amp;lt;br&amp;gt; '''Red:''' Predominantly ethnic Serbs &amp;lt;br&amp;gt; '''Blue:''' Predominantly ethnic Croats]]\n | url = http://patft.uspto.gov/netacgi/nph-Parser?Sect1=PTO2&amp;amp;Sect2=HITOFF&amp;amp;u=%2Fnetahtml%2Fsearch-adv.htm&amp;amp;r=0&amp;amp;p=1&amp;amp;f=S&amp;amp;l=50&amp;amp;Query=an%2F%22Cessna+Aircraft+Company%22&amp;amp;d=ptxt\n\n* [[Panama Canal]]\n*[http://www.liverpoolmuseums.org.uk/walker/exhibitions/heathrobinson Heath Robinson online exhibition]\n\n* Doyle, J. &amp;amp; Carlson, J. M. (2000) ''Phys. Rev. Lett.'' '''84,''' 5656–5659.\n  &lt;page&gt;\n\n[[lt:Žemynai]]\n{{coor dm|15|00|N|86|30|W|type:country}}\n    &lt;revision&gt;\n***[[Newberry Caldera]] ([[Oregon]], US)\nThe programme makers emphasised that it was to be about 'everyday life' in the inner city 'today' and regard it as a 'slice of life'. Creator/ producer Julia Smith declared that 'we don't make life, we reflect</td>
    </tr>
  </tbody>
</table>
</div>

</div>

</div>
</div>

    </details>
</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">vocab_sz</span> <span class="o">=</span> <span class="n">btt</span><span class="o">.</span><span class="n">vocab_size</span>
</pre></div>

    </div>
</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">xb</span><span class="p">,</span> <span class="n">yb</span> <span class="o">=</span> <span class="n">dls</span><span class="o">.</span><span class="n">one_batch</span><span class="p">()</span>
<span class="n">xb</span><span class="o">.</span><span class="n">shape</span><span class="p">,</span> <span class="n">yb</span><span class="o">.</span><span class="n">shape</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">



<div class="output_text output_subarea output_execute_result">
<pre>((4, 4096), (4, 4096))</pre>
</div>

</div>

</div>
</div>

</div>
    {% endraw %}

<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h2 id="Training">Training<a class="anchor-link" href="#Training"> </a></h2>
</div>
</div>
</div>
    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">wandb</span><span class="o">.</span><span class="n">init</span><span class="p">(</span><span class="n">reinit</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">project</span><span class="o">=</span><span class="s2">&quot;reformer-fastai&quot;</span><span class="p">,</span> <span class="n">entity</span><span class="o">=</span><span class="s2">&quot;fastai_community&quot;</span><span class="p">,</span> 
           <span class="n">name</span><span class="o">=</span><span class="n">WANDB_NAME</span><span class="p">,</span> <span class="n">group</span><span class="o">=</span><span class="n">GROUP</span><span class="p">,</span> <span class="n">notes</span><span class="o">=</span><span class="n">NOTES</span><span class="p">,</span> <span class="n">tags</span><span class="o">=</span><span class="n">TAGS</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">CONFIG</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">config</span> <span class="o">=</span> <span class="n">NHashesConfig</span><span class="p">(</span><span class="n">n_hashes</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">learn</span> <span class="o">=</span> <span class="n">Learner</span><span class="p">(</span><span class="n">dls</span><span class="p">,</span> <span class="n">LSHLM</span><span class="o">.</span><span class="n">from_config</span><span class="p">(</span><span class="n">config</span><span class="p">),</span>
                <span class="n">loss_func</span><span class="o">=</span><span class="n">CrossEntropyLossFlat</span><span class="p">(),</span> <span class="n">opt_func</span><span class="o">=</span><span class="n">adafactor</span><span class="p">,</span>
                <span class="n">cbs</span> <span class="o">=</span> <span class="p">[</span><span class="n">GradientAccumulation</span><span class="p">(</span><span class="n">n_acc</span><span class="o">=</span><span class="mi">8</span><span class="p">),</span> <span class="n">GradientClip</span><span class="p">(</span><span class="mf">1.0</span><span class="p">),</span>
                       <span class="n">PadBatchCallback</span><span class="p">(</span><span class="n">bucket_size</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">bucket_size</span><span class="p">)],</span>
                <span class="n">metrics</span><span class="o">=</span><span class="p">[</span><span class="n">accuracy</span><span class="p">,</span> <span class="n">perplexity</span><span class="p">,</span> <span class="n">bpc</span><span class="p">])</span><span class="o">.</span><span class="n">to_fp16</span><span class="p">()</span>
</pre></div>

    </div>
</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">learn</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">cbs</span><span class="o">=</span><span class="n">WandbCallback</span><span class="p">(</span><span class="n">log_model</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">log_preds</span><span class="o">=</span><span class="kc">False</span><span class="p">))</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">


<div class="output_html rendered_html output_subarea ">

    <div>
        <style>
            /* Turns off some styling */
            progress {
                /* gets rid of default border in Firefox and Opera. */
                border: none;
                /* Needs to be in here for Safari polyfill so background images work as expected. */
                background-size: auto;
            }
            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {
                background: #F44336;
            }
        </style>
      <progress value='0' class='' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>
      0.00% [0/1 00:00<00:00]
    </div>
    
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: left;">
      <th>epoch</th>
      <th>train_loss</th>
      <th>valid_loss</th>
      <th>accuracy</th>
      <th>perplexity</th>
      <th>bpc</th>
      <th>time</th>
    </tr>
  </thead>
  <tbody>
  </tbody>
</table><p>

    <div>
        <style>
            /* Turns off some styling */
            progress {
                /* gets rid of default border in Firefox and Opera. */
                border: none;
                /* Needs to be in here for Safari polyfill so background images work as expected. */
                background-size: auto;
            }
            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {
                background: #F44336;
            }
        </style>
      <progress value='2323' class='' max='5494' style='width:300px; height:20px; vertical-align: middle;'></progress>
      42.28% [2323/5494 29:55<40:50 1.9139]
    </div>
    
</div>

</div>

<div class="output_area">


<div class="output_html rendered_html output_subarea ">
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: left;">
      <th>epoch</th>
      <th>train_loss</th>
      <th>valid_loss</th>
      <th>accuracy</th>
      <th>perplexity</th>
      <th>bpc</th>
      <th>time</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>0</td>
      <td>1.703077</td>
      <td>1.629281</td>
      <td>0.525945</td>
      <td>5.100204</td>
      <td>2.350555</td>
      <td>1:12:49</td>
    </tr>
  </tbody>
</table>
</div>

</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">learn</span><span class="o">.</span><span class="n">recorder</span><span class="o">.</span><span class="n">plot_loss</span><span class="p">()</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">



<div class="output_png output_subarea ">
<img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAfh0lEQVR4nO3de3SddZ3v8fd33/fO/dY2TdomBaSFtkBba5FaEdQDiA4cuXlg9Hgcu+SooLNcnjIezzizdMajZ1wDM6jgbY2jgFjEKogI2gIqtxZKKfRGsZc0TZOmzT07+/Y7f+ydNG2TNi1J9pPk81orq89+9rP3/v5g99Nffs/v+T3mnENERLzLl+8CRETk5BTUIiIep6AWEfE4BbWIiMcpqEVEPC4wFm9aWVnp6urqxuKtRUQmpY0bNx5yzlUN9dyYBHVdXR0bNmwYi7cWEZmUzGzPcM9p6ENExOMU1CIiHqegFhHxuDEZoxYROR3JZJKGhgbi8Xi+SxlzkUiE2tpagsHgiF+joBaRvGtoaKCoqIi6ujrMLN/ljBnnHK2trTQ0NFBfXz/i12noQ0TyLh6PU1FRMalDGsDMqKioOO3fHBTUIuIJkz2k+51JOz0V1Hf9fidP7WjJdxkiIp7iqaD+zvpd/HGnglpExldbWxvf/va3T/t1V111FW1tbWNQ0bE8FdQBn5HK6EYGIjK+hgvqVCp10tf95je/obS0dKzKGuCpWR8+n5FRUIvIOFu9ejW7du3iwgsvJBgMEolEKCsrY9u2bezYsYNrrrmGffv2EY/Huf3221m1ahVwdLmMrq4urrzySlasWMGf//xnampqWLt2LdFodFTq81RQ+31GWrcGE5nS/uHXr/F6Y8eovud5M4v5+w+eP+zzX//619myZQubNm1i/fr1fOADH2DLli0DU+h++MMfUl5eTm9vL29/+9v58Ic/TEVFxTHvsXPnTu6//36+973vccMNN/DQQw9xyy23jEr93gtq9ahFJM+WLVt2zDznu+66i4cffhiAffv2sXPnzhOCur6+ngsvvBCAJUuWsHv37lGrx1tBbQpqkanuZD3f8VJQUDCwvX79ep588kmeffZZYrEYl1566ZDzoMPh8MC23++nt7d31Orx1MnEbI8631WIyFRTVFREZ2fnkM+1t7dTVlZGLBZj27ZtPPfcc+Ncndd61D4jnVFSi8j4qqio4JJLLmHBggVEo1GmT58+8NwVV1zBd7/7XebPn8+5557L8uXLx70+7wW1Rj5EJA/uu+++IfeHw2Eee+yxIZ/rH4eurKxky5YtA/u/8IUvjGptnhr68BmanicichxPBXXA59PJRBGR43gqqH26MlFE5ASeCmq/DzK64EVE5BgeC2oNfYiIHG9EQW1mpWa2xsy2mdlWM7t4LIrxGwpqEZHjjLRHfSfwW+fcPOACYOtYFKNLyEVkoigsLASgsbGR6667bshjLr30UjZs2PCWP+uU86jNrARYCfx3AOdcAki85U8eghZlEpGJZubMmaxZs2ZMP2MkPep6oAX4kZm9bGbfN7OC4w8ys1VmtsHMNrS0nNni/+pRi0i+rF69mrvvvnvg8Ve+8hW++tWvcvnll7N48WIWLlzI2rVrT3jd7t27WbBgAQC9vb3cdNNNzJ8/n2uvvXbU1vsYyZWJAWAx8Fnn3PNmdiewGvjy4IOcc/cC9wIsXbr0jNLWp0WZROSx1dD06ui+54yFcOXXT3rIjTfeyOc+9zk+/elPA/Dggw/y+OOPc9ttt1FcXMyhQ4dYvnw5H/rQh4a97+F3vvMdYrEYW7duZfPmzSxevHhUyh9JUDcADc6553OP15AN6lEX8Jmm54lIXlx00UU0NzfT2NhIS0sLZWVlzJgxg89//vM8/fTT+Hw+9u/fz8GDB5kxY8aQ7/H0009z2223AbBo0SIWLVo0KrWdMqidc01mts/MznXObQcuB14flU8/jt9npLTYh8jUdoqe71i6/vrrWbNmDU1NTdx444389Kc/paWlhY0bNxIMBqmrqxtyidOxNtJZH58Ffmpmm4ELgX8ai2I0Ri0i+XTjjTfywAMPsGbNGq6//nra29uZNm0awWCQdevWsWfPnpO+fuXKlQOLO23ZsoXNmzePSl0jWj3PObcJWDoqn3gSAZ9Psz5EJG/OP/98Ojs7qampobq6mptvvpkPfvCDLFy4kKVLlzJv3ryTvv7WW2/l4x//OPPnz2f+/PksWbJkVOry3jKn6lGLSB69+urRE5mVlZU8++yzQx7X1dUFZG9w27/EaTQa5YEHHhj1mjx1CXnAZ6R04wARkWN4Kqh1MlFE5ESeCuqA36dlTkWmKDdFzk+dSTu9FdQaoxaZkiKRCK2trZM+rJ1ztLa2EolETut1njuZmNJtyEWmnNraWhoaGjjT5ScmkkgkQm1t7Wm9xlNBrR61yNQUDAapr6/Pdxme5amhD7/fSCqoRUSO4amgVo9aROREHgvq7K24JvsJBRGR0+GxoM4uHahetYjIUZ4Kar8/G9SaSy0icpSnglo9ahGRE3kqqP2+bDm6jFxE5ChPBXVwYOhDF72IiPTzVFD7NfQhInICTwV1/xi1TiaKiBzlqaDuH6NWj1pE5ChPBXV/jzqphZlERAZ4Kqg1Ri0iciJPBXVQF7yIiJzAU0GtMWoRkRN5Kqg1Ri0iciJvBbWGPkRETuCtoM4NfahHLSJylKeCeuBkotb6EBEZ4KmgDvhzizJprQ8RkQGeCur+HnUipR61iEg/jwW1etQiIsfzVFAPLMqkMWoRkQGeCur+HrVmfYiIHOWpoNY8ahGREwVGcpCZ7QY6gTSQcs4tHZNiBm7FpR61iEi/EQV1znucc4fGrBKOzvpIaoxaRGSAx4Y+NOtDROR4Iw1qB/zOzDaa2aqhDjCzVWa2wcw2tLS0nFExRxdlUo9aRKTfSIN6hXNuMXAl8GkzW3n8Ac65e51zS51zS6uqqs6oGM36EBE50YiC2jm3P/dnM/AwsGwsivH7DJ9pHrWIyGCnDGozKzCzov5t4P3AlrEqKOD3kdQYtYjIgJHM+pgOPGxm/cff55z77VgVFPb7SGqtDxGRAacMaufcm8AF41ALAOGgj75Uerw+TkTE8zw1PQ8g5PfRl9LQh4hIP88FdTjoV1CLiAzivaAO+OhLauhDRKSfJ4M6oXnUIiIDPBjUfvqSCmoRkX6eC+pQQLM+REQG81xQhwOa9SEiMpj3gjqooBYRGcx7QR3wk1BQi4gM8GBQa4xaRGQwTwZ1XLM+REQGeC6oIyE/vbrgRURkgOeCOhYMkEhlSOtO5CIigAeDOhrKltSTSOW5EhERb/BgUGdXXtXwh4hIlueCOhb0A9CbUFCLiIAXgzqUDeoeBbWICODBoI4qqEVEjuG5oI71j1ErqEVEAE8GdX+PWrM+RETAg0HdP/ShWR8iIlmeC+rCcHboozOuHrWICHgwqEuiQQDae5N5rkRExBs8F9SRoJ9QwEeHglpEBPBgUAOURoPqUYuI5HgyqEsU1CIiAxTUIiIe59mgbutRUIuIgIeDWj1qEZEsbwZ1LKhZHyIiOd4M6miQzr4UqbTunSgi4tmgBujQ1YkiIt4Oao1Ti4icRlCbmd/MXjazR8ayIFBQi4gMdjo96tuBrWNVyGClMQW1iEi/EQW1mdUCHwC+P7blZPX3qNt6EuPxcSIinjbSHvW/Al8Ehp2GYWarzGyDmW1oaWl5S0UV959MVI9aROTUQW1mVwPNzrmNJzvOOXevc26pc25pVVXVWypKY9QiIkeNpEd9CfAhM9sNPABcZmY/GcuiwgE/kaBPQS0iwgiC2jl3h3Ou1jlXB9wE/ME5d8tYF1YaDWm9DxERPDqPGrTeh4hIv8DpHOycWw+sH5NKjlMSC9KmoBYR8W6PuqooTEtnX77LEBHJO88G9YziCE3tcZxz+S5FRCSvPB3Uvcm0FmYSkSnPs0E9vSQCwMGOeJ4rERHJL88G9YzibFA3tSuoRWRq82xQV+d61E3qUYvIFOfZoJ5WHAbgQJuCWkSmNs8GdTjgp7IwzIH23nyXIiKSV54NaoCZpREaNUYtIlOcp4O6uiTCgTb1qEVkavN4UEc5oB61iExxHg/qCF19KS3OJCJTmqeDurYsBsD+Ixr+EJGpy9NBPas8CsC+Iz15rkREJH88HdT9Pep9hxXUIjJ1eTqoy2JBCsMBBbWITGmeDmozY25VAbtauvNdiohI3ng6qAHOmVbEjoOd+S5DRCRvvB/U0wtp7uyjXTe6FZEpyvNB/bbphQDsbFavWkSmJs8H9TnTigDY2dyV50pERPLD80FdUxolGvRrnFpEpizPB7XPZ8yvLmLL/vZ8lyIikheeD2qAi2aXsbmhnWQ6k+9SRETG3QQJ6lL6Uhm2HdDwh4hMPRMkqMsAeGnvkTxXIiIy/iZEUM8siTC9OMzLCmoRmYImRFCbGUvmlPHcm4dxzuW7HBGRcTUhghrg0nOn0dQR57XGjnyXIiIyriZMUF82bxoBn/HzDfvyXYqIyLiaMEFdWRjm6kXV/OLl/cST6XyXIyIybiZMUAPc8PZZdMZTPLr5QL5LEREZN6cMajOLmNkLZvaKmb1mZv8wHoUN5eK5FcytLOC+F/bmqwQRkXE3kh51H3CZc+4C4ELgCjNbPrZlDc3M+Miy2Wzcc4TtTbr4RUSmhlMGtcvqX7oumPvJ2xy5Dy+pJRTw8W9/2JmvEkRExtWIxqjNzG9mm4Bm4Ann3PNDHLPKzDaY2YaWlpbRrnNAeUGIz7znbB7ZfIDfvdY0Zp8jIuIVIwpq51zaOXchUAssM7MFQxxzr3NuqXNuaVVV1WjXeYxbLz2LeTOK+N+/3EJ7r+78IiKT22nN+nDOtQHrgCvGppyRCfp9fPO6C2jtTvCVX72Wz1JERMbcSGZ9VJlZaW47CrwP2DbWhZ3KwtoSPvOes3n45f08qItgRGQSG0mPuhpYZ2abgRfJjlE/MrZljcxnLjubi+dWsPqhzTy7qzXf5YiIjImRzPrY7Jy7yDm3yDm3wDn3j+NR2EgE/T7u+egSZpXH+OsfPM/3nn6TdEaLNonI5DKhrkwcSnEkyJpPvZNLzq7ka7/ZylV3PsOvX2mkN6HLzEVkcrCxWDZ06dKlbsOGDaP+vifjnOPRVw/wzce3s6e1h0jQR11FAfOri1n5tkreO386RZHguNYkIjJSZrbRObd0yOcmS1D3y2Qcf97Vyi837ae5s49XG9o40pPEDObNKKY4EmDxnDKuX1JLYThAOOCnJKYAF5H8mlJBfbx0xrFpXxtPbj3I640dtPcm2dzQxuCh7GlFYYoiARbUlHDL8jnUVRTQEU9SHAlSWRjCzPLXABGZEk4W1IHxLma8+X3Zu8MsmVM2sK/hSA/rtjXTnUiTzjj2tHZzuDvB2k2NrN3UeMzriyIBzqsu5qxphZRGg5w3s5jzqoupqyjA51OAi8jYm/RBPZTashh/fXHdCfsbjvSwcc8RWrsSBP1GbzLNmy3dvLD7MDsOdtIRTw3MKikI+ZlVHmNZfTlzKgqYX13E0jnlhAIT/vysiHjMlAzq4dSWxagtiw37fF8qzc6DXbze2MHrBzrY3tTJAy/sI5HOABAL+Xnv/Ol8ZNlsls8t15CJiIwKBfVpCAf8LKgpYUFNycC+eDJNd1+Kl/a2sW57M7/e1MivXmlkWV05d1w1j4tml53kHUVETm3Sn0wcb72JNA+91MC3ntjB4e4El5xdwd9/8HzeNr0o36WJiIed7GSiBlRHWTTk55blc3jmi+/h766ax+uNHVx15zP882NbdRGOiJwRBfUYKQgHWLXyLJ7823fzXxfXcM9Tb7Lym+tYt70536WJyASjoB5jFYVhvnHdBfxs1XJKo0E+/qMX+Zv/2EBTezzfpYnIBKGgHifvmFvBI7etYPWV8/jTG4e44s6nWbtpP2NxjkBEJhcF9TgKB/x86t1n8ehtK6irKOD2BzbxhZ9vJpHK5Ls0EfEwBXUezK0q5KFb38ntl5/DQy818Dc/3qBbionIsBTUeeL3GZ9/39v4xocX8ec3DnHtt//Ey3uP5LssEfEgBXWe3fD2WfznJ95BPJHmhnue5Z6ndmncWkSOoaD2gIvPquCx21fy3vnT+efHtvGpn2ykI66hEBHJUlB7REksyLdvXsyXrz6PJ7c281f//ie2N3XmuywR8QAFtYeYGZ9YUc/9n1xOV1+Ka+7+E2s37c93WSKSZwpqD1pWX86jn13Bgppibn9gE1/+5RZSaU3hE5mqFNQeNa04wn2fXM4n31XPfz63h0/+eAPdfal8lyUieaCg9rCg38eXPnAeX7t2AU/taOGGe55lx0GNW4tMNQrqCeDmd8zh+x9bSlN7nKv/7Y98/5k3yWQ0hU9kqlBQTxCXzZvO459fycpzqvjqo1v52I9eoLlDCzuJTAUK6gmksjDM9z66hH+6diEv7j7Mpf9vPf9rzWZ2tXTluzQRGUMK6gnGzPhv75jNo7e9i3e/rYqfb9zHlXc+wx2/eJWWzr58lyciY0C34prgDnbE+Zffbeehl/aTzjhuu/wcbrvsbAJ+/RssMpHoVlyT2PTiCN+47gJ+e/u7mDejiLt+v5PL/uUpHnxxH2mdcBSZFNSjnkScczzx+kHuXvcGrzS0U1Ma5a8unMl1S2qZW1WY7/JE5CRO1qNWUE9CmYzjt681cf8Le/nTG4fIOFhxdiXXL61l5TlVlBWE8l2iiBxHQT2FNXfEeXDDPv7j2T20dPZhBsvrK7j6gmounltBfWUBZpbvMkWmvLcU1GY2C/gxMB1wwL3OuTtP9hoFtffEk2k27jnC82+28uvNB/jLoW4AphWFOXdGEWdVFbJ4Thk1pVHOn1lMJOjPc8UiU8tbDepqoNo595KZFQEbgWucc68P9xoFtbc559jV0sWLu7PBvaulmx0HO+nL3bvRDObNKOaqBTO45qIaZpXH8lyxyOQ3qkMfZrYW+Hfn3BPDHaOgnni6+1LsPdzDvsM9bGns4I87W3hpbxsA82YUsbCmhPnVxSyqLeG8mcXEQoE8VywyuYxaUJtZHfA0sMA51zHccQrqyWHf4R5+8vwetuxvZ9uBTlq7EwBEgj7On1nCotoSymMhSgtC1FcUUF9VwMySiMa8Rc7AqAS1mRUCTwFfc879YojnVwGrAGbPnr1kz549Z16xeFJzR5zn/nKY595sZduBDrYe6KQ3mT7mmOqSCPWVBUSCfqYXh6kuiVJRGKKiIMzi2aVMK47kqXoRb3vLQW1mQeAR4HHn3LdOdbx61FODc45EOsPB9j52Nneyu7WHl/ceoeFIL+29STrjSVq7E/R/xcxgWV05s8pjzCqLMas8ypyKGHMrCymNBdUTlyntZEF9yoFGy/7t+QGwdSQhLVOHmREO+JldEWN2Rf8Jx/pjjunqS3G4K8Gh7j7Wb2/hidcPsn57M4e6EsccFw36OW9mMTNKIkwvilAY9lNRGKaqKIzPoKY0xuzyGAVhP36fKdRlShnJrI8VwDPAq0D//aD+zjn3m+Feox61nEo8mabhSC+7D3Wzu7WbvYd72Hagk31Hemju7CPjHCf7aob8PmaWRphdUUAs6Kc4GuCsqkIKwgEW1JRw7vQioiFNMZSJ4y31qJ1zfwTUfZFRFQn6OXtaIWdPO/HS9kzG4YDD3QmaO+N096Vp7eqj4Ugv3Yns7ch6c0G/t7WHbR1xmo9bOTDoN86qKqSlsw+fz5hWFB6YHx7y+yiJBgkGfFSXRAgH/Jhla6ooCDGtOMy0Io2li3dojpV4js+X7RdUFWWHPk7FOUc64+hJpmnvSfJaYzsv7W1jV3MXC2tKSDtHS2cfT25tJuMcvYn0wJzx4RSGA5QVBIkE/FQVhakoDBML+sk4R3lBiLOmFVIUDhDw+zCgKBIgHPRTHAlQGA4QDfkpCAUG2iLyViioZcIzMwJ+o9jvozgSZFZ5jCsWVA97fP9J0O6+NIe7+4gns6EdT6Y51JWgsa2XNw910daTzP70JtjZ3IXPwDAOdydIjOCu8GWxIAtqSkimMyTT2XGcWMhPNOgnmc5wzvQi5s0oojeZpq0nSSzkpyAcIBzwUVkYJhL04Vx2hUQzqCmNamx+ilJQy5TTfxI0HPBTfgYLVKXSGfYc7iGRypBKO5KZDB29SRKpDG09Sbr6UqQyGV5paGff4R4iQT+RoI9U2tERT3Ewdwu1Z3YeInUaS9EWRQKUxULEk2nKC0KUxUJUFIaYVR4j5Pfh9xmpdIbW7gRmUF4QZnZ5jLOnFZJKZ2jqiFMeC1FTFqW6JEoooFWOJwoFtchpCvh9nDUKy8Z2xpM0tsUpjgYoiQbp6E2RTGfoSaRpONKD32ekM479bb0kUhl2t3bT1N5HWSxIW2+SI90JNje089iWpoG1x82gLBYikcrQ1Zcavg0+o7IwzKzyKPFkZuBzYyE/c6sKiAYDBHxGJOijKBLk/JnFdMZTtPUmONKTpDwWoqWrj+nFERbWlBAN+qkujRAN+omF/Or5jzIFtUieFEWCnDsjOPB48GX5584oGvH7pNKZgZ55MNez7t+/u7WH7U2dRII+asqiHO5OsP9IL3851M2e1h5auvoojgbw+3wcbI8zrTjC640dJNOOjngSyE6xHGoGjhnD7o8G/RiQyjgyzhHy+ygvDDGtKDIw/BMN+UmlHeGAj6DfR2dfkuqSKBnnMIyKwhB+nxFPpimKBIkn0/h9RsCXfa4wHKQ4EqCqKEwincFnxoySCEXhAF19KXxm9CbT2c8K+if0+QIFtcgEF/D7CAwxEzHg9w07s+Z09CbSvLT3CNOLIxRHA5RGQxxo76W2LEZTR5ydBzuJJ9McaI/TFU8RT6XpjGeDMuMcsVCAnkSKIz1JDnf30dWXoqk9Tntvdly+qy9NxjmiQT+/39qMGaTS7rSGhQbr/01ksEjQR01plMJwgFDARzQUYEZxmFDuHwm/Gf2/BJgZhblzBeGAj1DAT18q+9vGrLIY9VUFxIIBYmE/wXG65Z2CWkROKhryc8nZlcfsm1NRAGRPcNaURkfts/qv6zDL9qS7+1KURIO09yYJ53rpiVSG1xqPLjXU3BnHZ0ZPIk1HPElHb5KSaJC0c/gsG9otnX3sbu0m4yCRStPS2cf2pg5S6eyJ5cHBnso4EqeYFdSvJBrEDNJpRzjoo76ygJ9/6p2j9t+jn4JaRDxj8Nh29iRs9leFisKj0zQLwrDinMoTXjtanHMkcwHel0wPzPBJZxw7DnbS2BanL5U9gdzWkyDjIOA3EqkMpbHgKd79zCioRUQGMTNCASMU8FEYPjYia8vysza75ueIiHicglpExOMU1CIiHqegFhHxOAW1iIjHKahFRDxOQS0i4nEKahERj1NQi4h4nIJaRMTjFNQiIh53yruQn9GbmrUAe87w5ZXAoVEsx0vUtolrMrdvMrcNJk775jjnqoZ6YkyC+q0wsw3D3TJ9olPbJq7J3L7J3DaYHO3T0IeIiMcpqEVEPM6LQX1vvgsYQ2rbxDWZ2zeZ2waToH2eG6MWEZFjebFHLSIigyioRUQ8zjNBbWZXmNl2M3vDzFbnu56RMrMfmlmzmW0ZtK/czJ4ws525P8ty+83M7sq1cbOZLR70mo/ljt9pZh/LR1uOZ2azzGydmb1uZq+Z2e25/RO+fWYWMbMXzOyVXNv+Ibe/3syez7XhZ2YWyu0P5x6/kXu+btB73ZHbv93M/kt+WnQiM/Ob2ctm9kju8WRq224ze9XMNpnZhty+Cf+9HJZzLu8/gB/YBcwFQsArwHn5rmuEta8EFgNbBu37BrA6t70a+L+57auAxwADlgPP5/aXA2/m/izLbZd5oG3VwOLcdhGwAzhvMrQvV2NhbjsIPJ+r+UHgptz+7wK35rb/J/Dd3PZNwM9y2+flvq9hoD73Pfbn+/9drra/Be4DHsk9nkxt2w1UHrdvwn8vh21vvgvI/Qe7GHh80OM7gDvyXddp1F93XFBvB6pz29XA9tz2PcBHjj8O+Ahwz6D9xxznlR9gLfC+ydY+IAa8BLyD7BVsgdz+ge8l8DhwcW47kDvOjv+uDj4uz22qBX4PXAY8kqt1UrQtV8tQQT2pvpeDf7wy9FED7Bv0uCG3b6Ka7pw7kNtuAqbntodrp+fbn/t1+CKyPc9J0b7c0MAmoBl4gmyPsc05l8odMrjOgTbknm8HKvBo24B/Bb4IZHKPK5g8bQNwwO/MbKOZrcrtmxTfy6EE8l3AZOecc2Y2oedAmlkh8BDwOedch5kNPDeR2+ecSwMXmlkp8DAwL88ljQozuxpods5tNLNL813PGFnhnNtvZtOAJ8xs2+AnJ/L3cihe6VHvB2YNelyb2zdRHTSzaoDcn825/cO107PtN7Mg2ZD+qXPuF7ndk6Z9AM65NmAd2eGAUjPr78AMrnOgDbnnS4BWvNm2S4APmdlu4AGywx93MjnaBoBzbn/uz2ay/8guY5J9LwfzSlC/CJyTOysdIntC41d5rumt+BXQfwb5Y2THdvv3fzR3Fno50J77Ve1x4P1mVpY7U/3+3L68smzX+QfAVufctwY9NeHbZ2ZVuZ40ZhYlO/a+lWxgX5c77Pi29bf5OuAPLjuw+SvgptzMiXrgHOCF8WnF0Jxzdzjnap1zdWT/Lv3BOXczk6BtAGZWYGZF/dtkv09bmATfy2Hle5B80ED+VWRnFewCvpTvek6j7vuBA0CS7BjXJ8iO7/0e2Ak8CZTnjjXg7lwbXwWWDnqf/wG8kfv5eL7blatpBdmxwM3AptzPVZOhfcAi4OVc27YA/ye3fy7ZMHoD+DkQzu2P5B6/kXt+7qD3+lKuzduBK/PdtuPaeSlHZ31Mirbl2vFK7ue1/ryYDN/L4X50CbmIiMd5ZehDRESGoaAWEfE4BbWIiMcpqEVEPE5BLSLicQpqERGPU1CLiHjc/weO/s5va3DTzgAAAABJRU5ErkJggg==
"
>
</div>

</div>

</div>
</div>

</div>
    {% endraw %}

</div>