reformer_fastai/docs/experiment-configs.html

---

title: Experiment Configs


keywords: fastai
sidebar: home_sidebar



nb_path: "nbs/21_experiment-configs.ipynb"
---
<!--

#################################################
### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ###
#################################################
# file to edit: nbs/21_experiment-configs.ipynb
# command to build the docs after a change: nbdev_build_docs

-->

<div class="container" id="notebook-container">
        
    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

<div class="output_wrapper">
<div class="output">

<div class="output_area">


<div class="output_markdown rendered_html output_subarea ">
<h4 id="update_sig" class="doc_header"><code>update_sig</code><a href="https://github.com/arampacha/reformer_fastai/tree/master/reformer_fastai/configs.py#L21" class="source_link" style="float:right">[source]</a></h4><blockquote><p><code>update_sig</code>(<strong><code>d</code></strong>)</p>
</blockquote>
<p>Update signature of <code>f</code> from dict <code>d</code></p>

</div>

</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

<div class="output_wrapper">
<div class="output">

<div class="output_area">


<div class="output_markdown rendered_html output_subarea ">
<h2 id="ConfigBase" class="doc_header"><code>class</code> <code>ConfigBase</code><a href="https://github.com/arampacha/reformer_fastai/tree/master/reformer_fastai/configs.py#L34" class="source_link" style="float:right">[source]</a></h2><blockquote><p><code>ConfigBase</code>(<strong><code>verbose</code></strong>=<em><code>False</code></em>, <strong><code>warn</code></strong>=<em><code>True</code></em>, <strong>**<code>kwargs</code></strong>)</p>
</blockquote>
<p>Base class for Configs</p>

</div>

</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

<div class="output_wrapper">
<div class="output">

<div class="output_area">


<div class="output_markdown rendered_html output_subarea ">
<h2 id="SyntheticConfig" class="doc_header"><code>class</code> <code>SyntheticConfig</code><a href="https://github.com/arampacha/reformer_fastai/tree/master/reformer_fastai/configs.py#L95" class="source_link" style="float:right">[source]</a></h2><blockquote><p><code>SyntheticConfig</code>(<strong><code>vocab_sz</code></strong>=<em><code>128</code></em>, <strong><code>d_model</code></strong>=<em><code>256</code></em>, <strong><code>n_layers</code></strong>=<em><code>1</code></em>, <strong><code>n_heads</code></strong>=<em><code>4</code></em>, <strong><code>d_ff</code></strong>=<em><code>256</code></em>, <strong><code>attn_dropout</code></strong>=<em><code>0.0</code></em>, <strong><code>ff_dropout</code></strong>=<em><code>0.0</code></em>, <strong><code>emb_dropout</code></strong>=<em><code>0.0</code></em>, <strong><code>tie_weights</code></strong>=<em><code>True</code></em>, <strong><code>causal</code></strong>=<em><code>True</code></em>, <strong><code>pos_enc</code></strong>=<em><code>'absolute'</code></em>, <strong><code>max_seq_len</code></strong>=<em><code>1024</code></em>, <strong><code>axial_shape</code></strong>=<em><code>None</code></em>, <strong><code>axial_emb_dims</code></strong>=<em><code>None</code></em>, <strong><code>pad_idx</code></strong>=<em><code>None</code></em>, <strong><code>prenorm</code></strong>=<em><code>False</code></em>, <strong><code>attn_bias</code></strong>=<em><code>False</code></em>, <strong><code>bucket_size</code></strong>=<em><code>64</code></em>, <strong><code>use_lsh</code></strong>=<em><code>True</code></em>, <strong><code>n_hashes</code></strong>=<em><code>4</code></em>, <strong><code>seed</code></strong>=<em><code>123</code></em>) :: <a href="/reformer_fastai/experiment-configs.html#ConfigBase"><code>ConfigBase</code></a></p>
</blockquote>
<p>Config for Synthetic Experiment.
See <a href="https://arampacha.github.io/reformer_fastai/experiment.synthetic-task.html">https://arampacha.github.io/reformer_fastai/experiment.synthetic-task.html</a> for details</p>

</div>

</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">synt_config</span> <span class="o">=</span> <span class="n">SyntheticConfig</span><span class="p">(</span><span class="n">n_hashes</span><span class="o">=</span><span class="mi">8</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">foo</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">synt_config</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">

<div class="output_subarea output_stream output_stdout output_text">
<pre>Setting `n_hashes` = 8
Setting `seed` = 1
Parameter `foo` is not accepted by LSHLM. Skipped
</pre>
</div>
</div>

<div class="output_area">



<div class="output_text output_subarea output_execute_result">
<pre>LSHLM config 
--------------------
vocab_sz        128
d_model         256
n_layers        1
n_heads         4
d_ff            256
attn_dropout    0.0
ff_dropout      0.0
emb_dropout     0.0
tie_weights     True
causal          True
pos_enc         absolute
max_seq_len     1024
axial_shape     None
axial_emb_dims  None
pad_idx         None
prenorm         False
attn_bias       False
bucket_size     64
use_lsh         True
n_hashes        8
seed            1</pre>
</div>

</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">synt_config</span><span class="o">.</span><span class="n">d_model</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">



<div class="output_text output_subarea output_execute_result">
<pre>256</pre>
</div>

</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">synt_config</span><span class="o">.</span><span class="n">n_hashes</span> <span class="o">=</span> <span class="mi">4</span>
<span class="k">assert</span> <span class="n">synt_config</span><span class="o">.</span><span class="n">_d</span><span class="p">[</span><span class="s1">&#39;n_hashes&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="mi">4</span>
<span class="k">try</span><span class="p">:</span> <span class="n">synt_config</span><span class="o">.</span><span class="n">foo</span> <span class="o">=</span> <span class="mi">1</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> <span class="nb">print</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">

<div class="output_subarea output_stream output_stdout output_text">
<pre>LSHLM does not accept `foo` argument
</pre>
</div>
</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">synt_config</span><span class="p">[</span><span class="s1">&#39;n_layers&#39;</span><span class="p">]</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">



<div class="output_text output_subarea output_execute_result">
<pre>1</pre>
</div>

</div>

</div>
</div>

</div>
    {% endraw %}

<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<p>{% include note.html content='Config for envik8 needs to be updated when we diside on sequence length.' %}</p>

</div>
</div>
</div>
    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

<div class="output_wrapper">
<div class="output">

<div class="output_area">


<div class="output_markdown rendered_html output_subarea ">
<h2 id="TransformerLMConfigEnwik8" class="doc_header"><code>class</code> <code>TransformerLMConfigEnwik8</code><a href="https://github.com/arampacha/reformer_fastai/tree/master/reformer_fastai/configs.py#L129" class="source_link" style="float:right">[source]</a></h2><blockquote><p><code>TransformerLMConfigEnwik8</code>(<strong><code>vocab_sz</code></strong>=<em><code>256</code></em>, <strong><code>d_model</code></strong>=<em><code>1024</code></em>, <strong><code>n_layers</code></strong>=<em><code>3</code></em>, <strong><code>n_heads</code></strong>=<em><code>8</code></em>, <strong><code>d_ff</code></strong>=<em><code>4096</code></em>, <strong><code>attn_dropout</code></strong>=<em><code>0.1</code></em>, <strong><code>ff_dropout</code></strong>=<em><code>0.1</code></em>, <strong><code>emb_dropout</code></strong>=<em><code>0.1</code></em>, <strong><code>tie_weights</code></strong>=<em><code>True</code></em>, <strong><code>causal</code></strong>=<em><code>True</code></em>, <strong><code>pos_enc</code></strong>=<em><code>'axial'</code></em>, <strong><code>max_seq_len</code></strong>=<em><code>2048</code></em>, <strong><code>axial_shape</code></strong>=<em><code>(64, 32)</code></em>, <strong><code>axial_emb_dims</code></strong>=<em><code>None</code></em>, <strong><code>pad_idx</code></strong>=<em><code>None</code></em>, <strong><code>prenorm</code></strong>=<em><code>False</code></em>, <strong><code>attn_bias</code></strong>=<em><code>False</code></em>, <strong><code>shared_qk</code></strong>=<em><code>False</code></em>) :: <a href="/reformer_fastai/experiment-configs.html#ConfigBase"><code>ConfigBase</code></a></p>
</blockquote>
<p>Config for enwik8 Experiment.
See <a href="https://arampacha.github.io/reformer_fastai/experiment.enwik8-baseline.html">https://arampacha.github.io/reformer_fastai/experiment.enwik8-baseline.html</a> for details</p>

</div>

</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">TransformerLMConfigEnwik8</span><span class="p">()</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">



<div class="output_text output_subarea output_execute_result">
<pre>TransformerLM config 
--------------------
vocab_sz        256
d_model         1024
n_layers        3
n_heads         8
d_ff            4096
attn_dropout    0.1
ff_dropout      0.1
emb_dropout     0.1
tie_weights     True
causal          True
pos_enc         axial
max_seq_len     2048
axial_shape     (64, 32)
axial_emb_dims  None
pad_idx         None
prenorm         False
attn_bias       False
shared_qk       False</pre>
</div>

</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

<div class="output_wrapper">
<div class="output">

<div class="output_area">


<div class="output_markdown rendered_html output_subarea ">
<h2 id="ReversibleLMConfigEnwik8" class="doc_header"><code>class</code> <code>ReversibleLMConfigEnwik8</code><a href="https://github.com/arampacha/reformer_fastai/tree/master/reformer_fastai/configs.py#L160" class="source_link" style="float:right">[source]</a></h2><blockquote><p><code>ReversibleLMConfigEnwik8</code>(<strong><code>vocab_sz</code></strong>=<em><code>256</code></em>, <strong><code>d_model</code></strong>=<em><code>1024</code></em>, <strong><code>n_layers</code></strong>=<em><code>3</code></em>, <strong><code>n_heads</code></strong>=<em><code>8</code></em>, <strong><code>d_ff</code></strong>=<em><code>4096</code></em>, <strong><code>attn_dropout</code></strong>=<em><code>0.1</code></em>, <strong><code>ff_dropout</code></strong>=<em><code>0.1</code></em>, <strong><code>emb_dropout</code></strong>=<em><code>0.1</code></em>, <strong><code>tie_weights</code></strong>=<em><code>True</code></em>, <strong><code>causal</code></strong>=<em><code>True</code></em>, <strong><code>pos_enc</code></strong>=<em><code>'axial'</code></em>, <strong><code>max_seq_len</code></strong>=<em><code>2048</code></em>, <strong><code>axial_shape</code></strong>=<em><code>(64, 32)</code></em>, <strong><code>axial_emb_dims</code></strong>=<em><code>None</code></em>, <strong><code>pad_idx</code></strong>=<em><code>None</code></em>, <strong><code>prenorm</code></strong>=<em><code>True</code></em>, <strong><code>attn_bias</code></strong>=<em><code>False</code></em>, <strong><code>rev_thres</code></strong>=<em><code>0</code></em>) :: <a href="/reformer_fastai/experiment-configs.html#ConfigBase"><code>ConfigBase</code></a></p>
</blockquote>
<p>Config for enwik8 Experiment.
See <a href="https://arampacha.github.io/reformer_fastai/experiment.enwik8-reversible.html">https://arampacha.github.io/reformer_fastai/experiment.enwik8-reversible.html</a> for details</p>

</div>

</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">ReversibleLMConfigEnwik8</span><span class="p">(</span><span class="n">rev_thres</span><span class="o">=</span><span class="mi">128</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">



<div class="output_text output_subarea output_execute_result">
<pre>ReversibleLM config 
--------------------
vocab_sz        256
d_model         1024
n_layers        3
n_heads         8
d_ff            4096
attn_dropout    0.1
ff_dropout      0.1
emb_dropout     0.1
tie_weights     True
causal          True
pos_enc         axial
max_seq_len     2048
axial_shape     (64, 32)
axial_emb_dims  None
pad_idx         None
prenorm         True
attn_bias       False
rev_thres       128</pre>
</div>

</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

<div class="output_wrapper">
<div class="output">

<div class="output_area">


<div class="output_markdown rendered_html output_subarea ">
<h2 id="NHashesConfig" class="doc_header"><code>class</code> <code>NHashesConfig</code><a href="https://github.com/arampacha/reformer_fastai/tree/master/reformer_fastai/configs.py#L191" class="source_link" style="float:right">[source]</a></h2><blockquote><p><code>NHashesConfig</code>(<strong><code>vocab_sz</code></strong>=<em><code>256</code></em>, <strong><code>d_model</code></strong>=<em><code>1024</code></em>, <strong><code>n_layers</code></strong>=<em><code>3</code></em>, <strong><code>n_heads</code></strong>=<em><code>8</code></em>, <strong><code>d_ff</code></strong>=<em><code>4096</code></em>, <strong><code>attn_dropout</code></strong>=<em><code>0.1</code></em>, <strong><code>ff_dropout</code></strong>=<em><code>0.1</code></em>, <strong><code>emb_dropout</code></strong>=<em><code>0.1</code></em>, <strong><code>tie_weights</code></strong>=<em><code>True</code></em>, <strong><code>causal</code></strong>=<em><code>True</code></em>, <strong><code>pos_enc</code></strong>=<em><code>'axial'</code></em>, <strong><code>max_seq_len</code></strong>=<em><code>4096</code></em>, <strong><code>axial_shape</code></strong>=<em><code>None</code></em>, <strong><code>axial_emb_dims</code></strong>=<em><code>None</code></em>, <strong><code>pad_idx</code></strong>=<em><code>None</code></em>, <strong><code>prenorm</code></strong>=<em><code>False</code></em>, <strong><code>attn_bias</code></strong>=<em><code>False</code></em>, <strong><code>bucket_size</code></strong>=<em><code>64</code></em>, <strong><code>use_lsh</code></strong>=<em><code>True</code></em>, <strong><code>n_hashes</code></strong>=<em><code>2</code></em>, <strong><code>seed</code></strong>=<em><code>842</code></em>) :: <a href="/reformer_fastai/experiment-configs.html#ConfigBase"><code>ConfigBase</code></a></p>
</blockquote>
<p>Config for evaluating performance as function of <code>n_hashes</code>.
See <a href="https://arampacha.github.io/reformer_fastai/experiment.enwik8-n_hashes.html">https://arampacha.github.io/reformer_fastai/experiment.enwik8-n_hashes.html</a> for details</p>

</div>

</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">cfg</span> <span class="o">=</span> <span class="n">NHashesConfig</span><span class="p">()</span>
<span class="n">cfg</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">



<div class="output_text output_subarea output_execute_result">
<pre>LSHLM config 
--------------------
vocab_sz        256
d_model         1024
n_layers        3
n_heads         8
d_ff            4096
attn_dropout    0.1
ff_dropout      0.1
emb_dropout     0.1
tie_weights     True
causal          True
pos_enc         axial
max_seq_len     4096
axial_shape     None
axial_emb_dims  None
pad_idx         None
prenorm         False
attn_bias       False
bucket_size     64
use_lsh         True
n_hashes        2
seed            842</pre>
</div>

</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

<div class="output_wrapper">
<div class="output">

<div class="output_area">


<div class="output_markdown rendered_html output_subarea ">
<h2 id="NLayersConfig" class="doc_header"><code>class</code> <code>NLayersConfig</code><a href="https://github.com/arampacha/reformer_fastai/tree/master/reformer_fastai/configs.py#L225" class="source_link" style="float:right">[source]</a></h2><blockquote><p><code>NLayersConfig</code>(<strong><code>vocab_sz</code></strong>=<em><code>256</code></em>, <strong><code>d_model</code></strong>=<em><code>1024</code></em>, <strong><code>n_layers</code></strong>=<em><code>3</code></em>, <strong><code>n_heads</code></strong>=<em><code>8</code></em>, <strong><code>d_ff</code></strong>=<em><code>4096</code></em>, <strong><code>ff_chunks</code></strong>=<em><code>64</code></em>, <strong><code>attn_dropout</code></strong>=<em><code>0.1</code></em>, <strong><code>ff_dropout</code></strong>=<em><code>0.1</code></em>, <strong><code>emb_dropout</code></strong>=<em><code>0.1</code></em>, <strong><code>tie_weights</code></strong>=<em><code>True</code></em>, <strong><code>causal</code></strong>=<em><code>True</code></em>, <strong><code>pos_enc</code></strong>=<em><code>'axial'</code></em>, <strong><code>max_seq_len</code></strong>=<em><code>16384</code></em>, <strong><code>axial_shape</code></strong>=<em><code>None</code></em>, <strong><code>axial_emb_dims</code></strong>=<em><code>None</code></em>, <strong><code>pad_idx</code></strong>=<em><code>None</code></em>, <strong><code>prenorm</code></strong>=<em><code>True</code></em>, <strong><code>attn_bias</code></strong>=<em><code>False</code></em>, <strong><code>bucket_size</code></strong>=<em><code>64</code></em>, <strong><code>use_lsh</code></strong>=<em><code>True</code></em>, <strong><code>n_hashes</code></strong>=<em><code>8</code></em>, <strong><code>rev_thres</code></strong>=<em><code>0</code></em>, <strong><code>seed</code></strong>=<em><code>842</code></em>) :: <a href="/reformer_fastai/experiment-configs.html#ConfigBase"><code>ConfigBase</code></a></p>
</blockquote>
<p>Config for evaluating performance as function of <code>n_layers</code>.
See <a href="https://arampacha.github.io/reformer_fastai/experiment.enwik8-n_layers.html">https://arampacha.github.io/reformer_fastai/experiment.enwik8-n_layers.html</a> for details</p>

</div>

</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">cfg</span> <span class="o">=</span> <span class="n">NLayersConfig</span><span class="p">()</span>
<span class="n">cfg</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">



<div class="output_text output_subarea output_execute_result">
<pre>ReformerLM config 
--------------------
vocab_sz        256
d_model         1024
n_layers        3
n_heads         8
d_ff            4096
ff_chunks       64
attn_dropout    0.1
ff_dropout      0.1
emb_dropout     0.1
tie_weights     True
causal          True
pos_enc         axial
max_seq_len     16384
axial_shape     None
axial_emb_dims  None
pad_idx         None
prenorm         True
attn_bias       False
bucket_size     64
use_lsh         True
n_hashes        8
rev_thres       0
seed            842</pre>
</div>

</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

<div class="output_wrapper">
<div class="output">

<div class="output_area">


<div class="output_markdown rendered_html output_subarea ">
<h2 id="ReversibleTransformerConfigWMT" class="doc_header"><code>class</code> <code>ReversibleTransformerConfigWMT</code><a href="https://github.com/arampacha/reformer_fastai/tree/master/reformer_fastai/configs.py#L261" class="source_link" style="float:right">[source]</a></h2><blockquote><p><code>ReversibleTransformerConfigWMT</code>(<strong><code>enc_vocab_sz</code></strong>=<em><code>33708</code></em>, <strong><code>dec_vocab_sz</code></strong>=<em><code>33708</code></em>, <strong><code>n_enc_layers</code></strong>=<em><code>6</code></em>, <strong><code>n_dec_layers</code></strong>=<em><code>6</code></em>, <strong><code>n_heads</code></strong>=<em><code>8</code></em>, <strong><code>d_model</code></strong>=<em><code>512</code></em>, <strong><code>d_ff</code></strong>=<em><code>2048</code></em>, <strong><code>attn_dropout</code></strong>=<em><code>0.1</code></em>, <strong><code>ff_dropout</code></strong>=<em><code>0.1</code></em>, <strong><code>emb_dropout</code></strong>=<em><code>0.1</code></em>, <strong><code>tie_weights</code></strong>=<em><code>True</code></em>, <strong><code>shared_emb</code></strong>=<em><code>True</code></em>, <strong><code>pos_enc</code></strong>=<em><code>'fixed'</code></em>, <strong><code>max_seq_len</code></strong>=<em><code>256</code></em>, <strong><code>axial_shape</code></strong>=<em><code>(64, 32)</code></em>, <strong><code>axial_emb_dims</code></strong>=<em><code>None</code></em>, <strong><code>pad_idx</code></strong>=<em><code>None</code></em>, <strong><code>prenorm</code></strong>=<em><code>False</code></em>, <strong><code>attn_bias</code></strong>=<em><code>False</code></em>, <strong><code>comb_attn</code></strong>=<em><code>False</code></em>) :: <a href="/reformer_fastai/experiment-configs.html#ConfigBase"><code>ConfigBase</code></a></p>
</blockquote>
<p>Config for WMT Experiment.
See <a href="https://arampacha.github.io/reformer_fastai/">https://arampacha.github.io/reformer_fastai/</a></p>

</div>

</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">ReversibleTransformerConfigWMT</span><span class="p">(</span><span class="n">n_enc_layers</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">n_dec_layers</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">



<div class="output_text output_subarea output_execute_result">
<pre>ReversibleTransformer config 
--------------------
enc_vocab_sz    33708
dec_vocab_sz    33708
n_enc_layers    2
n_dec_layers    2
n_heads         8
d_model         512
d_ff            2048
attn_dropout    0.1
ff_dropout      0.1
emb_dropout     0.1
tie_weights     True
shared_emb      True
pos_enc         fixed
max_seq_len     256
axial_shape     (64, 32)
axial_emb_dims  None
pad_idx         None
prenorm         False
attn_bias       False
comb_attn       False</pre>
</div>

</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

<div class="output_wrapper">
<div class="output">

<div class="output_area">


<div class="output_markdown rendered_html output_subarea ">
<h2 id="TransformerConfigWMT" class="doc_header"><code>class</code> <code>TransformerConfigWMT</code><a href="https://github.com/arampacha/reformer_fastai/tree/master/reformer_fastai/configs.py#L294" class="source_link" style="float:right">[source]</a></h2><blockquote><p><code>TransformerConfigWMT</code>(<strong><code>enc_vocab_sz</code></strong>=<em><code>33708</code></em>, <strong><code>dec_vocab_sz</code></strong>=<em><code>33708</code></em>, <strong><code>n_enc_layers</code></strong>=<em><code>6</code></em>, <strong><code>n_dec_layers</code></strong>=<em><code>6</code></em>, <strong><code>n_heads</code></strong>=<em><code>8</code></em>, <strong><code>d_model</code></strong>=<em><code>512</code></em>, <strong><code>d_ff</code></strong>=<em><code>2048</code></em>, <strong><code>attn_dropout</code></strong>=<em><code>0.1</code></em>, <strong><code>ff_dropout</code></strong>=<em><code>0.1</code></em>, <strong><code>emb_dropout</code></strong>=<em><code>0.1</code></em>, <strong><code>tie_weights</code></strong>=<em><code>True</code></em>, <strong><code>shared_emb</code></strong>=<em><code>True</code></em>, <strong><code>pos_enc</code></strong>=<em><code>'fixed'</code></em>, <strong><code>max_seq_len</code></strong>=<em><code>256</code></em>, <strong><code>axial_shape</code></strong>=<em><code>(64, 32)</code></em>, <strong><code>axial_emb_dims</code></strong>=<em><code>None</code></em>, <strong><code>pad_idx</code></strong>=<em><code>None</code></em>, <strong><code>prenorm</code></strong>=<em><code>False</code></em>, <strong><code>attn_bias</code></strong>=<em><code>False</code></em>, <strong><code>comb_attn</code></strong>=<em><code>True</code></em>) :: <a href="/reformer_fastai/experiment-configs.html#ConfigBase"><code>ConfigBase</code></a></p>
</blockquote>
<p>Config for WMT Experiment.
See <a href="https://arampacha.github.io/reformer_fastai/">https://arampacha.github.io/reformer_fastai/</a></p>

</div>

</div>

</div>
</div>

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">

</div>
    {% endraw %}

    {% raw %}
    
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">TransformerConfigWMT</span><span class="p">(</span><span class="n">n_enc_layers</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">n_dec_layers</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">



<div class="output_text output_subarea output_execute_result">
<pre>Transformer config 
--------------------
enc_vocab_sz    33708
dec_vocab_sz    33708
n_enc_layers    1
n_dec_layers    1
n_heads         8
d_model         512
d_ff            2048
attn_dropout    0.1
ff_dropout      0.1
emb_dropout     0.1
tie_weights     True
shared_emb      True
pos_enc         fixed
max_seq_len     256
axial_shape     (64, 32)
axial_emb_dims  None
pad_idx         None
prenorm         False
attn_bias       False
comb_attn       True</pre>
</div>

</div>

</div>
</div>

</div>
    {% endraw %}

</div>