ds4se/docs/exp.i.html

---

title: Exploration of your data

keywords: fastai
sidebar: home_sidebar

summary: "This module comprises all the statistical and inference techniques to describe the inner properties of software data. The submodules might include:"
---
<!--

#################################################
### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ###
#################################################
# file to edit: nbs/01_exp.i.ipynb
# command to build the docs after a change: nbdev_build_docs

-->

<div class="container" id="notebook-container">
    
<div class="cell border-box-sizing code_cell rendered">

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># Imports</span>
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
<span class="kn">import</span> <span class="nn">sentencepiece</span> <span class="k">as</span> <span class="nn">sp</span>
<span class="kn">import</span> <span class="nn">dit</span>

<span class="kn">from</span> <span class="nn">collections</span> <span class="k">import</span> <span class="n">Counter</span>
<span class="kn">from</span> <span class="nn">scipy.stats</span> <span class="k">import</span> <span class="n">sem</span><span class="p">,</span> <span class="n">t</span>
<span class="kn">from</span> <span class="nn">numpy</span> <span class="k">import</span> <span class="n">mean</span>
<span class="kn">from</span> <span class="nn">numpy</span> <span class="k">import</span> <span class="n">std</span>
<span class="kn">import</span> <span class="nn">statistics</span> <span class="k">as</span> <span class="nn">stat</span>
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>


<span class="c1"># TODO: Remove when mongo call is implemented</span>
<span class="kn">import</span> <span class="nn">os</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">nbdev.showdoc</span> <span class="k">import</span> <span class="o">*</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h1 id="MOVED-NECESSARY-PIECES-TO-08">MOVED NECESSARY PIECES TO 08<a class="anchor-link" href="#MOVED-NECESSARY-PIECES-TO-08">&#182;</a></h1>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># TODO: Replace with actual mongo call</span>
<span class="k">def</span> <span class="nf">simulate_getting_dataframes_from_mongo</span><span class="p">():</span>
    <span class="n">corpus_data</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;file_name&#39;</span><span class="p">:</span> <span class="p">[],</span> <span class="s1">&#39;data_type&#39;</span><span class="p">:</span> <span class="p">[],</span> <span class="s1">&#39;contents&#39;</span><span class="p">:</span> <span class="p">[]}</span>
    <span class="n">path</span> <span class="o">=</span> <span class="s2">&quot;./requirements&quot;</span>
    <span class="k">for</span> <span class="n">file</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">listdir</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
        <span class="n">corpus_data</span><span class="p">[</span><span class="s1">&#39;file_name&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">file</span><span class="p">)</span>
        <span class="n">corpus_data</span><span class="p">[</span><span class="s1">&#39;data_type&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">&#39;req&#39;</span><span class="p">)</span>
        <span class="k">with</span> <span class="nb">open</span> <span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">file</span><span class="p">),</span> <span class="s2">&quot;r&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
            <span class="n">corpus_data</span><span class="p">[</span><span class="s1">&#39;contents&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>
    <span class="n">path</span> <span class="o">=</span> <span class="s2">&quot;./source_code&quot;</span>
    <span class="k">for</span> <span class="n">file</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">listdir</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
        <span class="n">corpus_data</span><span class="p">[</span><span class="s1">&#39;file_name&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">file</span><span class="p">)</span>
        <span class="n">corpus_data</span><span class="p">[</span><span class="s1">&#39;data_type&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">&#39;src&#39;</span><span class="p">)</span>
        <span class="k">with</span> <span class="nb">open</span> <span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">file</span><span class="p">),</span> <span class="s2">&quot;r&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
            <span class="n">corpus_data</span><span class="p">[</span><span class="s1">&#39;contents&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>
    <span class="n">path</span> <span class="o">=</span> <span class="s2">&quot;./tests&quot;</span>
    <span class="k">for</span> <span class="n">file</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">listdir</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
        <span class="n">corpus_data</span><span class="p">[</span><span class="s1">&#39;file_name&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">file</span><span class="p">)</span>
        <span class="n">corpus_data</span><span class="p">[</span><span class="s1">&#39;data_type&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">&#39;test&#39;</span><span class="p">)</span>
        <span class="k">with</span> <span class="nb">open</span> <span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">file</span><span class="p">),</span> <span class="s2">&quot;r&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
            <span class="n">corpus_data</span><span class="p">[</span><span class="s1">&#39;contents&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>
    <span class="n">corpus_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">data</span> <span class="o">=</span> <span class="n">corpus_data</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">corpus_df</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># def df_to_txt_file(df, output, cols):</span>
<span class="c1">#     &quot;&quot;&quot;Converts a dataframe into a text file that SentencePiece can use to train a BPE model&quot;&quot;&quot;</span>
<span class="c1">#     if cols is None: cols = list(df.columns)</span>
<span class="c1">#     merged_df = pd.concat([df[col] for col in cols])</span>
    
<span class="c1">#     with open(output + &#39;_text.txt&#39;, &#39;w&#39;) as f:</span>
<span class="c1">#         f.write(&#39;\n&#39;.join(list(merged_df)))</span>
<span class="c1">#     return output + &#39;_text.txt&#39;</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># def gen_sp_model(df, output, model_name, cols=None):</span>
<span class="c1">#     &quot;&quot;&quot;Trains a SentencePiece BPE model from a pandas dataframe&quot;&quot;&quot;</span>
<span class="c1">#     fname = df_to_txt_file(df, output, cols)</span>
<span class="c1">#     sp.SentencePieceTrainer.train(f&#39;--input={fname} --model_prefix={output + model_name} --hard_vocab_limit=false --model_type=bpe&#39;)</span>
<span class="c1">#     return output + model_name</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">def</span> <span class="nf">encode_text</span><span class="p">(</span><span class="n">text</span><span class="p">,</span> <span class="n">model_prefix</span><span class="p">):</span>
    <span class="sd">&#39;&#39;&#39;Encodes text using a pre-trained sp model, returns the occurrences of each token in the text&#39;&#39;&#39;</span>
    <span class="n">sp_processor</span> <span class="o">=</span> <span class="n">sp</span><span class="o">.</span><span class="n">SentencePieceProcessor</span><span class="p">()</span>
    <span class="n">sp_processor</span><span class="o">.</span><span class="n">Load</span><span class="p">(</span><span class="n">f</span><span class="s2">&quot;</span><span class="si">{model_prefix}</span><span class="s2">.model&quot;</span><span class="p">)</span>
    <span class="n">token_counts</span> <span class="o">=</span> <span class="n">Counter</span><span class="p">()</span>
    <span class="n">encoding</span> <span class="o">=</span> <span class="n">sp_processor</span><span class="o">.</span><span class="n">encode_as_pieces</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
    <span class="k">for</span> <span class="n">piece</span> <span class="ow">in</span> <span class="n">encoding</span><span class="p">:</span>
        <span class="n">token_counts</span><span class="p">[</span><span class="n">piece</span><span class="p">]</span> <span class="o">+=</span> <span class="mi">1</span>
    <span class="k">return</span> <span class="n">token_counts</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">def</span> <span class="nf">dit_shannon</span><span class="p">(</span><span class="n">token_counts</span><span class="p">):</span>
    <span class="sd">&#39;&#39;&#39;Takes in a counter object of token occurrences, computes the entropy of the corpus that produced it&#39;&#39;&#39;</span>
    <span class="n">num_tokens</span> <span class="o">=</span> <span class="mi">0</span>
    <span class="k">for</span> <span class="n">token</span> <span class="ow">in</span> <span class="n">token_counts</span><span class="p">:</span>
        <span class="n">num_tokens</span> <span class="o">+=</span> <span class="n">token_counts</span><span class="p">[</span><span class="n">token</span><span class="p">]</span>
    <span class="n">outcomes</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">token_counts</span><span class="o">.</span><span class="n">elements</span><span class="p">()))</span>
    <span class="n">frequencies</span> <span class="o">=</span> <span class="p">[]</span>
    <span class="k">for</span> <span class="n">token</span> <span class="ow">in</span> <span class="n">token_counts</span><span class="p">:</span>
        <span class="n">frequencies</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">token_counts</span><span class="p">[</span><span class="n">token</span><span class="p">])</span><span class="o">/</span><span class="n">num_tokens</span><span class="p">)</span>
    <span class="n">d</span> <span class="o">=</span> <span class="n">dit</span><span class="o">.</span><span class="n">ScalarDistribution</span><span class="p">(</span><span class="n">outcomes</span><span class="p">,</span> <span class="n">frequencies</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">dit</span><span class="o">.</span><span class="n">shannon</span><span class="o">.</span><span class="n">entropy</span><span class="p">(</span><span class="n">d</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">def</span> <span class="nf">entropies_of_df_entries</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">col</span><span class="p">,</span> <span class="n">model_prefix</span><span class="p">):</span>
    <span class="sd">&#39;&#39;&#39;Returns a list of the entropies of each entry in a dataframe column&#39;&#39;&#39;</span>
    <span class="n">entropies</span> <span class="o">=</span> <span class="p">[]</span>
    <span class="k">for</span> <span class="n">data</span> <span class="ow">in</span> <span class="n">df</span><span class="p">[</span><span class="n">col</span><span class="p">]:</span>
        <span class="n">token_counts</span><span class="o">=</span> <span class="n">encode_text</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">model_prefix</span><span class="p">)</span>
        <span class="n">entropies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dit_shannon</span><span class="p">(</span><span class="n">token_counts</span><span class="p">))</span>
    <span class="k">return</span> <span class="n">entropies</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># TODO: Finish this such that is finds the entropy of the entire corpus \</span>
<span class="c1">#       and preserves the individual token frequencies so that we can   \</span>
<span class="c1">#      compute the most common tokens</span>

<span class="c1"># def entropy_of_whole_corpus(df, col, model_prefix):</span>
<span class="c1">#     &#39;&#39;&#39;Returns a dictionary of the entropies of each token in a dataframe corpus&#39;&#39;&#39;</span>
<span class="c1">#     entropies = {}</span>
<span class="c1">#     token_counts = encode_text(pd.concat[col], model_prefix)</span>
<span class="c1">#     entropies.append(dit_shannon(token_counts))</span>
<span class="c1">#     return entropies</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># TODO: Do we need this function?</span>
<span class="kn">import</span> <span class="nn">math</span>
<span class="k">def</span> <span class="nf">manual_shannon</span><span class="p">(</span><span class="n">token_freqs</span><span class="p">):</span>
    <span class="nb">sum</span> <span class="o">=</span> <span class="mi">0</span>
    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">token_freqs</span><span class="p">:</span>
        <span class="nb">sum</span> <span class="o">+=</span> <span class="n">i</span> <span class="o">*</span> <span class="n">math</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="mi">1</span><span class="o">/</span><span class="n">i</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
    <span class="k">return</span> <span class="nb">sum</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># TODO: Do we need this function?</span>
<span class="k">def</span> <span class="nf">sort_token_data</span><span class="p">(</span><span class="n">token_data</span><span class="p">):</span>
    <span class="k">return</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">token_data</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="p">,</span>  <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">1</span><span class="p">][</span><span class="s2">&quot;Occurrences&quot;</span><span class="p">])</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h1 id="EXPLORATORY-ANALYSIS">EXPLORATORY ANALYSIS<a class="anchor-link" href="#EXPLORATORY-ANALYSIS">&#182;</a></h1>
</div>
</div>
</div>
<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h2 id="LIBest-Corpus">LIBest Corpus<a class="anchor-link" href="#LIBest-Corpus">&#182;</a></h2>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># Create a dataframe of the requirements, source code and test case data for LIBest</span>
<span class="c1"># Create a sentencepiece model using the entire LIBest corpus</span>
<span class="n">LIB_corpus_df</span> <span class="o">=</span> <span class="n">simulate_getting_dataframes_from_mongo</span><span class="p">()</span>
<span class="n">LIB_model</span> <span class="o">=</span> <span class="n">gen_sp_model</span><span class="p">(</span><span class="n">LIB_corpus_df</span><span class="p">,</span> <span class="n">output</span><span class="o">=</span><span class="s1">&#39;LIBest&#39;</span><span class="p">,</span> <span class="n">model_name</span><span class="o">=</span><span class="s1">&#39;_sp_bpe_modal&#39;</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;contents&#39;</span><span class="p">])</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h2 id="Looking-at-Individual-Files">Looking at Individual Files<a class="anchor-link" href="#Looking-at-Individual-Files">&#182;</a></h2>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># Use the model to compute each file&#39;s entropy</span>
<span class="n">LIB_entropies</span> <span class="o">=</span> <span class="n">entropies_of_df_entries</span><span class="p">(</span><span class="n">LIB_corpus_df</span><span class="p">,</span> <span class="s1">&#39;contents&#39;</span><span class="p">,</span> <span class="n">LIB_model</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># Calculate metrics on the LIBest corpus entropies</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Max entropy:&quot;</span><span class="p">,</span> <span class="nb">max</span><span class="p">(</span><span class="n">LIB_entropies</span><span class="p">))</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Min entropy:&quot;</span><span class="p">,</span> <span class="nb">min</span><span class="p">(</span><span class="n">LIB_entropies</span><span class="p">))</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Average entropy:&quot;</span><span class="p">,</span> <span class="n">mean</span><span class="p">(</span><span class="n">LIB_entropies</span><span class="p">))</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Median entropy:&quot;</span><span class="p">,</span> <span class="n">stat</span><span class="o">.</span><span class="n">median</span><span class="p">(</span><span class="n">LIB_entropies</span><span class="p">))</span>

<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Entropy Standard Deviation:&quot;</span><span class="p">,</span> <span class="n">std</span><span class="p">(</span><span class="n">LIB_entropies</span><span class="p">))</span>

<span class="n">confidence</span> <span class="o">=</span> <span class="mf">0.95</span>
<span class="n">n</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">LIB_entropies</span><span class="p">)</span>
<span class="n">m</span> <span class="o">=</span> <span class="n">mean</span><span class="p">(</span><span class="n">LIB_entropies</span><span class="p">)</span>
<span class="n">std_err</span> <span class="o">=</span> <span class="n">sem</span><span class="p">(</span><span class="n">LIB_entropies</span><span class="p">)</span>
<span class="n">h</span> <span class="o">=</span> <span class="n">std_err</span> <span class="o">*</span> <span class="n">t</span><span class="o">.</span><span class="n">ppf</span><span class="p">((</span><span class="mi">1</span> <span class="o">+</span> <span class="n">confidence</span><span class="p">)</span> <span class="o">/</span> <span class="mi">2</span><span class="p">,</span> <span class="n">n</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span>

<span class="n">start</span> <span class="o">=</span> <span class="n">m</span> <span class="o">-</span> <span class="n">h</span>
<span class="n">end</span> <span class="o">=</span> <span class="n">m</span> <span class="o">+</span> <span class="n">h</span>
<span class="nb">print</span><span class="p">(</span><span class="n">f</span><span class="s2">&quot;95</span><span class="si">% o</span><span class="s2">f entropies fall within </span><span class="si">{start}</span><span class="s2"> and </span><span class="si">{end}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">

<div class="output_subarea output_stream output_stdout output_text">
<pre>Max entropy: 8.737176307586141
Min entropy: 3.979797585487148
Average entropy: 6.910210324792078
Median entropy: 7.086176924292342
Entropy Standard Deviation: 1.0693292402540278
95% of entropies fall within 6.680984141366826 and 7.1394365082173294
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># Create a histogram of the entropy distribution</span>
<span class="n">plt</span><span class="o">.</span><span class="n">hist</span><span class="p">(</span><span class="n">LIB_entropies</span><span class="p">,</span> <span class="n">bins</span> <span class="o">=</span> <span class="mi">20</span><span class="p">)</span>
<span class="n">plt</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="s2">&quot;Num Files&quot;</span><span class="p">)</span>
<span class="n">plt</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="s2">&quot;Entropy&quot;</span><span class="p">)</span>
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">



<div class="output_png output_subarea ">
<img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAX4AAAEGCAYAAABiq/5QAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz
AAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0
dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAQEklEQVR4nO3df5BdZX3H8fdHoiOhQVR2nEJMQ61D
x1FUZkcRrFpAiw2CQ9spjDhonaY/EX+0DrYd0bF/UH+0an/AZADBSqE1Iv7qUBjBqlNlTAKDQLBa
jRhAE8aW34rRb/+4N7qEzeaS3XNOdp/3a2Znzz3n7H2+JwmffXjuOc+TqkKS1I7HDV2AJKlfBr8k
Ncbgl6TGGPyS1BiDX5Ias2zoAiZx8MEH1+rVq4cuQ5IWlY0bN95dVVO77l8Uwb969Wo2bNgwdBmS
tKgk+c5s+x3qkaTGGPyS1BiDX5IaY/BLUmMMfklqjMEvSY3pLPiTXJRkW5KbZ+x7b5LbktyU5BNJ
DuqqfUnS7Lrs8V8MnLDLvmuAZ1fVEcB/A2/vsH1J0iw6C/6q+gLwg132XV1VO8YvvwKs7Kp9SdLs
hnxy9/eAf93dwSRrgbUAq1at6qsmqQmrz/7sIO1uOXfNIO3qkQb5cDfJXwI7gEt3d05Vrauq6aqa
npp61FQTkqS91HuPP8nrgBOB48p1HyWpd70Gf5ITgLcBL62qB/tsW5I00uXtnJcBXwYOT7I1yRuA
fwBWANckuTHJ+V21L0maXWc9/qo6bZbdF3bVniRpMj65K0mNMfglqTEGvyQ1xuCXpMYY/JLUGINf
khpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLUmMMfklqjMEvSY0x+CWp
MQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mN6Sz4k1yUZFuSm2fse0qSa5J8Y/z9yV21
L0maXZc9/ouBE3bZdzbwuap6JvC58WtJUo86C/6q+gLwg112nwxcMt6+BHh1V+1LkmbX9xj/06rq
rvH294Cn7e7EJGuTbEiyYfv27f1UJ0kNGOzD3aoqoOY4vq6qpqtqempqqsfKJGlp6zv4v5/kFwHG
37f13L4kNa/v4P8UcMZ4+wzgkz23L0nN6/J2zsuALwOHJ9ma5A3AucDLk3wDOH78WpLUo2VdvXFV
nbabQ8d11aYkac98cleSGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtS
Ywx+SWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLUmMMfklqjMEvSY0x+CWpMQa/JDXG
4Jekxhj8ktQYg1+SGmPwS1JjBgn+JG9OckuSm5NcluSJQ9QhSS3qPfiTHAq8EZiuqmcD+wGn9l2H
JLVqqKGeZcD+SZYBy4E7B6pDkprTe/BX1R3A+4DbgbuAe6rq6l3PS7I2yYYkG7Zv3953mZK0ZA0x
1PNk4GTgMOAQ4IAkp+96XlWtq6rpqpqemprqu0xJWrKGGOo5Hvh2VW2vqh8DVwBHD1CHJDVpiOC/
HTgqyfIkAY4DNg9QhyQ1aYgx/uuB9cAm4GvjGtb1XYcktWrZEI1W1TnAOUO0LUmt88ldSWrMHoM/
yXuSHJjk8Uk+l2T7bHfhSJIWh0l6/K+oqnuBE4EtwK8Af95lUZKk7kwS/Ds/B1gDfKyq7umwHklS
xyb5cPczSW4DHgL+KMkU8MNuy5IkdWWPPf6qOpvRA1bT4weuHmT05K0kaRGa5MPd5cAfA+eNdx0C
THdZlCSpO5OM8X8YeJifT6twB/DXnVUkSerUJMH/jKp6D/BjgKp6EEinVUmSOjNJ8D+cZH+gAJI8
A/hRp1VJkjozyV095wBXAU9PcilwDPC6LouSJHVnj8FfVdck2QQcxWiI56yqurvzyiRJndht8Cc5
cpddd42/r0qyqqo2dVeWJKkrc/X43z/HsQKOXeBaJEk92G3wV9Wv91mIJKkfcw31HFtV1yY5Zbbj
VXVFd2VJkroy11DPS4FrgVfNcqwYrZUrSVpk5hrqOWf8/fX9lSNJ6tpuH+BKcvGM7TN6qUaS1Lm5
ntx97ozts7ouRJLUj7mCv3qrQpLUm7k+3F2Z5EOMntbduf0zVfXGTiuTJHViruCfua7uhq4LkST1
Y667ei7psxBJUj8mmZZZkrSEDBL8SQ5Ksj7JbUk2J3nREHVIUosmmY+/Cx8Erqqq307yBGD5QHVI
UnP2GPxJDgPOBFbPPL+qTtqbBpM8CXgJ48VcquphRmv6SpJ6MEmP/0rgQuDTwE8XoM3DgO3Ah5M8
F9jIaHGXB2aelGQtsBZg1apVC9CstO9ZffZnhy5hUZnPn9eWc9csYCWL2yRj/D+sqg9V1XVV9Z87
v+bR5jLgSOC8qno+8ABw9q4nVdW6qpququmpqal5NCdJmmmSHv8Hk5wDXM2MRdbnsQLXVmBrVV0/
fr2eWYJfktSNSYL/OcBrGa24tXOoZ69X4Kqq7yX5bpLDq+rrwHHArXvzXpKkx26S4P8d4JfHH8Iu
lDOBS8d39HwLcOpnSerJJMF/M3AQsG2hGq2qG4HphXo/SdLkJgn+g4DbknyVR47x79XtnJKkYU0S
/Od0XoUkqTd7DP553ropSdrHTPLk7n38fFGWJwCPBx6oqgO7LEyS1I1Jevwrdm4nCXAycFSXRUmS
uvOYZueskSuB3+ioHklSxyYZ6jllxsvHMboN84edVSRJ6tQkd/W8asb2DmALo+EeSdIiNMkYv0/V
StISstvgT/KOOX6uqurdHdQjSerYXD3+B2bZdwDwBuCpgMEvSYvQboO/qt6/czvJCuAsRpOpXQ68
f3c/J0nat805xp/kKcBbgNcAlwBHVtX/9lGY2jPU6kqu6qQuzXeVtS7+jc01xv9e4BRgHfCcqrp/
wVuXJPVurge43gocAvwVcGeSe8df9yW5t5/yJEkLba4x/sf0VK8kaXEw3CWpMQa/JDXG4Jekxhj8
ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1ZrDgT7JfkhuSfGaoGiSpRUP2+M8CNg/YviQ1
aZDgT7ISWANcMET7ktSyPS623pEPAG8DVuzuhCRrgbUAq1at6qksLVbzXexisbW7WC3WP6/FWvfu
9N7jT3IisK2qNs51XlWtq6rpqpqemprqqTpJWvqGGOo5BjgpyRZG6/cem+SjA9QhSU3qPfir6u1V
tbKqVgOnAtdW1el91yFJrfI+fklqzFAf7gJQVZ8HPj9kDZLUGnv8ktQYg1+SGmPwS1JjDH5JaozB
L0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwadq0e7N5+FH7acu2YBK5GWhqW2mMp82OOX
pMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLUmMMfklq
TO/Bn+TpSa5LcmuSW5Kc1XcNktSyIaZl3gG8tao2JVkBbExyTVXdOkAtktSc3nv8VXVXVW0ab98H
bAYO7bsOSWrVoAuxJFkNPB+4fpZja4G1AKtWreq1rsXOBSckzWWwD3eT/ALwceBNVXXvrseral1V
TVfV9NTUVP8FStISNUjwJ3k8o9C/tKquGKIGSWrVEHf1BLgQ2FxVf9t3+5LUuiF6/McArwWOTXLj
+Os3B6hDkprU+4e7VfUlIH23K0ka8cldSWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BL
UmMMfklqjMEvSY0x+CWpMQa/JDVm0BW4+jCf1ai2nLtmsLYlqSv2+CWpMQa/JDXG4Jekxhj8ktQY
g1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhozSPAnOSHJ15N8M8nZQ9Qg
Sa3qPfiT7Af8I/BK4FnAaUme1XcdktSqIXr8LwC+WVXfqqqHgcuBkweoQ5KaNMRCLIcC353xeivw
wl1PSrIWWDt+eX+Sr8+jzYOBux/rD+Vv5tHivmWvrn8J8fq9/kV7/fPMoV+abec+uwJXVa0D1i3E
eyXZUFXTC/Fei5HX7/V7/e1e/2yGGOq5A3j6jNcrx/skST0YIvi/CjwzyWFJngCcCnxqgDokqUm9
D/VU1Y4kfwr8B7AfcFFV3dJxswsyZLSIef1t8/r1CKmqoWuQJPXIJ3clqTEGvyQ1pongT7JfkhuS
fGboWvqWZEuSryW5McmGoevpW5KDkqxPcluSzUleNHRNfUly+PjvfefXvUneNHRdfUry5iS3JLk5
yWVJnjh0TfuCJsb4k7wFmAYOrKoTh66nT0m2ANNVtWgfYJmPJJcAX6yqC8Z3kS2vqv8buq6+jadK
uQN4YVV9Z+h6+pDkUOBLwLOq6qEk/wb8e1VdPGxlw1vyPf4kK4E1wAVD16J+JXkS8BLgQoCqerjF
0B87DvifVkJ/hmXA/kmWAcuBOweuZ5+w5IMf+ADwNuCnQxcykAKuTrJxPA1GSw4DtgMfHg/1XZDk
gKGLGsipwGVDF9GnqroDeB9wO3AXcE9VXT1sVfuGJR38SU4EtlXVxqFrGdCLq+pIRrOh/kmSlwxd
UI+WAUcC51XV84EHgOamAR8PcZ0EfGzoWvqU5MmMJoA8DDgEOCDJ6cNWtW9Y0sEPHAOcNB7nvhw4
NslHhy2pX+NeD1W1DfgEo9lRW7EV2FpV149fr2f0i6A1rwQ2VdX3hy6kZ8cD366q7VX1Y+AK4OiB
a9onLOngr6q3V9XKqlrN6H91r62qZn7jJzkgyYqd28ArgJuHrao/VfU94LtJDh/vOg64dcCShnIa
jQ3zjN0OHJVkeZIw+vvfPHBN+4R9dnZOLYinAZ8Y/ZtnGfAvVXXVsCX17kzg0vFwx7eA1w9cT6/G
v/BfDvzB0LX0raquT7Ie2ATsAG7A6RuARm7nlCT93JIe6pEkPZrBL0mNMfglqTEGvyQ1xuCXpMZ4
O6ealOQnwNdm7Lq8qs6d4/yXAQ9X1X91XZvUNYNfrXqoqp73GM5/GXA/8KjgT7KsqnYsVGFS1xzq
kWYYr1/wriSbxusY/GqS1cAfAm8ez2v/a0kuTnJ+kuuB9yR5SpIrk9yU5CtJjhi/3zuT/HOSLyf5
RpLfH+//SJJXz2j30iQnD3DJapDBr1btv8siJb8749jd44ntzgP+rKq2AOcDf1dVz6uqL47PWwkc
XVVvAd4F3FBVRwB/AXxkxvsdARwLvAh4R5JDGE0V/Tr42fTRRwOf7ehapUdwqEetmmuo54rx943A
KXO8x8eq6ifj7RcDvwVQVdcmeWqSA8fHPllVDwEPJbkOeEFVXZnkn5JMjX/u4w4XqS8Gv/RoPxp/
/wlz/zfywITvt+u8KDtffwQ4ndEEgk3NIaRhOdQjTeY+YMUcx78IvAZ+dgfQ3VV17/jYyUmemOSp
jD4k/up4/8XAmwCqqsVZQzUQe/xq1f5Jbpzx+qqqmmuRlk8D68cfwJ45y/F3AhcluQl4EDhjxrGb
gOuAg4F3V9WdAFX1/SSbgSv3/jKkx87ZOaUOJXkncH9VvW+WY8sZPUtwZFXd03dtapdDPdIAkhzP
aFGQvzf01Td7/JLUGHv8ktQYg1+SGmPwS1JjDH5JaozBL0mN+X8h+epAaV5iEQAAAABJRU5ErkJg
gg==
"
>
</div>

</div>

</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h2 id="Looking-at-the-Corpus-as-a-Whole">Looking at the Corpus as a Whole<a class="anchor-link" href="#Looking-at-the-Corpus-as-a-Whole">&#182;</a></h2>
</div>
</div>
</div>
<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h1 id="Scratch-Code-(Testing)">Scratch Code (Testing)<a class="anchor-link" href="#Scratch-Code-(Testing)">&#182;</a></h1>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">collections</span> <span class="k">import</span> <span class="n">Counter</span>
<span class="n">tok_counts</span> <span class="o">=</span> <span class="n">Counter</span><span class="p">()</span>
<span class="n">tok_counts</span><span class="p">[</span><span class="s1">&#39;hi&#39;</span><span class="p">]</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="n">tok_counts</span><span class="p">[</span><span class="s1">&#39;ooooo&#39;</span><span class="p">]</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="n">tok_counts</span><span class="p">[</span><span class="s1">&#39;ooooo&#39;</span><span class="p">]</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="c1"># print(p)</span>
<span class="c1"># print(len(p))</span>
<span class="c1"># for i in p.elements():</span>
<span class="c1">#   print(i)</span>
<span class="nb">print</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">tok_counts</span><span class="o">.</span><span class="n">elements</span><span class="p">())))</span>



<span class="n">num_tokens</span> <span class="o">=</span> <span class="mi">3</span>
<span class="n">frequencies</span> <span class="o">=</span> <span class="p">[</span><span class="n">count</span><span class="o">/</span><span class="n">num_tokens</span> <span class="k">for</span> <span class="n">token</span> <span class="ow">in</span> <span class="n">outcomes</span> <span class="k">for</span> <span class="n">count</span> <span class="ow">in</span> <span class="p">]</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">

<div class="output_subarea output_stream output_stdout output_text">
<pre>[&#39;ooooo&#39;, &#39;hi&#39;]
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">freq</span> <span class="o">=</span> <span class="p">[</span><span class="o">.</span><span class="mi">8</span><span class="p">,</span> <span class="o">.</span><span class="mi">1</span><span class="p">,</span> <span class="o">.</span><span class="mi">1</span><span class="p">]</span>
<span class="c1"># toks = [str(i) for i in range(len(freq))]</span>
<span class="n">toks</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;a&#39;</span><span class="p">,</span> <span class="s1">&#39;bb&#39;</span><span class="p">,</span> <span class="s1">&#39;ccc&#39;</span><span class="p">)</span>

<span class="n">d</span> <span class="o">=</span> <span class="n">dit</span><span class="o">.</span><span class="n">ScalarDistribution</span><span class="p">(</span><span class="n">toks</span><span class="p">,</span> <span class="n">freq</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">dit</span><span class="o">.</span><span class="n">shannon</span><span class="o">.</span><span class="n">entropy</span><span class="p">(</span><span class="n">d</span><span class="p">))</span>
<span class="nb">print</span><span class="p">(</span><span class="n">manual_shannon</span><span class="p">(</span><span class="n">freq</span><span class="p">))</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">

<div class="output_subarea output_stream output_stdout output_text">
<pre>0.9219280948873623
0.9219280948873625
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">a_corpus</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;file_name&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;aa&quot;</span><span class="p">,</span> <span class="s2">&quot;ab&quot;</span><span class="p">,</span> <span class="s2">&quot;ac&quot;</span><span class="p">],</span> <span class="s1">&#39;contents&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]}</span>
<span class="n">a_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">data</span> <span class="o">=</span> <span class="n">a_corpus</span><span class="p">)</span>
<span class="n">b_corpus</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;file_name&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;ba&quot;</span><span class="p">,</span> <span class="s2">&quot;bb&quot;</span><span class="p">,</span> <span class="s2">&quot;bc&quot;</span><span class="p">],</span> <span class="s1">&#39;contents&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]}</span>
<span class="n">b_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">data</span> <span class="o">=</span> <span class="n">b_corpus</span><span class="p">)</span>
<span class="n">c_corpus</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;file_name&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;ca&quot;</span><span class="p">,</span> <span class="s2">&quot;cb&quot;</span><span class="p">,</span> <span class="s2">&quot;cc&quot;</span><span class="p">],</span> <span class="s1">&#39;contents&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mi">7</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">9</span><span class="p">]}</span>
<span class="n">c_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">data</span> <span class="o">=</span> <span class="n">c_corpus</span><span class="p">)</span>

<span class="n">corpus_data</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;a&#39;</span><span class="p">:</span><span class="n">a_df</span><span class="p">,</span> <span class="s1">&#39;b&#39;</span><span class="p">:</span><span class="n">b_df</span><span class="p">,</span> <span class="s1">&#39;d&#39;</span><span class="p">:</span><span class="n">c_df</span><span class="p">}</span>

<span class="c1"># corpus_contents = pd.Series([])</span>
<span class="n">corpus_contents</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">data_type</span> <span class="ow">in</span> <span class="n">corpus_data</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
    <span class="n">corpus_contents</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">corpus_data</span><span class="p">[</span><span class="n">data_type</span><span class="p">]</span><span class="o">.</span><span class="n">contents</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">(</span><span class="n">corpus_contents</span><span class="p">))</span>


<span class="c1"># print([corpus_data[i] for i in corpus_data.keys()])</span>

<span class="c1"># merged_corpus = pd.concat([df.contents for df in corpus_data[data_type] for data_type in corpus_data.keys()])</span>
<span class="c1"># merged_corpus = pd.concat([df for data_type in corpus_data.keys() for df in corpus_data[data_type].contents])</span>
<span class="c1"># flatten_matrix = [val for sublist in matrix for val in sublist] </span>

<span class="c1"># something = [df[&quot;contents&quot;] for data_type in corpus_data.keys() for df in corpus_data[data_type]]</span>
<span class="c1"># print(something)</span>
<span class="c1"># print(pd.concat(something))</span>
<span class="c1"># print([i for i in range(10)])</span>

<span class="c1"># print(merged_df)</span>
<span class="c1"># print(pd.concat([a_df.contents, b_df.contents]))</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">

<div class="output_subarea output_stream output_stdout output_text">
<pre>0    1
1    2
2    3
0    4
1    5
2    6
0    7
1    8
2    9
Name: contents, dtype: int64
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1">#Rank the system/datasets according to the confidence intervals</span>
<span class="c1">#Compute the confidence intervals for all cross-entropy values</span>
<span class="c1">#Rank the systems/datasets according to cross-entropy values</span>
<span class="c1">#Top 50 most frequent tokens of each system and corpus (one system has generally two corpora)</span>
<span class="c1">#Top 50 least frequent tokes of each system and corpus</span>
<span class="c1">#What are the tokens that are in the target and not in the source (and the other way around)? Compute the distribution for those tokens</span>
<span class="c1">#What are the mutual tokens (source and target)? please compute distribution</span>
<span class="c1">#-Compute confidence intervals for the software metrics on source code (e.g., cyclo, loc, lcom5)</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">Visualize</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">Push</span> <span class="n">updated</span> <span class="n">fields</span> <span class="n">to</span> <span class="n">Mongo</span>
</pre></div>

    </div>
</div>
</div>

</div>
</div>