ds4se/docs/3.2_mutual_information_theory.eval.html

---

title: Title

keywords: fastai
sidebar: home_sidebar

summary: "summary"
---
<!--

#################################################
### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ###
#################################################
# file to edit: nbs/3.2_mutual_information_theory.eval.ipynb
# command to build the docs after a change: nbdev_build_docs

-->

<div class="container" id="notebook-container">
    
<div class="cell border-box-sizing code_cell rendered">

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h1 id="Evaluating-Mutual-Information-Theory-for-SE-Traceability">Evaluating Mutual Information Theory for SE Traceability<a class="anchor-link" href="#Evaluating-Mutual-Information-Theory-for-SE-Traceability">&#182;</a></h1>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="kn">import</span> <span class="nn">gensim</span>
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
<span class="kn">from</span> <span class="nn">itertools</span> <span class="k">import</span> <span class="n">product</span> 
<span class="kn">from</span> <span class="nn">random</span> <span class="k">import</span> <span class="n">sample</span> 
<span class="kn">import</span> <span class="nn">functools</span> 
<span class="kn">import</span> <span class="nn">os</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span>
<span class="kn">import</span> <span class="nn">seaborn</span> <span class="k">as</span> <span class="nn">sns</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="k">import</span> <span class="n">precision_recall_curve</span>
<span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="k">import</span> <span class="n">plot_precision_recall_curve</span>
<span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="k">import</span> <span class="n">auc</span>
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
<span class="kn">from</span> <span class="nn">prg</span> <span class="k">import</span> <span class="n">prg</span>
<span class="kn">from</span> <span class="nn">pandas.plotting</span> <span class="k">import</span> <span class="n">scatter_matrix</span>
<span class="kn">from</span> <span class="nn">pandas.plotting</span> <span class="k">import</span> <span class="n">lag_plot</span>
<span class="kn">import</span> <span class="nn">math</span> <span class="k">as</span> <span class="nn">m</span>
<span class="kn">import</span> <span class="nn">random</span> <span class="k">as</span> <span class="nn">r</span>
<span class="kn">import</span> <span class="nn">collections</span>
<span class="kn">from</span> <span class="nn">sklearn.metrics.pairwise</span> <span class="k">import</span> <span class="n">cosine_similarity</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">gensim.models</span> <span class="k">import</span> <span class="n">WordEmbeddingSimilarityIndex</span>
<span class="kn">from</span> <span class="nn">gensim.similarities</span> <span class="k">import</span> <span class="n">SparseTermSimilarityMatrix</span>
<span class="kn">from</span> <span class="nn">gensim</span> <span class="k">import</span> <span class="n">corpora</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">scipy.spatial</span> <span class="k">import</span> <span class="n">distance</span>
<span class="kn">from</span> <span class="nn">scipy.stats</span> <span class="k">import</span> <span class="n">pearsonr</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="k">import</span> <span class="n">average_precision_score</span>
<span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="k">import</span> <span class="n">roc_curve</span>
<span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="k">import</span> <span class="n">roc_auc_score</span>
<span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="k">import</span> <span class="n">confusion_matrix</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h1 id="Artifacts-Similarity-with-BasicSequenceVectorization">Artifacts Similarity with BasicSequenceVectorization<a class="anchor-link" href="#Artifacts-Similarity-with-BasicSequenceVectorization">&#182;</a></h1>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">enum</span> <span class="k">import</span> <span class="n">Enum</span><span class="p">,</span> <span class="n">unique</span><span class="p">,</span> <span class="n">auto</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="nd">@unique</span>
<span class="k">class</span> <span class="nc">VectorizationType</span><span class="p">(</span><span class="n">Enum</span><span class="p">):</span>
    <span class="n">word2vec</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
    <span class="n">doc2vec</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
    <span class="n">vsm2vec</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="nd">@unique</span>
<span class="k">class</span> <span class="nc">LinkType</span><span class="p">(</span><span class="n">Enum</span><span class="p">):</span>
    <span class="n">req2tc</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
    <span class="n">req2src</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="nd">@unique</span>
<span class="k">class</span> <span class="nc">DistanceMetric</span><span class="p">(</span><span class="n">Enum</span><span class="p">):</span>
    <span class="n">WMD</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
    <span class="n">COS</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
    <span class="n">SCM</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
    <span class="n">EUC</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
    <span class="n">MAN</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="nd">@unique</span>
<span class="k">class</span> <span class="nc">SimilarityMetric</span><span class="p">(</span><span class="n">Enum</span><span class="p">):</span>
    <span class="n">WMD_sim</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
    <span class="n">COS_sim</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
    <span class="n">SCM_sim</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
    <span class="n">EUC_sim</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
    <span class="n">MAN_sim</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
    <span class="n">Pearson</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">def</span> <span class="nf">default_params</span><span class="p">():</span>
    <span class="k">return</span> <span class="p">{</span>
        <span class="s2">&quot;vectorizationType&quot;</span><span class="p">:</span> <span class="n">VectorizationType</span><span class="o">.</span><span class="n">word2vec</span><span class="p">,</span>
        <span class="s2">&quot;linkType&quot;</span><span class="p">:</span> <span class="n">LinkType</span><span class="o">.</span><span class="n">req2tc</span><span class="p">,</span>
        <span class="s2">&quot;system&quot;</span><span class="p">:</span> <span class="s1">&#39;libest&#39;</span><span class="p">,</span>
        <span class="s2">&quot;path_to_trained_model&quot;</span><span class="p">:</span> <span class="s1">&#39;test_data/models/word2vec_libest.model&#39;</span><span class="p">,</span>
        <span class="s2">&quot;source_path&quot;</span><span class="p">:</span> <span class="s1">&#39;/tf/main/benchmarking/traceability/testbeds/nltk/[libest-pre-req].csv&#39;</span><span class="p">,</span>
        <span class="s2">&quot;target_path&quot;</span><span class="p">:</span> <span class="s1">&#39;/tf/main/benchmarking/traceability/testbeds/nltk/[libest-pre-tc].csv&#39;</span><span class="p">,</span>
        <span class="s2">&quot;system_path&quot;</span><span class="p">:</span> <span class="s1">&#39;/tf/main/benchmarking/traceability/testbeds/nltk/[libest-pre-all].csv&#39;</span><span class="p">,</span>
        <span class="s2">&quot;saving_path&quot;</span><span class="p">:</span> <span class="s1">&#39;test_data/&#39;</span><span class="p">,</span>
        <span class="s2">&quot;names&quot;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;Source&#39;</span><span class="p">,</span><span class="s1">&#39;Target&#39;</span><span class="p">,</span><span class="s1">&#39;Linked?&#39;</span><span class="p">]</span>
    <span class="p">}</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">parameters</span> <span class="o">=</span> <span class="n">default_params</span><span class="p">()</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="nb">print</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">VectorizationType</span><span class="p">),</span> <span class="nb">list</span><span class="p">(</span><span class="n">DistanceMetric</span><span class="p">),</span> <span class="nb">list</span><span class="p">(</span><span class="n">SimilarityMetric</span><span class="p">),</span> <span class="nb">list</span><span class="p">(</span><span class="n">LinkType</span><span class="p">))</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">

<div class="output_subarea output_stream output_stdout output_text">
<pre>[&lt;VectorizationType.word2vec: 1&gt;, &lt;VectorizationType.doc2vec: 2&gt;, &lt;VectorizationType.vsm2vec: 3&gt;] [&lt;DistanceMetric.WMD: 1&gt;, &lt;DistanceMetric.COS: 2&gt;, &lt;DistanceMetric.SCM: 3&gt;, &lt;DistanceMetric.EUC: 4&gt;, &lt;DistanceMetric.MAN: 5&gt;] [&lt;SimilarityMetric.WMD_sim: 1&gt;, &lt;SimilarityMetric.COS_sim: 2&gt;, &lt;SimilarityMetric.SCM_sim: 3&gt;, &lt;SimilarityMetric.EUC_sim: 4&gt;, &lt;SimilarityMetric.MAN_sim: 5&gt;, &lt;SimilarityMetric.Pearson: 6&gt;] [&lt;LinkType.req2tc: 1&gt;, &lt;LinkType.req2src: 2&gt;]
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">class</span> <span class="nc">BasicSequenceVectorization</span><span class="p">():</span>
    <span class="sd">&#39;&#39;&#39;Implementation of the class sequence-vanilla-vectorization other classes can inheritance this one&#39;&#39;&#39;</span>
    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
                
        <span class="bp">self</span><span class="o">.</span><span class="n">df_source</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;source_path&#39;</span><span class="p">],</span> <span class="n">names</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;ids&#39;</span><span class="p">,</span> <span class="s1">&#39;text&#39;</span><span class="p">],</span> <span class="n">header</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39; &#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">df_target</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;target_path&#39;</span><span class="p">],</span> <span class="n">names</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;ids&#39;</span><span class="p">,</span> <span class="s1">&#39;text&#39;</span><span class="p">],</span> <span class="n">header</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39; &#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">df_all_system</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;system_path&#39;</span><span class="p">],</span> <span class="n">names</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;ids&#39;</span><span class="p">,</span> <span class="s1">&#39;text&#39;</span><span class="p">],</span> 
                                         <span class="n">header</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">index_col</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;,&#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">params</span> <span class="o">=</span> <span class="n">params</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">df_nonground_link</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">df_ground_link</span> <span class="o">=</span> <span class="kc">None</span>
        
        <span class="bp">self</span><span class="o">.</span><span class="n">documents</span> <span class="o">=</span> <span class="p">[</span><span class="n">doc</span><span class="o">.</span><span class="n">split</span><span class="p">()</span> <span class="k">for</span> <span class="n">doc</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">df_all_system</span><span class="p">[</span><span class="s1">&#39;text&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">values</span><span class="p">]</span> <span class="c1">#Preparing Corpus</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dictionary</span> <span class="o">=</span> <span class="n">corpora</span><span class="o">.</span><span class="n">Dictionary</span><span class="p">(</span> <span class="bp">self</span><span class="o">.</span><span class="n">documents</span> <span class="p">)</span> <span class="c1">#Preparing Dictionary</span>
        
        
        <span class="c1">#This can be extended for future metrics &lt;---------------------</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dict_labels</span> <span class="o">=</span> <span class="p">{</span>
            <span class="n">DistanceMetric</span><span class="o">.</span><span class="n">COS</span><span class="p">:[</span><span class="n">DistanceMetric</span><span class="o">.</span><span class="n">COS</span><span class="p">,</span> <span class="n">SimilarityMetric</span><span class="o">.</span><span class="n">COS_sim</span><span class="p">],</span>
            <span class="n">SimilarityMetric</span><span class="o">.</span><span class="n">Pearson</span><span class="p">:[</span><span class="n">SimilarityMetric</span><span class="o">.</span><span class="n">Pearson</span><span class="p">],</span>
            <span class="n">DistanceMetric</span><span class="o">.</span><span class="n">EUC</span><span class="p">:[</span><span class="n">DistanceMetric</span><span class="o">.</span><span class="n">EUC</span><span class="p">,</span> <span class="n">SimilarityMetric</span><span class="o">.</span><span class="n">EUC_sim</span><span class="p">],</span>
            <span class="n">DistanceMetric</span><span class="o">.</span><span class="n">WMD</span><span class="p">:[</span><span class="n">DistanceMetric</span><span class="o">.</span><span class="n">WMD</span><span class="p">,</span> <span class="n">SimilarityMetric</span><span class="o">.</span><span class="n">WMD_sim</span><span class="p">],</span>
            <span class="n">DistanceMetric</span><span class="o">.</span><span class="n">SCM</span><span class="p">:[</span><span class="n">DistanceMetric</span><span class="o">.</span><span class="n">SCM</span><span class="p">,</span> <span class="n">SimilarityMetric</span><span class="o">.</span><span class="n">SCM_sim</span><span class="p">],</span>
            <span class="n">DistanceMetric</span><span class="o">.</span><span class="n">MAN</span><span class="p">:[</span><span class="n">DistanceMetric</span><span class="o">.</span><span class="n">MAN</span><span class="p">,</span> <span class="n">SimilarityMetric</span><span class="o">.</span><span class="n">MAN_sim</span><span class="p">]</span>
        <span class="p">}</span>

        
    <span class="k">def</span> <span class="nf">ground_truth_processing</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path_to_ground_truth</span><span class="p">):</span>
        <span class="s1">&#39;Optional class when corpus has ground truth&#39;</span>
        <span class="n">ground_truth</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">path_to_ground_truth</span><span class="p">,</span><span class="s1">&#39;r&#39;</span><span class="p">)</span>
        <span class="c1">#Organizing The Ground Truth under the given format</span>
        <span class="n">ground_links</span> <span class="o">=</span> <span class="p">[</span> <span class="p">[(</span><span class="n">line</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">()[</span><span class="mi">0</span><span class="p">],</span> <span class="n">elem</span><span class="p">)</span> <span class="k">for</span> <span class="n">elem</span> <span class="ow">in</span> <span class="n">line</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">()[</span><span class="mi">1</span><span class="p">:]]</span> <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">ground_truth</span><span class="p">]</span>
        <span class="n">ground_links</span> <span class="o">=</span> <span class="n">functools</span><span class="o">.</span><span class="n">reduce</span><span class="p">(</span><span class="k">lambda</span> <span class="n">a</span><span class="p">,</span><span class="n">b</span> <span class="p">:</span> <span class="n">a</span><span class="o">+</span><span class="n">b</span><span class="p">,</span><span class="n">ground_links</span><span class="p">)</span> <span class="c1">#reducing into one list</span>
        <span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">ground_links</span><span class="p">)</span> <span class="o">==</span>  <span class="nb">len</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">ground_links</span><span class="p">))</span> <span class="c1">#To Verify Redundancies in the file</span>
        <span class="k">return</span> <span class="n">ground_links</span>
    
    <span class="k">def</span> <span class="nf">samplingLinks</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sampling</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> <span class="n">samples</span> <span class="o">=</span> <span class="mi">10</span><span class="p">):</span>
        <span class="n">source</span> <span class="o">=</span> <span class="p">[</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">basename</span><span class="p">(</span><span class="n">elem</span><span class="p">)</span> <span class="k">for</span> <span class="n">elem</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">df_source</span><span class="p">[</span><span class="s1">&#39;ids&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">values</span> <span class="p">]</span> 
        <span class="n">target</span> <span class="o">=</span> <span class="p">[</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">basename</span><span class="p">(</span><span class="n">elem</span><span class="p">)</span> <span class="k">for</span> <span class="n">elem</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">df_target</span><span class="p">[</span><span class="s1">&#39;ids&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">values</span> <span class="p">]</span>

        <span class="k">if</span> <span class="n">sampling</span><span class="p">:</span>
            <span class="n">links</span> <span class="o">=</span> <span class="n">sample</span><span class="p">(</span> <span class="nb">list</span><span class="p">(</span> <span class="n">product</span><span class="p">(</span> <span class="n">source</span> <span class="p">,</span> <span class="n">target</span> <span class="p">)</span> <span class="p">),</span> <span class="n">samples</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">links</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span> <span class="n">product</span><span class="p">(</span> <span class="n">source</span> <span class="p">,</span> <span class="n">target</span> <span class="p">))</span>

        <span class="k">return</span> <span class="n">links</span>
    
    <span class="k">def</span> <span class="nf">cos_scipy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">vector_v</span><span class="p">,</span> <span class="n">vector_w</span><span class="p">):</span>
        <span class="n">cos</span> <span class="o">=</span>  <span class="n">distance</span><span class="o">.</span><span class="n">cosine</span><span class="p">(</span> <span class="n">vector_v</span><span class="p">,</span> <span class="n">vector_w</span> <span class="p">)</span>
        <span class="k">return</span> <span class="p">[</span><span class="n">cos</span><span class="p">,</span> <span class="mf">1.</span><span class="o">-</span><span class="n">cos</span><span class="p">]</span>
    
    <span class="k">def</span> <span class="nf">euclidean_scipy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">vector_v</span><span class="p">,</span> <span class="n">vector_w</span><span class="p">):</span>
        <span class="n">dst</span> <span class="o">=</span> <span class="n">distance</span><span class="o">.</span><span class="n">euclidean</span><span class="p">(</span><span class="n">vector_v</span><span class="p">,</span><span class="n">vector_w</span><span class="p">)</span>
        <span class="k">return</span> <span class="p">[</span><span class="n">dst</span><span class="p">,</span> <span class="mf">1.</span><span class="o">/</span><span class="p">(</span><span class="mf">1.</span><span class="o">+</span><span class="n">dst</span><span class="p">)]</span> <span class="c1">#Computing the inverse for similarity</span>
    
    <span class="k">def</span> <span class="nf">manhattan_scipy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">vector_v</span><span class="p">,</span> <span class="n">vector_w</span><span class="p">):</span>
        <span class="n">dst</span> <span class="o">=</span> <span class="n">distance</span><span class="o">.</span><span class="n">cityblock</span><span class="p">(</span><span class="n">vector_v</span><span class="p">,</span><span class="n">vector_w</span><span class="p">)</span>
        <span class="n">n</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">vector_v</span><span class="p">)</span>
        <span class="k">return</span> <span class="p">[</span><span class="n">dst</span><span class="p">,</span> <span class="mf">1.</span><span class="o">/</span><span class="p">(</span><span class="mf">1.</span><span class="o">+</span><span class="n">dst</span><span class="p">)]</span> <span class="c1">#Computing the inverse for similarity</span>
    
    <span class="k">def</span> <span class="nf">pearson_abs_scipy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">vector_v</span><span class="p">,</span> <span class="n">vector_w</span><span class="p">):</span>
        <span class="sd">&#39;&#39;&#39;We are not sure that pearson correlation works well on doc2vec inference vectors&#39;&#39;&#39;</span>
        <span class="n">corr</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">pearsonr</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
        <span class="k">return</span> <span class="p">[</span><span class="nb">abs</span><span class="p">(</span><span class="n">corr</span><span class="p">)]</span> <span class="c1">#Absolute value of the correlation</span>
    

    <span class="k">def</span> <span class="nf">computeDistanceMetric</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">links</span><span class="p">,</span> <span class="n">metric_list</span><span class="p">):</span>
        <span class="sd">&#39;&#39;&#39;Metric List Iteration&#39;&#39;&#39;</span> 
        
        <span class="n">metric_labels</span> <span class="o">=</span> <span class="p">[</span> <span class="bp">self</span><span class="o">.</span><span class="n">dict_labels</span><span class="p">[</span><span class="n">metric</span><span class="p">]</span> <span class="k">for</span> <span class="n">metric</span> <span class="ow">in</span> <span class="n">metric_list</span><span class="p">]</span> <span class="c1">#tracking of the labels</span>
        <span class="n">distSim</span> <span class="o">=</span> <span class="p">[[</span><span class="n">link</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">link</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">distance</span><span class="p">(</span> <span class="n">metric_list</span><span class="p">,</span> <span class="n">link</span> <span class="p">)]</span> <span class="k">for</span> <span class="n">link</span> <span class="ow">in</span> <span class="n">links</span><span class="p">]</span> <span class="c1">#Return the link with metrics</span>
        <span class="n">distSim</span> <span class="o">=</span> <span class="p">[[</span><span class="n">elem</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">elem</span><span class="p">[</span><span class="mi">1</span><span class="p">]]</span> <span class="o">+</span> <span class="n">elem</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="k">for</span> <span class="n">elem</span> <span class="ow">in</span> <span class="n">distSim</span><span class="p">]</span> <span class="c1">#Return the link with metrics</span>
        
        <span class="k">return</span> <span class="n">distSim</span><span class="p">,</span> <span class="n">functools</span><span class="o">.</span><span class="n">reduce</span><span class="p">(</span><span class="k">lambda</span> <span class="n">a</span><span class="p">,</span><span class="n">b</span> <span class="p">:</span> <span class="n">a</span><span class="o">+</span><span class="n">b</span><span class="p">,</span> <span class="n">metric_labels</span><span class="p">)</span>
    
    <span class="k">def</span> <span class="nf">ComputeDistanceArtifacts</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">metric_list</span><span class="p">,</span> <span class="n">sampling</span> <span class="o">=</span> <span class="kc">False</span> <span class="p">,</span> <span class="n">samples</span> <span class="o">=</span> <span class="mi">10</span><span class="p">):</span>
        <span class="sd">&#39;&#39;&#39;Acticates Distance and Similarity Computations</span>
<span class="sd">        @metric_list if [] then Computes All metrics</span>
<span class="sd">        @sampling is False by the default</span>
<span class="sd">        @samples is the number of samples (or links) to be generated&#39;&#39;&#39;</span>
        <span class="n">links_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">samplingLinks</span><span class="p">(</span> <span class="n">sampling</span><span class="p">,</span> <span class="n">samples</span> <span class="p">)</span>
        
        <span class="n">docs</span><span class="p">,</span> <span class="n">metric_labels</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">computeDistanceMetric</span><span class="p">(</span> <span class="n">metric_list</span><span class="o">=</span><span class="n">metric_list</span><span class="p">,</span> <span class="n">links</span><span class="o">=</span><span class="n">links_</span><span class="p">)</span> <span class="c1">#checkpoints</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">df_nonground_link</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">docs</span><span class="p">,</span> <span class="n">columns</span> <span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;names&#39;</span><span class="p">][</span><span class="mi">0</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;names&#39;</span><span class="p">][</span><span class="mi">1</span><span class="p">]]</span><span class="o">+</span> <span class="n">metric_labels</span><span class="p">)</span> <span class="c1">#Transforming into a Pandas</span>
        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Non-groundtruth links computed&quot;</span><span class="p">)</span>
        <span class="k">pass</span> 
    
    
    <span class="k">def</span> <span class="nf">SaveLinks</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">grtruth</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39; &#39;</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;a&#39;</span><span class="p">):</span>
        <span class="n">timestamp</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">())</span>
        <span class="n">path_to_link</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;saving_path&#39;</span><span class="p">]</span> <span class="o">+</span> <span class="s1">&#39;[&#39;</span><span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;system&#39;</span><span class="p">]</span> <span class="o">+</span> <span class="s1">&#39;-&#39;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;vectorizationType&#39;</span><span class="p">])</span> <span class="o">+</span> <span class="s1">&#39;-&#39;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;linkType&#39;</span><span class="p">])</span> <span class="o">+</span> <span class="s1">&#39;-&#39;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">grtruth</span><span class="p">)</span> <span class="o">+</span> <span class="s1">&#39;-</span><span class="si">{}</span><span class="s1">].csv&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">timestamp</span><span class="p">)</span>
        
        <span class="k">if</span> <span class="n">grtruth</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">df_ground_link</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="n">path_to_link</span><span class="p">,</span> <span class="n">header</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">index</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="n">sep</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="n">mode</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">df_nonground_link</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="n">path_to_link</span><span class="p">,</span> <span class="n">header</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">index</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="n">sep</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="n">mode</span><span class="p">)</span>
        
        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Saving in...&#39;</span> <span class="o">+</span> <span class="n">path_to_link</span><span class="p">)</span>
        <span class="k">pass</span>
    
    <span class="k">def</span> <span class="nf">findDistInDF</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">g_tuple</span><span class="p">):</span>
        <span class="n">dist</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">df_ground_link</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">df_ground_link</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;names&#39;</span><span class="p">][</span><span class="mi">0</span><span class="p">]]</span><span class="o">.</span><span class="n">str</span><span class="o">.</span><span class="n">contains</span><span class="p">(</span> <span class="n">g_tuple</span><span class="p">[</span><span class="mi">0</span><span class="p">][:</span><span class="n">g_tuple</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)]</span> <span class="o">+</span> <span class="s1">&#39;-&#39;</span> <span class="p">)</span> 
                     <span class="o">&amp;</span> <span class="bp">self</span><span class="o">.</span><span class="n">df_ground_link</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;names&#39;</span><span class="p">][</span><span class="mi">1</span><span class="p">]]</span><span class="o">.</span><span class="n">str</span><span class="o">.</span><span class="n">contains</span><span class="p">(</span><span class="n">g_tuple</span><span class="p">[</span><span class="mi">1</span><span class="p">][:</span><span class="n">g_tuple</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)])</span> <span class="p">]</span>        
        <span class="k">return</span> <span class="n">dist</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">values</span>
        
    <span class="k">def</span> <span class="nf">MatchWithGroundTruth</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path_to_ground_truth</span> <span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">df_ground_link</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">df_nonground_link</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
        
        <span class="n">matchGT</span> <span class="o">=</span> <span class="p">[</span> <span class="bp">self</span><span class="o">.</span><span class="n">findDistInDF</span><span class="p">(</span> <span class="n">g</span> <span class="p">)</span> <span class="k">for</span> <span class="n">g</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">ground_truth_processing</span><span class="p">(</span><span class="n">path_to_ground_truth</span><span class="p">)]</span>
        <span class="n">matchGT</span> <span class="o">=</span> <span class="n">functools</span><span class="o">.</span><span class="n">reduce</span><span class="p">(</span><span class="k">lambda</span> <span class="n">a</span><span class="p">,</span><span class="n">b</span> <span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">([</span><span class="n">a</span><span class="p">,</span><span class="n">b</span><span class="p">]),</span> <span class="n">matchGT</span><span class="p">)</span>
        
        <span class="bp">self</span><span class="o">.</span><span class="n">df_ground_link</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;names&#39;</span><span class="p">][</span><span class="mi">2</span><span class="p">]]</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="n">new_column</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">full</span><span class="p">([</span><span class="nb">len</span><span class="p">(</span><span class="n">matchGT</span><span class="p">)],</span> <span class="mi">1</span> <span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;names&#39;</span><span class="p">][</span><span class="mi">2</span><span class="p">],</span> <span class="n">index</span> <span class="o">=</span> <span class="n">matchGT</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">df_ground_link</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">new_column</span><span class="p">)</span>
        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Groundtruth links computed&quot;</span><span class="p">)</span>
        
        <span class="k">pass</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">general2vec</span> <span class="o">=</span>  <span class="n">BasicSequenceVectorization</span><span class="p">(</span><span class="n">params</span> <span class="o">=</span> <span class="n">parameters</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
<div class="text_cell_render border-box-sizing rendered_html">
<h1 id="Artifacts-Similarity-with-Word2Vec">Artifacts Similarity with Word2Vec<a class="anchor-link" href="#Artifacts-Similarity-with-Word2Vec">&#182;</a></h1>
</div>
</div>
</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">class</span> <span class="nc">Word2VecSeqVect</span><span class="p">(</span><span class="n">BasicSequenceVectorization</span><span class="p">):</span>       
    
    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">params</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">new_model</span> <span class="o">=</span> <span class="n">gensim</span><span class="o">.</span><span class="n">models</span><span class="o">.</span><span class="n">Word2Vec</span><span class="o">.</span><span class="n">load</span><span class="p">(</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;path_to_trained_model&#39;</span><span class="p">]</span> <span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">new_model</span><span class="o">.</span><span class="n">init_sims</span><span class="p">(</span><span class="n">replace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>  <span class="c1"># Normalizes the vectors in the word2vec class.</span>
        <span class="c1">#Computes cosine similarities between word embeddings and retrieves the closest </span>
        <span class="c1">#word embeddings by cosine similarity for a given word embedding.</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">similarity_index</span> <span class="o">=</span> <span class="n">WordEmbeddingSimilarityIndex</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">new_model</span><span class="o">.</span><span class="n">wv</span><span class="p">)</span>
        <span class="c1">#Build a term similarity matrix and compute the Soft Cosine Measure.</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">similarity_matrix</span> <span class="o">=</span> <span class="n">SparseTermSimilarityMatrix</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">similarity_index</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">dictionary</span><span class="p">)</span>
        
        <span class="bp">self</span><span class="o">.</span><span class="n">dict_distance_dispatcher</span> <span class="o">=</span> <span class="p">{</span>
            <span class="n">DistanceMetric</span><span class="o">.</span><span class="n">COS</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">cos_scipy</span><span class="p">,</span>
            <span class="n">SimilarityMetric</span><span class="o">.</span><span class="n">Pearson</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">pearson_abs_scipy</span><span class="p">,</span>
            <span class="n">DistanceMetric</span><span class="o">.</span><span class="n">WMD</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">wmd_gensim</span><span class="p">,</span>
            <span class="n">DistanceMetric</span><span class="o">.</span><span class="n">SCM</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">scm_gensim</span>
        <span class="p">}</span>
    
    <span class="k">def</span> <span class="nf">wmd_gensim</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sentence_a</span><span class="p">,</span> <span class="n">sentence_b</span> <span class="p">):</span>
        <span class="n">wmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">new_model</span><span class="o">.</span><span class="n">wv</span><span class="o">.</span><span class="n">wmdistance</span><span class="p">(</span><span class="n">sentence_a</span><span class="p">,</span> <span class="n">sentence_b</span><span class="p">)</span>
        <span class="k">return</span> <span class="p">[</span><span class="n">wmd</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">wmd_similarity</span><span class="p">(</span><span class="n">wmd</span><span class="p">)]</span>
    
    <span class="k">def</span> <span class="nf">wmd_similarity</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dist</span><span class="p">):</span>
        <span class="k">return</span> <span class="mf">1.</span><span class="o">/</span><span class="p">(</span> <span class="mf">1.</span><span class="o">+</span><span class="nb">float</span><span class="p">(</span> <span class="n">dist</span> <span class="p">)</span> <span class="p">)</span> <span class="c1">#Associated Similarity</span>
    
    <span class="k">def</span> <span class="nf">scm_gensim</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sentence_a</span><span class="p">,</span> <span class="n">sentence_b</span> <span class="p">):</span>
        <span class="sd">&#39;&#39;&#39;Compute SoftCosine Similarity of Gensim&#39;&#39;&#39;</span>
        <span class="c1">#Convert the sentences into bag-of-words vectors.</span>
        <span class="n">sentence_1</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dictionary</span><span class="o">.</span><span class="n">doc2bow</span><span class="p">(</span><span class="n">sentence_a</span><span class="p">)</span>
        <span class="n">sentence_2</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dictionary</span><span class="o">.</span><span class="n">doc2bow</span><span class="p">(</span><span class="n">sentence_b</span><span class="p">)</span>
        
        <span class="c1">#Return the inner product(s) between real vectors / corpora vec1 and vec2 expressed in a non-orthogonal normalized basis,</span>
        <span class="c1">#where the dot product between the basis vectors is given by the sparse term similarity matrix.</span>
        <span class="n">scm_similarity</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">similarity_matrix</span><span class="o">.</span><span class="n">inner_product</span><span class="p">(</span><span class="n">sentence_1</span><span class="p">,</span> <span class="n">sentence_2</span><span class="p">,</span> <span class="n">normalized</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
        <span class="k">return</span> <span class="p">[</span><span class="mi">1</span><span class="o">-</span><span class="n">scm_similarity</span><span class="p">,</span> <span class="n">scm_similarity</span><span class="p">]</span>
    
    <span class="k">def</span> <span class="nf">distance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">metric_list</span><span class="p">,</span><span class="n">link</span><span class="p">):</span>
        <span class="sd">&#39;&#39;&#39;Iterate on the metrics&#39;&#39;&#39;</span>
        <span class="c1">#Computation of sentences can be moved directly to wmd_gensim method if we cannot generalize it for </span>
        <span class="c1">#the remaining metrics</span>
        <span class="n">sentence_a</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">df_source</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">df_source</span><span class="p">[</span><span class="s1">&#39;ids&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">str</span><span class="o">.</span><span class="n">contains</span><span class="p">(</span><span class="n">link</span><span class="p">[</span><span class="mi">0</span><span class="p">])][</span><span class="s1">&#39;text&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
        <span class="n">sentence_b</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">df_target</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">df_target</span><span class="p">[</span><span class="s1">&#39;ids&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">str</span><span class="o">.</span><span class="n">contains</span><span class="p">(</span><span class="n">link</span><span class="p">[</span><span class="mi">1</span><span class="p">])][</span><span class="s1">&#39;text&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
        
        <span class="n">dist</span> <span class="o">=</span> <span class="p">[</span> <span class="bp">self</span><span class="o">.</span><span class="n">dict_distance_dispatcher</span><span class="p">[</span><span class="n">metric</span><span class="p">](</span><span class="n">sentence_a</span><span class="p">,</span><span class="n">sentence_b</span><span class="p">)</span> <span class="k">for</span> <span class="n">metric</span> <span class="ow">in</span> <span class="n">metric_list</span><span class="p">]</span>
        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Computed distances or similarities &quot;</span><span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">link</span><span class="p">)</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">dist</span><span class="p">))</span>    
        <span class="k">return</span> <span class="n">functools</span><span class="o">.</span><span class="n">reduce</span><span class="p">(</span><span class="k">lambda</span> <span class="n">a</span><span class="p">,</span><span class="n">b</span> <span class="p">:</span> <span class="n">a</span><span class="o">+</span><span class="n">b</span><span class="p">,</span> <span class="n">dist</span><span class="p">)</span> <span class="c1">#Always return a list</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">def</span> <span class="nf">LoadLinks</span><span class="p">(</span><span class="n">timestamp</span><span class="p">,</span> <span class="n">params</span><span class="p">,</span> <span class="n">grtruth</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39; &#39;</span> <span class="p">):</span>
    <span class="sd">&#39;&#39;&#39;Returns a pandas from a saved link computation at a give timestamp</span>
<span class="sd">    @timestamp is the version of the model for a given system&#39;&#39;&#39;</span>
    
    <span class="n">path</span><span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;saving_path&#39;</span><span class="p">]</span> <span class="o">+</span> <span class="s1">&#39;[&#39;</span><span class="o">+</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;system&#39;</span><span class="p">]</span> <span class="o">+</span> <span class="s1">&#39;-&#39;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;vectorizationType&#39;</span><span class="p">])</span> <span class="o">+</span> <span class="s1">&#39;-&#39;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;linkType&#39;</span><span class="p">])</span> <span class="o">+</span> <span class="s1">&#39;-&#39;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">grtruth</span><span class="p">)</span> <span class="o">+</span> <span class="s1">&#39;-</span><span class="si">{}</span><span class="s1">].csv&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">timestamp</span><span class="p">)</span>
    
    <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Loading computed links from... &quot;</span><span class="o">+</span> <span class="n">path</span><span class="p">)</span>

    <span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">header</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">index_col</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="n">sep</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">metric_list</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;a&#39;</span><span class="p">,</span><span class="s1">&#39;b&#39;</span><span class="p">]</span>
<span class="n">A</span> <span class="o">=</span> <span class="p">[[</span><span class="mi">1</span><span class="p">,</span><span class="mi">3</span><span class="p">,</span><span class="mi">4</span><span class="p">],[</span><span class="mi">4</span><span class="p">,</span><span class="mi">5</span><span class="p">],[</span><span class="mi">1</span><span class="p">,</span><span class="mi">8</span><span class="p">,</span><span class="mi">9</span><span class="p">,</span><span class="mi">7</span><span class="p">]]</span>
<span class="n">B</span> <span class="o">=</span> <span class="p">((</span><span class="mi">1</span><span class="p">,</span><span class="mi">3</span><span class="p">,</span><span class="mi">4</span><span class="p">),(</span><span class="mi">4</span><span class="p">,</span><span class="mi">5</span><span class="p">),(</span><span class="mi">1</span><span class="p">,</span><span class="mi">8</span><span class="p">,</span><span class="mi">9</span><span class="p">,</span><span class="mi">7</span><span class="p">))</span>
<span class="n">functools</span><span class="o">.</span><span class="n">reduce</span><span class="p">(</span><span class="k">lambda</span> <span class="n">a</span><span class="p">,</span><span class="n">b</span> <span class="p">:</span> <span class="n">a</span><span class="o">+</span><span class="n">b</span><span class="p">,</span> <span class="n">B</span><span class="p">)</span>
<span class="n">dist_sim_T</span> <span class="o">=</span> <span class="p">[([</span><span class="mi">12</span><span class="p">,</span><span class="mi">13</span><span class="p">],[</span><span class="s1">&#39;metric1&#39;</span><span class="p">,</span><span class="s1">&#39;metric2&#39;</span><span class="p">]),([</span><span class="mi">12</span><span class="p">,</span><span class="mi">13</span><span class="p">],[</span><span class="s1">&#39;metric1&#39;</span><span class="p">,</span><span class="s1">&#39;metric2&#39;</span><span class="p">])]</span>
<span class="n">dist_sim_T</span>
<span class="n">separated_merged_list_a</span> <span class="o">=</span> <span class="n">functools</span><span class="o">.</span><span class="n">reduce</span><span class="p">(</span><span class="k">lambda</span> <span class="n">a</span><span class="p">,</span><span class="n">b</span> <span class="p">:</span> <span class="n">a</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">+</span><span class="n">b</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">dist_sim_T</span><span class="p">)</span>
<span class="n">separated_merged_list_a</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">



<div class="output_text output_subarea output_execute_result">
<pre>[&#39;metric1&#39;, &#39;metric2&#39;, &#39;metric1&#39;, &#39;metric2&#39;]</pre>
</div>

</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">word2vec</span> <span class="o">=</span> <span class="n">Word2VecSeqVect</span><span class="p">(</span> <span class="n">params</span> <span class="o">=</span> <span class="n">parameters</span> <span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">

<div class="output_subarea output_text output_error">
<pre>
<span class="ansi-red-fg">---------------------------------------------------------------------------</span>
<span class="ansi-red-fg">FileNotFoundError</span>                         Traceback (most recent call last)
<span class="ansi-green-fg">&lt;ipython-input-21-46a2e5ff253c&gt;</span> in <span class="ansi-cyan-fg">&lt;module&gt;</span>
<span class="ansi-green-fg">----&gt; 1</span><span class="ansi-red-fg"> </span>word2vec <span class="ansi-blue-fg">=</span> Word2VecSeqVect<span class="ansi-blue-fg">(</span> params <span class="ansi-blue-fg">=</span> parameters <span class="ansi-blue-fg">)</span>

<span class="ansi-green-fg">&lt;ipython-input-18-6e7af5c67cec&gt;</span> in <span class="ansi-cyan-fg">__init__</span><span class="ansi-blue-fg">(self, params)</span>
<span class="ansi-green-intense-fg ansi-bold">      3</span>     <span class="ansi-green-fg">def</span> __init__<span class="ansi-blue-fg">(</span>self<span class="ansi-blue-fg">,</span> params<span class="ansi-blue-fg">)</span><span class="ansi-blue-fg">:</span>
<span class="ansi-green-intense-fg ansi-bold">      4</span>         super<span class="ansi-blue-fg">(</span><span class="ansi-blue-fg">)</span><span class="ansi-blue-fg">.</span>__init__<span class="ansi-blue-fg">(</span>params<span class="ansi-blue-fg">)</span>
<span class="ansi-green-fg">----&gt; 5</span><span class="ansi-red-fg">         </span>self<span class="ansi-blue-fg">.</span>new_model <span class="ansi-blue-fg">=</span> gensim<span class="ansi-blue-fg">.</span>models<span class="ansi-blue-fg">.</span>Word2Vec<span class="ansi-blue-fg">.</span>load<span class="ansi-blue-fg">(</span> params<span class="ansi-blue-fg">[</span><span class="ansi-blue-fg">&#39;path_to_trained_model&#39;</span><span class="ansi-blue-fg">]</span> <span class="ansi-blue-fg">)</span>
<span class="ansi-green-intense-fg ansi-bold">      6</span>         self<span class="ansi-blue-fg">.</span>new_model<span class="ansi-blue-fg">.</span>init_sims<span class="ansi-blue-fg">(</span>replace<span class="ansi-blue-fg">=</span><span class="ansi-green-fg">True</span><span class="ansi-blue-fg">)</span>  <span class="ansi-red-fg"># Normalizes the vectors in the word2vec class.</span>
<span class="ansi-green-intense-fg ansi-bold">      7</span>         <span class="ansi-red-fg">#Computes cosine similarities between word embeddings and retrieves the closest</span>

<span class="ansi-green-fg">/usr/local/lib/python3.6/dist-packages/gensim/models/word2vec.py</span> in <span class="ansi-cyan-fg">load</span><span class="ansi-blue-fg">(cls, *args, **kwargs)</span>
<span class="ansi-green-intense-fg ansi-bold">   1139</span>         &#34;&#34;&#34;
<span class="ansi-green-intense-fg ansi-bold">   1140</span>         <span class="ansi-green-fg">try</span><span class="ansi-blue-fg">:</span>
<span class="ansi-green-fg">-&gt; 1141</span><span class="ansi-red-fg">             </span>model <span class="ansi-blue-fg">=</span> super<span class="ansi-blue-fg">(</span>Word2Vec<span class="ansi-blue-fg">,</span> cls<span class="ansi-blue-fg">)</span><span class="ansi-blue-fg">.</span>load<span class="ansi-blue-fg">(</span><span class="ansi-blue-fg">*</span>args<span class="ansi-blue-fg">,</span> <span class="ansi-blue-fg">**</span>kwargs<span class="ansi-blue-fg">)</span>
<span class="ansi-green-intense-fg ansi-bold">   1142</span> 
<span class="ansi-green-intense-fg ansi-bold">   1143</span>             <span class="ansi-red-fg"># for backward compatibility for `max_final_vocab` feature</span>

<span class="ansi-green-fg">/usr/local/lib/python3.6/dist-packages/gensim/models/base_any2vec.py</span> in <span class="ansi-cyan-fg">load</span><span class="ansi-blue-fg">(cls, *args, **kwargs)</span>
<span class="ansi-green-intense-fg ansi-bold">   1228</span> 
<span class="ansi-green-intense-fg ansi-bold">   1229</span>         &#34;&#34;&#34;
<span class="ansi-green-fg">-&gt; 1230</span><span class="ansi-red-fg">         </span>model <span class="ansi-blue-fg">=</span> super<span class="ansi-blue-fg">(</span>BaseWordEmbeddingsModel<span class="ansi-blue-fg">,</span> cls<span class="ansi-blue-fg">)</span><span class="ansi-blue-fg">.</span>load<span class="ansi-blue-fg">(</span><span class="ansi-blue-fg">*</span>args<span class="ansi-blue-fg">,</span> <span class="ansi-blue-fg">**</span>kwargs<span class="ansi-blue-fg">)</span>
<span class="ansi-green-intense-fg ansi-bold">   1231</span>         <span class="ansi-green-fg">if</span> <span class="ansi-green-fg">not</span> hasattr<span class="ansi-blue-fg">(</span>model<span class="ansi-blue-fg">,</span> <span class="ansi-blue-fg">&#39;ns_exponent&#39;</span><span class="ansi-blue-fg">)</span><span class="ansi-blue-fg">:</span>
<span class="ansi-green-intense-fg ansi-bold">   1232</span>             model<span class="ansi-blue-fg">.</span>ns_exponent <span class="ansi-blue-fg">=</span> <span class="ansi-cyan-fg">0.75</span>

<span class="ansi-green-fg">/usr/local/lib/python3.6/dist-packages/gensim/models/base_any2vec.py</span> in <span class="ansi-cyan-fg">load</span><span class="ansi-blue-fg">(cls, fname_or_handle, **kwargs)</span>
<span class="ansi-green-intense-fg ansi-bold">    600</span> 
<span class="ansi-green-intense-fg ansi-bold">    601</span>         &#34;&#34;&#34;
<span class="ansi-green-fg">--&gt; 602</span><span class="ansi-red-fg">         </span><span class="ansi-green-fg">return</span> super<span class="ansi-blue-fg">(</span>BaseAny2VecModel<span class="ansi-blue-fg">,</span> cls<span class="ansi-blue-fg">)</span><span class="ansi-blue-fg">.</span>load<span class="ansi-blue-fg">(</span>fname_or_handle<span class="ansi-blue-fg">,</span> <span class="ansi-blue-fg">**</span>kwargs<span class="ansi-blue-fg">)</span>
<span class="ansi-green-intense-fg ansi-bold">    603</span> 
<span class="ansi-green-intense-fg ansi-bold">    604</span>     <span class="ansi-green-fg">def</span> save<span class="ansi-blue-fg">(</span>self<span class="ansi-blue-fg">,</span> fname_or_handle<span class="ansi-blue-fg">,</span> <span class="ansi-blue-fg">**</span>kwargs<span class="ansi-blue-fg">)</span><span class="ansi-blue-fg">:</span>

<span class="ansi-green-fg">/usr/local/lib/python3.6/dist-packages/gensim/utils.py</span> in <span class="ansi-cyan-fg">load</span><span class="ansi-blue-fg">(cls, fname, mmap)</span>
<span class="ansi-green-intense-fg ansi-bold">    433</span>         compress<span class="ansi-blue-fg">,</span> subname <span class="ansi-blue-fg">=</span> SaveLoad<span class="ansi-blue-fg">.</span>_adapt_by_suffix<span class="ansi-blue-fg">(</span>fname<span class="ansi-blue-fg">)</span>
<span class="ansi-green-intense-fg ansi-bold">    434</span> 
<span class="ansi-green-fg">--&gt; 435</span><span class="ansi-red-fg">         </span>obj <span class="ansi-blue-fg">=</span> unpickle<span class="ansi-blue-fg">(</span>fname<span class="ansi-blue-fg">)</span>
<span class="ansi-green-intense-fg ansi-bold">    436</span>         obj<span class="ansi-blue-fg">.</span>_load_specials<span class="ansi-blue-fg">(</span>fname<span class="ansi-blue-fg">,</span> mmap<span class="ansi-blue-fg">,</span> compress<span class="ansi-blue-fg">,</span> subname<span class="ansi-blue-fg">)</span>
<span class="ansi-green-intense-fg ansi-bold">    437</span>         logger<span class="ansi-blue-fg">.</span>info<span class="ansi-blue-fg">(</span><span class="ansi-blue-fg">&#34;loaded %s&#34;</span><span class="ansi-blue-fg">,</span> fname<span class="ansi-blue-fg">)</span>

<span class="ansi-green-fg">/usr/local/lib/python3.6/dist-packages/gensim/utils.py</span> in <span class="ansi-cyan-fg">unpickle</span><span class="ansi-blue-fg">(fname)</span>
<span class="ansi-green-intense-fg ansi-bold">   1393</span> 
<span class="ansi-green-intense-fg ansi-bold">   1394</span>     &#34;&#34;&#34;
<span class="ansi-green-fg">-&gt; 1395</span><span class="ansi-red-fg">     </span><span class="ansi-green-fg">with</span> open<span class="ansi-blue-fg">(</span>fname<span class="ansi-blue-fg">,</span> <span class="ansi-blue-fg">&#39;rb&#39;</span><span class="ansi-blue-fg">)</span> <span class="ansi-green-fg">as</span> f<span class="ansi-blue-fg">:</span>
<span class="ansi-green-intense-fg ansi-bold">   1396</span>         <span class="ansi-red-fg"># Because of loading from S3 load can&#39;t be used (missing readline in smart_open)</span>
<span class="ansi-green-intense-fg ansi-bold">   1397</span>         <span class="ansi-green-fg">if</span> sys<span class="ansi-blue-fg">.</span>version_info <span class="ansi-blue-fg">&gt;</span> <span class="ansi-blue-fg">(</span><span class="ansi-cyan-fg">3</span><span class="ansi-blue-fg">,</span> <span class="ansi-cyan-fg">0</span><span class="ansi-blue-fg">)</span><span class="ansi-blue-fg">:</span>

<span class="ansi-green-fg">/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py</span> in <span class="ansi-cyan-fg">open</span><span class="ansi-blue-fg">(uri, mode, buffering, encoding, errors, newline, closefd, opener, ignore_ext, transport_params)</span>
<span class="ansi-green-intense-fg ansi-bold">    185</span>         encoding<span class="ansi-blue-fg">=</span>encoding<span class="ansi-blue-fg">,</span>
<span class="ansi-green-intense-fg ansi-bold">    186</span>         errors<span class="ansi-blue-fg">=</span>errors<span class="ansi-blue-fg">,</span>
<span class="ansi-green-fg">--&gt; 187</span><span class="ansi-red-fg">         </span>newline<span class="ansi-blue-fg">=</span>newline<span class="ansi-blue-fg">,</span>
<span class="ansi-green-intense-fg ansi-bold">    188</span>     )
<span class="ansi-green-intense-fg ansi-bold">    189</span>     <span class="ansi-green-fg">if</span> fobj <span class="ansi-green-fg">is</span> <span class="ansi-green-fg">not</span> <span class="ansi-green-fg">None</span><span class="ansi-blue-fg">:</span>

<span class="ansi-green-fg">/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py</span> in <span class="ansi-cyan-fg">_shortcut_open</span><span class="ansi-blue-fg">(uri, mode, ignore_ext, buffering, encoding, errors, newline)</span>
<span class="ansi-green-intense-fg ansi-bold">    285</span>         open_kwargs<span class="ansi-blue-fg">[</span><span class="ansi-blue-fg">&#39;errors&#39;</span><span class="ansi-blue-fg">]</span> <span class="ansi-blue-fg">=</span> errors
<span class="ansi-green-intense-fg ansi-bold">    286</span> 
<span class="ansi-green-fg">--&gt; 287</span><span class="ansi-red-fg">     </span><span class="ansi-green-fg">return</span> _builtin_open<span class="ansi-blue-fg">(</span>local_path<span class="ansi-blue-fg">,</span> mode<span class="ansi-blue-fg">,</span> buffering<span class="ansi-blue-fg">=</span>buffering<span class="ansi-blue-fg">,</span> <span class="ansi-blue-fg">**</span>open_kwargs<span class="ansi-blue-fg">)</span>
<span class="ansi-green-intense-fg ansi-bold">    288</span> 
<span class="ansi-green-intense-fg ansi-bold">    289</span> 

<span class="ansi-red-fg">FileNotFoundError</span>: [Errno 2] No such file or directory: &#39;test_data/models/word2vec_libest.model&#39;</pre>
</div>
</div>

</div>
</div>

</div>
</div>