ds4se/docs/mgmnt.corpus.html

---

title: Corpus Representation

keywords: fastai
sidebar: home_sidebar

summary: "This module comprises corpus representation from a file or Mongo DB"
---
<!--

#################################################
### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ###
#################################################
# file to edit: nbs/24_mgmnt.corpus.ipynb
# command to build the docs after a change: nbdev_build_docs

-->

<div class="container" id="notebook-container">
    
<div class="cell border-box-sizing code_cell rendered">

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="o">!</span> pip install nltk
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">

<div class="output_subarea output_stream output_stdout output_text">
<pre>Collecting nltk
  Downloading https://files.pythonhosted.org/packages/f6/1d/d925cfb4f324ede997f6d47bea4d9babba51b49e87a767c170b77005889d/nltk-3.4.5.zip (1.5MB)
     |████████████████████████████████| 1.5MB 3.7MB/s eta 0:00:01
Requirement already satisfied: six in /usr/lib/python3/dist-packages (from nltk) (1.11.0)
Building wheels for collected packages: nltk
  Building wheel for nltk (setup.py) ... done
  Created wheel for nltk: filename=nltk-3.4.5-cp36-none-any.whl size=1450716 sha256=df506b56c0ffd02787fa55b4a93de88f9ecbc6e5e6568333dd200433c572382f
  Stored in directory: /root/.cache/pip/wheels/96/86/f6/68ab24c23f207c0077381a5e3904b2815136b879538a24b483
Successfully built nltk
Installing collected packages: nltk
Successfully installed nltk-3.4.5
<span class="ansi-yellow-fg">WARNING: You are using pip version 19.2.3, however version 20.0.2 is available.
You should consider upgrading via the &#39;pip install --upgrade pip&#39; command.</span>
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">class</span> <span class="nc">Corpus</span><span class="p">:</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Represents a set of source artifacts, a set of target artifacts,</span>
<span class="sd">    and a ground truth model that represents the links between them.</span>
<span class="sd">    NOTE: This assumes that artifacts separated by line in the same file begin with</span>
<span class="sd">    an ID. Thus, the first word of every line will not be considered part of the artifact</span>
<span class="sd">    for semantic purposes</span>
<span class="sd">    Attributes:</span>
<span class="sd">        _name (str): The identifier of the corpus dataset</span>
<span class="sd">        _corpus_root (str): Path to the root directory of the corpus, from</span>
<span class="sd">            which the source, target, and truth files will be derived</span>
<span class="sd">        _sources (list of str): one str per artifact</span>
<span class="sd">        _targets (list of str): one str per artifact</span>
<span class="sd">        _truth (dict of str:(dict of str:int)): Holds the truth values of links</span>
<span class="sd">            between sources and targets. _truth[source][target] == 1 if link exists,</span>
<span class="sd">            0 otherwise.</span>
<span class="sd">        _source_index (list of str): aligned with source so that source_index[i]</span>
<span class="sd">            contains the filename or identifier for source[i]</span>
<span class="sd">        _target_index (list of str): aligned with target so that</span>
<span class="sd">            target_index[i] contains the filename or identifier for target[i]</span>
<span class="sd">        _filetype_whitelist (list of str): See __init__ documentation</span>
<span class="sd">        _filetype_blacklist(list of str): See __init__ documentation</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">corpus_root</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">,</span> 
                 <span class="n">source_path</span><span class="o">=</span><span class="s1">&#39;requirements&#39;</span><span class="p">,</span> 
                 <span class="n">target_path</span><span class="o">=</span><span class="s1">&#39;source_code&#39;</span><span class="p">,</span> 
                 <span class="n">stop_words_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">truth_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> 
                 <span class="n">execution_traces</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">corpus_code</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> 
                 <span class="n">languages</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;english&#39;</span><span class="p">],</span> 
                 <span class="n">filetype_whitelist</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> 
                 <span class="n">filetype_blacklist</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> 
                 <span class="n">blank</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Args:</span>
<span class="sd">            name (str): The identifier of the corpus dataset</span>
<span class="sd">            corpus_root (str): Path to the root directory of the corpus, from</span>
<span class="sd">                which the source, target, and truth files will be derived</span>
<span class="sd">            source_path (str): Path from corpus root to file or directory of</span>
<span class="sd">                source corpus. If directory, artifacts will be parsed from</span>
<span class="sd">                files in the directory. If file, artifacts will be parsed from</span>
<span class="sd">                lines in the file.</span>
<span class="sd">            target_path (str): Path from corpus root to file or directory of</span>
<span class="sd">                target corpus. If directory, artifacts will be parsed from</span>
<span class="sd">                files in the directory. If file, artifacts will be parsed from</span>
<span class="sd">                lines in the file.</span>
<span class="sd">            truth_path (str): Path to file containing ground truth</span>
<span class="sd">            natural_language (str or list of str, optional): The human language(s) that artifacts</span>
<span class="sd">                are written in. Will be matched to the corresponding stop words</span>
<span class="sd">                list and stemmer.</span>
<span class="sd">            code_language (str or list of str, optional): The programming language(s) that artifacts</span>
<span class="sd">                are written in. Will be matched to the corresponding stop words list.</span>
<span class="sd">            filetype_whitelist (list of str, optional): List of file extensions</span>
<span class="sd">                which will be converted into artifacts. If None (default), all</span>
<span class="sd">                file extensions will be converted. Do not include dot.</span>
<span class="sd">            filetype_blacklist(list of str, optional): List of file extensions</span>
<span class="sd">                which will be ignored when converting into artifacts. If None</span>
<span class="sd">                (default), all file extensions will be converted. Do not include</span>
<span class="sd">                dot.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="n">name</span>

        <span class="k">if</span> <span class="n">blank</span><span class="p">:</span>
            <span class="k">return</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_corpus_root</span> <span class="o">=</span> <span class="n">corpus_root</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_filetype_whitelist</span> <span class="o">=</span> <span class="n">filetype_whitelist</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_filetype_blacklist</span> <span class="o">=</span> <span class="n">filetype_blacklist</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_source_index</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_target_index</span> <span class="o">=</span> <span class="p">[]</span>

        <span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">source_path</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="nb">list</span><span class="p">:</span>
            <span class="n">source_path</span> <span class="o">=</span> <span class="p">[</span><span class="n">source_path</span><span class="p">]</span>

        <span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">target_path</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="nb">list</span><span class="p">:</span>
            <span class="n">target_path</span> <span class="o">=</span> <span class="p">[</span><span class="n">target_path</span><span class="p">]</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_sources</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">for</span> <span class="n">sp</span> <span class="ow">in</span> <span class="n">source_path</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_sources</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_artifacts</span><span class="p">(</span><span class="n">sp</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_source_index</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_targets</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">for</span> <span class="n">tp</span> <span class="ow">in</span> <span class="n">target_path</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_targets</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parse_artifacts</span><span class="p">(</span><span class="n">tp</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_target_index</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_truth</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="k">if</span> <span class="n">truth_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_truth</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_all_links</span> <span class="o">=</span> <span class="p">[]</span>

            <span class="k">for</span> <span class="n">source</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_source_index</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_truth</span><span class="p">[</span><span class="n">source</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
                <span class="k">for</span> <span class="n">target</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_target_index</span><span class="p">:</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">_truth</span><span class="p">[</span><span class="n">source</span><span class="p">][</span><span class="n">target</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">_all_links</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">source</span><span class="p">,</span> <span class="n">target</span><span class="p">))</span>

            <span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">truth_path</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="nb">list</span><span class="p">:</span>
                <span class="n">truth_path</span> <span class="o">=</span> <span class="p">[</span><span class="n">truth_path</span><span class="p">]</span>

            <span class="k">for</span> <span class="n">tp</span> <span class="ow">in</span> <span class="n">truth_path</span><span class="p">:</span>
                <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">corpus_root</span> <span class="o">+</span> <span class="n">tp</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">truth_file</span><span class="p">:</span>
                    <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">truth_file</span><span class="o">.</span><span class="n">readlines</span><span class="p">():</span>
                        <span class="n">tokens</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>

                        <span class="n">source</span> <span class="o">=</span> <span class="n">tokens</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>

                        <span class="k">if</span> <span class="n">source</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_truth</span><span class="p">:</span>
                            <span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="s2">&quot;Source artifact </span><span class="se">\&#39;</span><span class="s2">&quot;</span> <span class="o">+</span> <span class="n">source</span> <span class="o">+</span> <span class="s2">&quot;</span><span class="se">\&#39;</span><span class="s2"> in truth file not a recognized artifact&quot;</span><span class="p">)</span>

                        <span class="k">for</span> <span class="n">target</span> <span class="ow">in</span> <span class="n">tokens</span><span class="p">[</span><span class="mi">1</span><span class="p">:]:</span>
                            <span class="k">if</span> <span class="n">target</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_truth</span><span class="p">[</span><span class="n">source</span><span class="p">]:</span>
                                <span class="nb">print</span><span class="p">(</span><span class="n">source</span><span class="p">)</span>
                                <span class="nb">print</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_truth</span><span class="p">[</span><span class="n">source</span><span class="p">])</span>
                                <span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="s2">&quot;Target artifact </span><span class="se">\&#39;</span><span class="s2">&quot;</span> <span class="o">+</span> <span class="n">target</span> <span class="o">+</span> <span class="s2">&quot;</span><span class="se">\&#39;</span><span class="s2"> in truth file not a recognized artifact&quot;</span><span class="p">)</span>

                            <span class="bp">self</span><span class="o">.</span><span class="n">_truth</span><span class="p">[</span><span class="n">source</span><span class="p">][</span><span class="n">target</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_execution_traces</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="k">if</span> <span class="n">execution_traces</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_execution_traces</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
            <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">corpus_root</span> <span class="o">+</span> <span class="n">execution_traces</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">et_file</span><span class="p">:</span>
                <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">et_file</span><span class="o">.</span><span class="n">readlines</span><span class="p">():</span>
                    <span class="n">tokens</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
                    <span class="k">if</span> <span class="n">tokens</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_execution_traces</span><span class="p">:</span>
                        <span class="bp">self</span><span class="o">.</span><span class="n">_execution_traces</span><span class="p">[</span><span class="n">tokens</span><span class="p">[</span><span class="mi">0</span><span class="p">]]</span> <span class="o">=</span> <span class="p">[]</span>
                    <span class="k">if</span> <span class="n">tokens</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_execution_traces</span><span class="p">:</span>
                        <span class="bp">self</span><span class="o">.</span><span class="n">_execution_traces</span><span class="p">[</span><span class="n">tokens</span><span class="p">[</span><span class="mi">1</span><span class="p">]]</span> <span class="o">=</span> <span class="p">[]</span>

                    <span class="bp">self</span><span class="o">.</span><span class="n">_execution_traces</span><span class="p">[</span><span class="n">tokens</span><span class="p">[</span><span class="mi">0</span><span class="p">]]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">tokens</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">_execution_traces</span><span class="p">[</span><span class="n">tokens</span><span class="p">[</span><span class="mi">1</span><span class="p">]]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">tokens</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_stop_words</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_stemmers</span> <span class="o">=</span> <span class="p">[]</span>

        <span class="k">if</span> <span class="n">languages</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">language</span> <span class="ow">in</span> <span class="n">languages</span><span class="p">:</span>
                <span class="c1"># check if there&#39;s a stop word list</span>
                <span class="n">stop_words_path</span> <span class="o">=</span>  <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">getcwd</span><span class="p">(),</span> <span class="s1">&#39;test_data/config_corpus/&#39;</span> <span class="o">+</span> 
                                                <span class="n">language</span> <span class="o">+</span> <span class="s1">&#39;_stop_words.txt&#39;</span><span class="p">)</span>
                <span class="k">try</span><span class="p">:</span>
                    <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">stop_words_path</span><span class="p">)</span> <span class="k">as</span> <span class="n">stop_words_file</span><span class="p">:</span>
                        <span class="bp">self</span><span class="o">.</span><span class="n">_stop_words</span> <span class="o">+=</span>         <span class="n">stop_words_file</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
                        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Found stop words file: &quot;</span> <span class="o">+</span> <span class="n">language</span> <span class="o">+</span> <span class="s2">&quot;_stop_words.txt&quot;</span><span class="p">)</span>
                <span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span>
                    <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;No stop words found for language: [&quot;</span> <span class="o">+</span> <span class="n">language</span> <span class="o">+</span> <span class="s2">&quot;]&quot;</span><span class="p">)</span>

                <span class="c1"># check if there&#39;s a stemmer</span>
                <span class="k">try</span><span class="p">:</span>
                    <span class="n">stemmer</span> <span class="o">=</span> <span class="n">SnowballStemmer</span><span class="p">(</span><span class="n">language</span><span class="p">)</span>
                    <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Detected natural language: [&quot;</span> <span class="o">+</span> <span class="n">language</span> <span class="o">+</span> <span class="s2">&quot;], generating stemmer&quot;</span><span class="p">)</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">_stemmers</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">stemmer</span><span class="p">)</span>
                <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
                    <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;No natural language stemmer detected for language: [&quot;</span> <span class="o">+</span> <span class="n">language</span> <span class="o">+</span> <span class="s2">&quot;]&quot;</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_corpus_code</span> <span class="o">=</span> <span class="n">corpus_code</span>

    <span class="k">def</span> <span class="nf">_parse_artifacts</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">index</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Reads and indexes artifacts from file or files given in path</span>
<span class="sd">        Args:</span>
<span class="sd">            path (str): Path to the file or folder from which to parse artifacts</span>
<span class="sd">            index (list of str): Stores the identifiers for each artifact. This</span>
<span class="sd">                list should be empty when this method is called (the method will</span>
<span class="sd">                populate it).</span>
<span class="sd">        Returns:</span>
<span class="sd">            list of str: The list of artifacts</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">root</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_corpus_root</span>
        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Finding artifacts in: &quot;</span> <span class="o">+</span> <span class="n">root</span> <span class="o">+</span> <span class="n">path</span><span class="p">)</span>
        <span class="n">store</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">root</span> <span class="o">+</span> <span class="n">path</span><span class="p">):</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Getting artifacts by line from file&quot;</span><span class="p">)</span>
            <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">root</span> <span class="o">+</span> <span class="n">path</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s1">&#39;ignore&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">artifacts_file</span><span class="p">:</span>
                <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">artifacts_file</span><span class="o">.</span><span class="n">readlines</span><span class="p">():</span>
                    <span class="n">tokens</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
                    <span class="n">artifact</span> <span class="o">=</span> <span class="s1">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">tokens</span><span class="p">[</span><span class="mi">1</span><span class="p">:])</span>
                    <span class="n">index</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">tokens</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
                    <span class="n">store</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">artifact</span><span class="p">)</span>
                <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Read &quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">store</span><span class="p">))</span> <span class="o">+</span> <span class="s2">&quot; artifacts from file&quot;</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Getting artifacts by file from directory&quot;</span><span class="p">)</span>
            <span class="k">for</span> <span class="n">element</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">listdir</span><span class="p">(</span><span class="n">root</span> <span class="o">+</span> <span class="n">path</span><span class="p">):</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_parse_artifacts_recur</span><span class="p">(</span><span class="n">path</span> <span class="o">+</span> <span class="s1">&#39;/&#39;</span><span class="p">,</span> <span class="n">element</span><span class="p">,</span> <span class="n">index</span><span class="p">,</span> <span class="n">store</span><span class="p">)</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Read &quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">store</span><span class="p">))</span> <span class="o">+</span> <span class="s2">&quot; artifacts from directory&quot;</span><span class="p">)</span>

        <span class="nb">print</span><span class="p">()</span>
        <span class="k">return</span> <span class="n">store</span>

    <span class="k">def</span> <span class="nf">_parse_artifacts_recur</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">subroot</span><span class="p">,</span> <span class="n">element</span><span class="p">,</span> <span class="n">index</span><span class="p">,</span> <span class="n">store</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Recursively searches for files in the directory given in the directory</span>
<span class="sd">        path from the corpus root.</span>
<span class="sd">        Args:</span>
<span class="sd">            path (str): Path to a file or directory. If path is a file, this is</span>
<span class="sd">                the base case and the file is read and an artifact is created</span>
<span class="sd">                and indexed</span>
<span class="sd">            index (list of str): Cumulatively stores the identifiers for each artifact.</span>
<span class="sd">            store (list of str): Cumulatively stores the artifacts.</span>
<span class="sd">        Returns:</span>
<span class="sd">            None</span>
<span class="sd">        &quot;&quot;&quot;</span>

        <span class="n">root</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_corpus_root</span>
        <span class="n">full_path</span> <span class="o">=</span> <span class="n">root</span> <span class="o">+</span> <span class="n">subroot</span> <span class="o">+</span> <span class="n">element</span>

        <span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">full_path</span><span class="p">):</span>
            <span class="n">extension</span> <span class="o">=</span> <span class="n">element</span><span class="p">[</span><span class="n">element</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span><span class="o">+</span><span class="mi">1</span><span class="p">:]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>

            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_filetype_whitelist</span> <span class="ow">and</span> <span class="n">extension</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_filetype_whitelist</span><span class="p">:</span>
                <span class="k">return</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_filetype_blacklist</span> <span class="ow">and</span> <span class="n">extension</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_filetype_blacklist</span><span class="p">:</span>
                <span class="k">return</span>

            <span class="n">element_clean</span> <span class="o">=</span> <span class="n">element</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">,</span> <span class="s1">&#39;.&#39;</span><span class="p">)</span>
            <span class="n">index</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">element_clean</span><span class="p">)</span>
            <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">full_path</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s1">&#39;ignore&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">artifact_file</span><span class="p">:</span>
                <span class="n">store</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">artifact_file</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>

        <span class="k">else</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">sub_element</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">listdir</span><span class="p">(</span><span class="n">root</span> <span class="o">+</span> <span class="n">subroot</span> <span class="o">+</span> <span class="n">element</span><span class="p">):</span>
                <span class="n">sub_element</span> <span class="o">=</span> <span class="n">element</span> <span class="o">+</span> <span class="s1">&#39;/&#39;</span> <span class="o">+</span> <span class="n">sub_element</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_parse_artifacts_recur</span><span class="p">(</span><span class="n">subroot</span><span class="p">,</span> <span class="n">sub_element</span><span class="p">,</span> <span class="n">index</span><span class="p">,</span> <span class="n">store</span><span class="p">)</span>

    <span class="c1">#Getters            </span>
    <span class="k">def</span> <span class="nf">get_sources</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sources</span>

    <span class="k">def</span> <span class="nf">get_targets</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_targets</span>

    <span class="k">def</span> <span class="nf">get_source_names</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_source_index</span>

    <span class="k">def</span> <span class="nf">get_target_names</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_target_index</span>

    <span class="k">def</span> <span class="nf">get_source_artifact_at_index</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">index</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sources</span><span class="p">[</span><span class="n">index</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">get_target_artifact_at_index</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">index</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_targets</span><span class="p">[</span><span class="n">index</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">get_source_artifact_by_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">index</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_source_index</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sources</span><span class="p">[</span><span class="n">index</span><span class="p">]</span>
        <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Source artifact </span><span class="se">\&#39;</span><span class="s2">&quot;</span> <span class="o">+</span> <span class="n">name</span> <span class="o">+</span> <span class="s2">&quot;</span><span class="se">\&#39;</span><span class="s2"> not found&quot;</span><span class="p">)</span>
            <span class="k">return</span> <span class="kc">None</span>

    <span class="k">def</span> <span class="nf">get_target_artifact_by_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">index</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_target_index</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_targets</span><span class="p">[</span><span class="n">index</span><span class="p">]</span>
        <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Target artifact </span><span class="se">\&#39;</span><span class="s2">&quot;</span> <span class="o">+</span> <span class="n">name</span> <span class="o">+</span> <span class="s2">&quot;</span><span class="se">\&#39;</span><span class="s2"> not found&quot;</span><span class="p">)</span>
            <span class="k">return</span> <span class="kc">None</span>

    <span class="k">def</span> <span class="nf">get_source_name_by_index</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">index</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_source_index</span><span class="p">[</span><span class="n">index</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">get_target_name_by_index</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">index</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_target_index</span><span class="p">[</span><span class="n">index</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">get_truth_value</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">source</span><span class="p">,</span> <span class="n">target</span><span class="p">):</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_truth</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_truth</span><span class="p">[</span><span class="n">source</span><span class="p">][</span><span class="n">target</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">get_truth_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="nb">dict</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_truth</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">get_execution_trace</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">artifact</span><span class="p">):</span>
        <span class="k">if</span> <span class="n">artifact</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_execution_traces</span><span class="p">:</span>
            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_execution_traces</span><span class="p">[</span><span class="n">artifact</span><span class="p">]</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">return</span> <span class="p">[]</span>

    <span class="k">def</span> <span class="nf">get_all_execution_traces</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="nb">dict</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_execution_traces</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">get_stop_words</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_stop_words</span>

    <span class="k">def</span> <span class="nf">get_stemmers</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_stemmers</span>

    <span class="k">def</span> <span class="nf">get_subset</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">percent</span><span class="p">):</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_truth</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;No ground truth in corpus&quot;</span><span class="p">)</span>
            <span class="k">return</span>

        <span class="n">total_num_links</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_all_links</span><span class="p">)</span>
        <span class="n">num_links</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">total_num_links</span> <span class="o">*</span> <span class="p">(</span><span class="n">percent</span> <span class="o">/</span> <span class="mi">100</span><span class="p">))</span>

        <span class="n">link_subset</span> <span class="o">=</span> <span class="n">random</span><span class="o">.</span><span class="n">sample</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_all_links</span><span class="p">,</span> <span class="n">num_links</span><span class="p">)</span>

        <span class="n">link_subset_dict</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>

        <span class="k">for</span> <span class="n">link</span> <span class="ow">in</span> <span class="n">link_subset</span><span class="p">:</span>
            <span class="n">source</span> <span class="o">=</span> <span class="n">link</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
            <span class="n">target</span> <span class="o">=</span> <span class="n">link</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
            <span class="n">link_status</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_truth_value</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>

            <span class="k">if</span> <span class="n">source</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">link_subset_dict</span><span class="p">:</span>
                <span class="n">link_subset_dict</span><span class="p">[</span><span class="n">source</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>

            <span class="n">link_subset_dict</span><span class="p">[</span><span class="n">source</span><span class="p">][</span><span class="n">target</span><span class="p">]</span> <span class="o">=</span> <span class="n">link_status</span>

        <span class="k">return</span> <span class="n">link_subset_dict</span>

    <span class="k">def</span> <span class="nf">get_subsets</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">percent</span><span class="p">,</span> <span class="n">n_trials</span><span class="p">):</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_truth</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;No ground truth in corpus&quot;</span><span class="p">)</span>
            <span class="k">return</span>

        <span class="n">subsets</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_trials</span><span class="p">):</span>
            <span class="n">subsets</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_subset</span><span class="p">(</span><span class="n">percent</span><span class="p">))</span>
        <span class="k">return</span> <span class="n">subsets</span>

    <span class="k">def</span> <span class="nf">get_positive_link_subset</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">num_sources</span><span class="p">,</span> <span class="n">accuracy</span><span class="o">=</span><span class="mf">0.75</span><span class="p">):</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_truth</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;No ground truth in corpus&quot;</span><span class="p">)</span>
            <span class="k">return</span>

        <span class="n">source_samples</span> <span class="o">=</span> <span class="n">random</span><span class="o">.</span><span class="n">sample</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_source_names</span><span class="p">(),</span> <span class="n">num_sources</span><span class="p">)</span>
        <span class="n">link_subset_dict</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>

        <span class="n">targets</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_target_names</span><span class="p">()</span>

        <span class="k">for</span> <span class="n">source</span> <span class="ow">in</span> <span class="n">source_samples</span><span class="p">:</span>
            <span class="n">link_subset_dict</span><span class="p">[</span><span class="n">source</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>

            <span class="n">recall_rate</span> <span class="o">=</span> <span class="n">random</span><span class="o">.</span><span class="n">gauss</span><span class="p">(</span><span class="n">accuracy</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">)</span>
            <span class="k">if</span> <span class="n">recall_rate</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">:</span>
                <span class="n">recall_rate</span> <span class="o">=</span> <span class="mi">0</span>
            <span class="k">elif</span> <span class="n">recall_rate</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
                <span class="n">recall_rate</span> <span class="o">=</span> <span class="mi">1</span>

            <span class="n">actual_links</span> <span class="o">=</span> <span class="p">[</span><span class="n">target</span> <span class="k">for</span> <span class="n">target</span> <span class="ow">in</span> <span class="n">targets</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_truth_value</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">]</span>

            <span class="n">num_links_found</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">actual_links</span><span class="p">)</span> <span class="o">*</span> <span class="n">recall_rate</span><span class="p">)</span>

            <span class="k">for</span> <span class="n">target</span> <span class="ow">in</span> <span class="n">targets</span><span class="p">:</span>
                <span class="n">link_subset_dict</span><span class="p">[</span><span class="n">source</span><span class="p">][</span><span class="n">target</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span>

            <span class="n">links_found</span> <span class="o">=</span> <span class="n">random</span><span class="o">.</span><span class="n">sample</span><span class="p">(</span><span class="n">actual_links</span><span class="p">,</span> <span class="n">num_links_found</span><span class="p">)</span>
            <span class="k">for</span> <span class="n">target</span> <span class="ow">in</span> <span class="n">links_found</span><span class="p">:</span>
                <span class="n">link_subset_dict</span><span class="p">[</span><span class="n">source</span><span class="p">][</span><span class="n">target</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span>

        <span class="k">return</span> <span class="n">link_subset_dict</span>

    <span class="k">def</span> <span class="nf">get_positive_link_subsets</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">num_sources</span><span class="p">,</span> <span class="n">n_trials</span><span class="p">,</span> <span class="n">accuracy</span><span class="o">=</span><span class="mf">0.75</span><span class="p">):</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_truth</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;No ground truth in corpus&quot;</span><span class="p">)</span>
            <span class="k">return</span>

        <span class="n">subsets</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_trials</span><span class="p">):</span>
            <span class="n">subsets</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_positive_link_subset</span><span class="p">(</span><span class="n">num_sources</span><span class="p">,</span> <span class="n">accuracy</span><span class="o">=</span><span class="n">accuracy</span><span class="p">))</span>

        <span class="k">return</span> <span class="n">subsets</span>

    <span class="k">def</span> <span class="nf">get_link_sample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">num_positive</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">num_negative</span><span class="o">=</span><span class="mi">5</span><span class="p">):</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_truth</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;No ground truth in corpus&quot;</span><span class="p">)</span>
            <span class="k">return</span>

        <span class="n">sources</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_source_names</span><span class="p">()</span>
        <span class="n">targets</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_target_names</span><span class="p">()</span>

        <span class="n">positives</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="n">negatives</span> <span class="o">=</span> <span class="p">[]</span>

        <span class="k">for</span> <span class="n">source</span> <span class="ow">in</span> <span class="n">sources</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">target</span> <span class="ow">in</span> <span class="n">targets</span><span class="p">:</span>
                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_truth_value</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
                    <span class="n">positives</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">source</span><span class="p">,</span> <span class="n">target</span><span class="p">))</span>
                <span class="k">else</span><span class="p">:</span>
                    <span class="n">negatives</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">source</span><span class="p">,</span> <span class="n">target</span><span class="p">))</span>

        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">positives</span><span class="p">)</span> <span class="o">&lt;</span> <span class="n">num_positive</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Warning: &quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">num_positive</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot; positive links were requested, but only &quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">positives</span><span class="p">))</span> <span class="o">+</span> <span class="s2">&quot; positive links exist.&quot;</span><span class="p">)</span>
            <span class="n">positive_subset</span> <span class="o">=</span> <span class="n">positives</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">positive_subset</span> <span class="o">=</span> <span class="n">random</span><span class="o">.</span><span class="n">sample</span><span class="p">(</span><span class="n">positives</span><span class="p">,</span> <span class="n">num_positive</span><span class="p">)</span>

        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">negatives</span><span class="p">)</span> <span class="o">&lt;</span> <span class="n">num_negative</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Warning: &quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">num_negative</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot; negative links were requested, but only &quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">negatives</span><span class="p">))</span> <span class="o">+</span> <span class="s2">&quot; negative links exist.&quot;</span><span class="p">)</span>
            <span class="n">negative_subset</span> <span class="o">=</span> <span class="n">negatives</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">negative_subset</span> <span class="o">=</span> <span class="n">random</span><span class="o">.</span><span class="n">sample</span><span class="p">(</span><span class="n">negatives</span><span class="p">,</span> <span class="n">num_negative</span><span class="p">)</span>

        <span class="k">return</span> <span class="n">positive_subset</span> <span class="o">+</span> <span class="n">negative_subset</span>

    <span class="k">def</span> <span class="nf">get_corpus_name</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_name</span>

    <span class="k">def</span> <span class="nf">get_corpus_code</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_corpus_code</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_corpus_code</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;No corpus code found for: &quot;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_corpus_name</span><span class="p">())</span>

    <span class="k">def</span> <span class="nf">get_corpus_root</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_corpus_root</span>

    <span class="k">def</span> <span class="nf">get_raw_string</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="n">output</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
        <span class="k">for</span> <span class="n">doc</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sources</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">_targets</span><span class="p">:</span>
            <span class="n">output</span> <span class="o">+=</span> <span class="n">doc</span> <span class="o">+</span> <span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span>
        <span class="k">return</span> <span class="n">output</span>

    <span class="k">def</span> <span class="nf">generate_raw_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">output_filename</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">if</span> <span class="n">output_filename</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">output_filename</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_corpus_root</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">_corpus_code</span> <span class="o">+</span> <span class="s1">&#39;_raw_corpus.txt&#39;</span>
        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">output_filename</span><span class="p">,</span> <span class="s1">&#39;w+&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">output_file</span><span class="p">:</span>
            <span class="n">output_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_raw_string</span><span class="p">())</span>

    <span class="k">def</span> <span class="nf">verify_datastore</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">datastore_manager</span><span class="p">):</span>
        <span class="n">sources</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_source_names</span><span class="p">()</span>
        <span class="n">targets</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_target_names</span><span class="p">()</span>

        <span class="n">complete</span> <span class="o">=</span> <span class="kc">True</span>
        <span class="k">for</span> <span class="n">source</span> <span class="ow">in</span> <span class="n">sources</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">target</span> <span class="ow">in</span> <span class="n">targets</span><span class="p">:</span>
                <span class="k">if</span> <span class="ow">not</span> <span class="n">datastore_manager</span><span class="o">.</span><span class="n">file_exists</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="s1">&#39;NUTS&#39;</span><span class="p">):</span>
                    <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Missing file for (&quot;</span> <span class="o">+</span> <span class="n">source</span> <span class="o">+</span> <span class="s2">&quot;, &quot;</span> <span class="o">+</span> <span class="n">target</span> <span class="o">+</span> <span class="s2">&quot;)&quot;</span><span class="p">)</span>
                    <span class="n">complete</span> <span class="o">=</span> <span class="kc">False</span>

        <span class="k">if</span> <span class="n">complete</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;All links have been generated for &quot;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_corpus_name</span><span class="p">()</span> <span class="o">+</span> <span class="s2">&quot;!&quot;</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Some links are missing for &quot;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_corpus_name</span><span class="p">()</span> <span class="o">+</span> <span class="s2">&quot;...&quot;</span><span class="p">)</span>
    <span class="c1"># STATIC</span>
    <span class="nd">@classmethod</span>
    <span class="k">def</span> <span class="nf">get_preset_corpus</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">corpus_code</span><span class="p">):</span>
        <span class="k">if</span> <span class="n">corpus_code</span> <span class="o">==</span> <span class="s1">&#39;LibEST&#39;</span><span class="p">:</span>
            <span class="n">corpus_code</span> <span class="o">=</span> <span class="s1">&#39;0_1&#39;</span>
        <span class="k">elif</span> <span class="n">corpus_code</span> <span class="o">==</span> <span class="s1">&#39;EBT&#39;</span><span class="p">:</span>
            <span class="n">corpus_code</span> <span class="o">=</span> <span class="s1">&#39;1_1&#39;</span>
        <span class="k">elif</span> <span class="n">corpus_code</span> <span class="o">==</span> <span class="s1">&#39;eTOUR&#39;</span><span class="p">:</span>
            <span class="n">corpus_code</span> <span class="o">=</span> <span class="s1">&#39;2_0&#39;</span>
        <span class="k">elif</span> <span class="n">corpus_code</span> <span class="o">==</span> <span class="s1">&#39;iTrust&#39;</span><span class="p">:</span>
            <span class="n">corpus_code</span> <span class="o">=</span> <span class="s1">&#39;3_0&#39;</span>
        <span class="k">elif</span> <span class="n">corpus_code</span> <span class="o">==</span> <span class="s1">&#39;Albergate&#39;</span><span class="p">:</span>
            <span class="n">corpus_code</span> <span class="o">=</span> <span class="s1">&#39;4_0&#39;</span>
        <span class="k">elif</span> <span class="n">corpus_code</span> <span class="o">==</span> <span class="s1">&#39;SMOS&#39;</span><span class="p">:</span>
            <span class="n">corpus_code</span> <span class="o">=</span> <span class="s1">&#39;5_0&#39;</span>

        <span class="k">try</span><span class="p">:</span>
            <span class="n">separator_index</span> <span class="o">=</span> <span class="n">corpus_code</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s1">&#39;_&#39;</span><span class="p">)</span>
        <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Invalid corpus code: &quot;</span> <span class="o">+</span> <span class="n">corpus_code</span><span class="p">)</span>
            <span class="k">return</span>

        <span class="n">dataset</span> <span class="o">=</span> <span class="n">corpus_code</span><span class="p">[:</span><span class="n">separator_index</span><span class="p">]</span>
        <span class="n">subset</span> <span class="o">=</span> <span class="n">corpus_code</span><span class="p">[</span><span class="n">separator_index</span><span class="o">+</span><span class="mi">1</span><span class="p">:]</span>

        <span class="n">dataset_name</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="n">modifier</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="n">source_path</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="n">target_path</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="n">truth_path</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="n">execution_traces</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="n">languages</span> <span class="o">=</span> <span class="kc">None</span>

        <span class="k">if</span> <span class="n">dataset</span> <span class="o">==</span> <span class="s1">&#39;0&#39;</span><span class="p">:</span>
            <span class="n">dataset_name</span> <span class="o">=</span> <span class="s1">&#39;LibEST&#39;</span>
            <span class="n">source_path</span> <span class="o">=</span> <span class="s1">&#39;requirements&#39;</span>
            <span class="n">languages</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;english&#39;</span><span class="p">,</span> <span class="s1">&#39;C&#39;</span><span class="p">]</span>
            <span class="n">execution_traces</span> <span class="o">=</span> <span class="s1">&#39;execution_traces.txt&#39;</span>

            <span class="k">if</span> <span class="n">subset</span> <span class="o">==</span> <span class="s1">&#39;0&#39;</span><span class="p">:</span>
                <span class="n">modifier</span> <span class="o">=</span> <span class="s1">&#39;(RQ to Code and Tests)&#39;</span>
                <span class="n">target_path</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;source_code&#39;</span><span class="p">,</span> <span class="s1">&#39;test&#39;</span><span class="p">]</span>
                <span class="n">truth_path</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;req_to_code_ground.txt&#39;</span><span class="p">,</span> <span class="s1">&#39;req_to_test_ground.txt&#39;</span><span class="p">]</span>

            <span class="k">elif</span> <span class="n">subset</span> <span class="o">==</span> <span class="s1">&#39;1&#39;</span><span class="p">:</span>
                <span class="n">modifier</span> <span class="o">=</span> <span class="s1">&#39;(RQ to Code)&#39;</span>
                <span class="n">target_path</span> <span class="o">=</span> <span class="s1">&#39;source_code&#39;</span>
                <span class="n">truth_path</span> <span class="o">=</span> <span class="s1">&#39;req_to_code_ground.txt&#39;</span>

            <span class="k">elif</span> <span class="n">subset</span> <span class="o">==</span> <span class="s1">&#39;2&#39;</span><span class="p">:</span>
                <span class="n">modifier</span> <span class="o">=</span> <span class="s1">&#39;(RQ to Tests)&#39;</span>
                <span class="n">target_path</span> <span class="o">=</span> <span class="s1">&#39;test&#39;</span>
                <span class="n">truth_path</span> <span class="o">=</span> <span class="s1">&#39;req_to_test_ground.txt&#39;</span>

            <span class="k">else</span><span class="p">:</span>
                <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Unrecognized corpus code: &quot;</span> <span class="o">+</span> <span class="n">corpus_code</span><span class="p">)</span>
                <span class="k">return</span>

        <span class="k">elif</span> <span class="n">dataset</span> <span class="o">==</span> <span class="s1">&#39;1&#39;</span><span class="p">:</span>
            <span class="n">dataset_name</span> <span class="o">=</span> <span class="s1">&#39;EBT&#39;</span>
            <span class="n">source_path</span> <span class="o">=</span> <span class="s1">&#39;requirements.txt&#39;</span>
            <span class="n">languages</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;english&#39;</span><span class="p">]</span>

            <span class="k">if</span> <span class="n">subset</span> <span class="o">==</span> <span class="s1">&#39;0&#39;</span><span class="p">:</span>
                <span class="n">modifier</span> <span class="o">=</span> <span class="s1">&#39;(RQ to Code and Tests)&#39;</span>
                <span class="n">target_path</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;source_code&#39;</span><span class="p">,</span> <span class="s1">&#39;test_cases.txt&#39;</span><span class="p">]</span>
                <span class="n">truth_path</span> <span class="o">=</span> <span class="s1">&#39;both_ground.txt&#39;</span>

            <span class="k">elif</span> <span class="n">subset</span> <span class="o">==</span> <span class="s1">&#39;1&#39;</span><span class="p">:</span>
                <span class="n">modifier</span> <span class="o">=</span> <span class="s1">&#39;(RQ to Code)&#39;</span>
                <span class="n">target_path</span> <span class="o">=</span> <span class="s1">&#39;source_code&#39;</span>
                <span class="n">truth_path</span> <span class="o">=</span> <span class="s1">&#39;code_ground.txt&#39;</span>

            <span class="k">elif</span> <span class="n">subset</span> <span class="o">==</span> <span class="s1">&#39;2&#39;</span><span class="p">:</span>
                <span class="n">modifier</span> <span class="o">=</span> <span class="s1">&#39;(RQ to Test)&#39;</span>
                <span class="n">target_path</span> <span class="o">=</span> <span class="s1">&#39;test_cases.txt&#39;</span>
                <span class="n">truth_path</span> <span class="o">=</span> <span class="s1">&#39;tests_ground.txt&#39;</span>

            <span class="k">else</span><span class="p">:</span>
                <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Unrecognized corpus code: &quot;</span> <span class="o">+</span> <span class="n">corpus_code</span><span class="p">)</span>
                <span class="k">return</span>

        <span class="k">elif</span> <span class="n">dataset</span> <span class="o">==</span> <span class="s1">&#39;2&#39;</span><span class="p">:</span>
            <span class="n">dataset_name</span> <span class="o">=</span> <span class="s1">&#39;eTOUR&#39;</span>
            <span class="n">source_path</span> <span class="o">=</span> <span class="s1">&#39;use_cases_with_translation&#39;</span>
            <span class="n">languages</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;english&#39;</span><span class="p">,</span> <span class="s1">&#39;italian&#39;</span><span class="p">,</span> <span class="s1">&#39;java&#39;</span><span class="p">]</span>
            <span class="k">if</span> <span class="n">subset</span> <span class="o">==</span> <span class="s1">&#39;0&#39;</span><span class="p">:</span>
                <span class="n">modifier</span> <span class="o">=</span> <span class="s1">&#39;(UC to Code)&#39;</span>
                <span class="n">target_path</span> <span class="o">=</span> <span class="s1">&#39;source_code&#39;</span>
                <span class="n">truth_path</span> <span class="o">=</span> <span class="s1">&#39;ground.txt&#39;</span>

            <span class="k">else</span><span class="p">:</span>
                <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Unrecognized corpus code: &quot;</span> <span class="o">+</span> <span class="n">corpus_code</span><span class="p">)</span>
                <span class="k">return</span>

        <span class="k">elif</span> <span class="n">dataset</span> <span class="o">==</span> <span class="s1">&#39;3&#39;</span><span class="p">:</span>
            <span class="n">dataset_name</span> <span class="o">=</span> <span class="s1">&#39;iTrust&#39;</span>
            <span class="n">source_path</span> <span class="o">=</span> <span class="s1">&#39;use_cases&#39;</span>
            <span class="n">languages</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;english&#39;</span><span class="p">,</span> <span class="s1">&#39;java&#39;</span><span class="p">]</span>

            <span class="k">if</span> <span class="n">subset</span> <span class="o">==</span> <span class="s1">&#39;0&#39;</span><span class="p">:</span>
                <span class="n">modifier</span> <span class="o">=</span> <span class="s1">&#39;(UC to Code)&#39;</span>
                <span class="n">target_path</span> <span class="o">=</span> <span class="s1">&#39;source_code&#39;</span>
                <span class="n">truth_path</span> <span class="o">=</span> <span class="s1">&#39;ground.txt&#39;</span>

            <span class="k">else</span><span class="p">:</span>
                <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Unrecognized corpus code: &quot;</span> <span class="o">+</span> <span class="n">corpus_code</span><span class="p">)</span>
                <span class="k">return</span>

        <span class="k">elif</span> <span class="n">dataset</span> <span class="o">==</span> <span class="s1">&#39;4&#39;</span><span class="p">:</span>
            <span class="n">dataset_name</span> <span class="o">=</span> <span class="s1">&#39;Albergate&#39;</span>
            <span class="n">source_path</span> <span class="o">=</span> <span class="s1">&#39;requirements&#39;</span>
            <span class="n">languages</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;italian&#39;</span><span class="p">,</span> <span class="s1">&#39;java&#39;</span><span class="p">]</span>

            <span class="k">if</span> <span class="n">subset</span> <span class="o">==</span> <span class="s1">&#39;0&#39;</span><span class="p">:</span>
                <span class="n">modifier</span> <span class="o">=</span> <span class="s1">&#39;(RQ to Code)&#39;</span>
                <span class="n">target_path</span> <span class="o">=</span> <span class="s1">&#39;source_code&#39;</span>
                <span class="n">truth_path</span> <span class="o">=</span> <span class="s1">&#39;ground.txt&#39;</span>

            <span class="k">else</span><span class="p">:</span>
                <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Unrecognized corpus code: &quot;</span> <span class="o">+</span> <span class="n">corpus_code</span><span class="p">)</span>
                <span class="k">return</span>

        <span class="k">elif</span> <span class="n">dataset</span> <span class="o">==</span> <span class="s1">&#39;5&#39;</span><span class="p">:</span>
            <span class="n">dataset_name</span> <span class="o">=</span> <span class="s1">&#39;SMOS&#39;</span>
            <span class="n">source_path</span> <span class="o">=</span> <span class="s1">&#39;use_cases&#39;</span>
            <span class="n">languages</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;italian&#39;</span><span class="p">,</span> <span class="s1">&#39;java&#39;</span><span class="p">]</span>

            <span class="k">if</span> <span class="n">subset</span> <span class="o">==</span> <span class="s1">&#39;0&#39;</span><span class="p">:</span>
                <span class="n">modifier</span> <span class="o">=</span> <span class="s1">&#39;(UC to Code)&#39;</span>
                <span class="n">target_path</span> <span class="o">=</span> <span class="s1">&#39;source_code&#39;</span>
                <span class="n">truth_path</span> <span class="o">=</span> <span class="s1">&#39;ground.txt&#39;</span>

            <span class="k">else</span><span class="p">:</span>
                <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Unrecognized corpus code: &quot;</span> <span class="o">+</span> <span class="n">corpus_code</span><span class="p">)</span>
                <span class="k">return</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Unrecognized corpus code: &quot;</span> <span class="o">+</span> <span class="n">corpus_code</span><span class="p">)</span>
            <span class="k">return</span>

        <span class="n">file_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="vm">__file__</span><span class="p">))</span>
        <span class="n">corpus_root</span> <span class="o">=</span> <span class="n">file_path</span> <span class="o">+</span> <span class="s1">&#39;/../../data/raw/&#39;</span> <span class="o">+</span> <span class="n">dataset_name</span> <span class="o">+</span> <span class="s1">&#39;_semeru_format/&#39;</span>

        <span class="n">corpus_name</span> <span class="o">=</span> <span class="n">dataset_name</span> <span class="o">+</span> <span class="s1">&#39; (&#39;</span> <span class="o">+</span> <span class="n">corpus_code</span> <span class="o">+</span> <span class="s1">&#39;)&#39;</span>
        <span class="n">filetype_whitelist</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;java&#39;</span><span class="p">,</span> <span class="s1">&#39;txt&#39;</span><span class="p">,</span> <span class="s1">&#39;jsp&#39;</span><span class="p">,</span> <span class="s1">&#39;h&#39;</span><span class="p">,</span> <span class="s1">&#39;c&#39;</span><span class="p">]</span>

        <span class="n">corpus</span> <span class="o">=</span> <span class="n">Corpus</span><span class="p">(</span>
            <span class="n">corpus_name</span><span class="p">,</span>
            <span class="n">corpus_root</span><span class="p">,</span>
            <span class="n">source_path</span><span class="p">,</span>
            <span class="n">target_path</span><span class="p">,</span>
            <span class="n">truth_path</span><span class="p">,</span>
            <span class="n">execution_traces</span><span class="o">=</span><span class="n">execution_traces</span><span class="p">,</span>
            <span class="n">corpus_code</span><span class="o">=</span><span class="n">corpus_code</span><span class="p">,</span>
            <span class="n">languages</span><span class="o">=</span><span class="n">languages</span><span class="p">,</span>
            <span class="n">filetype_whitelist</span><span class="o">=</span><span class="n">filetype_whitelist</span>
        <span class="p">)</span>

        <span class="k">return</span> <span class="n">corpus</span>

    <span class="nd">@classmethod</span>
    <span class="k">def</span> <span class="nf">get_all_preset_corpus_codes</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
        <span class="n">codes</span> <span class="o">=</span> <span class="p">[</span>
            <span class="s1">&#39;0_0&#39;</span><span class="p">,</span> <span class="s1">&#39;0_1&#39;</span><span class="p">,</span> <span class="s1">&#39;0_2&#39;</span><span class="p">,</span>
            <span class="s1">&#39;1_0&#39;</span><span class="p">,</span> <span class="s1">&#39;1_1&#39;</span><span class="p">,</span> <span class="s1">&#39;1_2&#39;</span><span class="p">,</span>
            <span class="s1">&#39;2_0&#39;</span><span class="p">,</span>
            <span class="s1">&#39;3_0&#39;</span><span class="p">,</span>
            <span class="s1">&#39;4_0&#39;</span><span class="p">,</span>
            <span class="s1">&#39;5_0&#39;</span>
        <span class="p">]</span>
        <span class="k">return</span> <span class="n">codes</span>

    <span class="nd">@classmethod</span>
    <span class="k">def</span> <span class="nf">get_all_preset_corpora</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
        <span class="n">codes</span> <span class="o">=</span> <span class="n">Corpus</span><span class="o">.</span><span class="n">get_all_preset_corpus_codes</span><span class="p">()</span>

        <span class="k">return</span> <span class="p">[</span><span class="n">Corpus</span><span class="o">.</span><span class="n">get_preset_corpus</span><span class="p">(</span><span class="n">code</span><span class="p">)</span> <span class="k">for</span> <span class="n">code</span> <span class="ow">in</span> <span class="n">codes</span><span class="p">]</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># Exploring!</span>
<span class="n">dirpath</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">getcwd</span><span class="p">()</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Current directory is : &quot;</span> <span class="o">+</span> <span class="n">dirpath</span><span class="p">)</span>
<span class="n">foldername</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">basename</span><span class="p">(</span><span class="n">dirpath</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Directory name is : &quot;</span> <span class="o">+</span> <span class="n">foldername</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">

<div class="output_subarea output_stream output_stdout output_text">
<pre>Current directory is : /tf/main/nbs
Directory name is : nbs
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">data_folder</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">dirpath</span><span class="p">,</span> <span class="s2">&quot;test_data&quot;</span><span class="p">)</span>
<span class="n">data_folder</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">data_folder</span><span class="p">,</span> <span class="s2">&quot;LibEST_semeru_format/&quot;</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">language</span> <span class="o">=</span> <span class="s1">&#39;english&#39;</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">corpus</span> <span class="o">=</span> <span class="n">Corpus</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s1">&#39;libest&#39;</span><span class="p">,</span> <span class="n">corpus_root</span><span class="o">=</span><span class="n">data_folder</span><span class="p">,</span> 
                        <span class="n">source_path</span><span class="o">=</span><span class="s1">&#39;requirements&#39;</span><span class="p">,</span> <span class="n">target_path</span><span class="o">=</span><span class="s1">&#39;source_code&#39;</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">

<div class="output_subarea output_stream output_stdout output_text">
<pre>Finding artifacts in: /tf/main/nbs/test_data/LibEST_semeru_format/requirements
Getting artifacts by file from directory
Read 52 artifacts from directory

Finding artifacts in: /tf/main/nbs/test_data/LibEST_semeru_format/source_code
Getting artifacts by file from directory
Read 14 artifacts from directory

Found stop words file: english_stop_words.txt
Detected natural language: [english], generating stemmer
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="nb">len</span><span class="p">(</span><span class="n">corpus</span><span class="o">.</span><span class="n">get_sources</span><span class="p">())</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">



<div class="output_text output_subarea output_execute_result">
<pre>52</pre>
</div>

</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="n">f</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">getcwd</span><span class="p">(),</span> <span class="s1">&#39;test_data/config_corpus/&#39;</span> <span class="o">+</span> <span class="n">language</span> <span class="o">+</span> <span class="s1">&#39;_stop_words.txt&#39;</span><span class="p">))</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="nb">print</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">

<div class="output_subarea output_stream output_stdout output_text">
<pre>a
about
above
across
after
afterwards
again
against
all
almost
alone
along
already
also
although
always
am
among
amongst
amoungst
amount
an
and
another
any
anyhow
anyone
anything
anyway
anywhere
are
around
as
at
back
be
became
because
become
becomes
becoming
been
before
beforehand
behind
being
below
beside
besides
between
beyond
bill
both
bottom
but
by
call
can
cannot
cant
co
computer
con
could
couldnt
cry
de
describe
detail
do
done
down
due
during
each
eg
eight
either
eleven
else
elsewhere
empty
enough
etc
even
ever
every
everyone
everything
everywhere
except
few
fifteen
fify
fill
find
fire
first
five
for
former
formerly
forty
found
four
from
front
full
further
get
give
go
had
has
hasnt
have
he
hence
her
here
hereafter
hereby
herein
hereupon
hers
herself
him
himself
his
how
however
hundred
i
ie
if
in
inc
indeed
interest
into
is
it
its
itself
keep
last
latter
latterly
least
less
ltd
made
many
may
me
meanwhile
might
mill
mine
more
moreover
most
mostly
move
much
must
my
myself
name
namely
neither
never
nevertheless
next
nine
no
nobody
none
noone
nor
not
nothing
now
nowhere
of
off
often
on
once
one
only
onto
or
other
others
otherwise
our
ours
ourselves
out
over
own
part
per
perhaps
please
put
rather
re
same
see
seem
seemed
seeming
seems
serious
several
she
should
show
side
since
sincere
six
sixty
so
some
somehow
someone
something
sometime
sometimes
somewhere
still
such
system
take
ten
than
that
the
their
them
themselves
then
thence
there
thereafter
thereby
therefore
therein
thereupon
these
they
thick
thin
third
this
those
though
three
through
throughout
thru
thus
to
together
too
top
toward
towards
twelve
twenty
two
un
under
until
up
upon
us
very
via
was
we
well
were
what
whatever
when
whence
whenever
where
whereafter
whereas
whereby
wherein
whereupon
wherever
whether
which
while
whither
who
whoever
whole
whom
whose
why
will
with
within
without
would
yet
you
your
yours
yourself
yourselves
may
require
shall
recommended
optional
figure

</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">class</span> <span class="nc">TestCorpus</span><span class="p">(</span><span class="n">unittest</span><span class="o">.</span><span class="n">TestCase</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">setUp</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="n">dirpath</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">getcwd</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_data_folder</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">dirpath</span><span class="p">,</span> <span class="s2">&quot;test_data&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_data_folder</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">data_folder</span><span class="p">,</span> <span class="s2">&quot;LibEST_semeru_format/&quot;</span><span class="p">)</span>
        <span class="k">pass</span>
    <span class="k">def</span> <span class="nf">test_corpus_creation</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="n">corpus</span> <span class="o">=</span> <span class="n">Corpus</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">corpus_root</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_data_folder</span><span class="p">,</span> 
                        <span class="n">source_path</span><span class="o">=</span><span class="s1">&#39;requirements&#39;</span><span class="p">,</span> <span class="n">target_path</span><span class="o">=</span><span class="s1">&#39;source_code&#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">assertEqual</span><span class="p">(</span><span class="n">corpus</span><span class="o">.</span><span class="n">get_sources</span><span class="p">(),</span> <span class="mi">52</span><span class="p">)</span>
</pre></div>

    </div>
</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="o">!</span>python -m unittest
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">

<div class="output_subarea output_stream output_stdout output_text">
<pre>
----------------------------------------------------------------------
Ran 0 tests in 0.000s

OK
</pre>
</div>
</div>

</div>
</div>

</div>
<div class="cell border-box-sizing code_cell rendered">
<div class="input">

<div class="inner_cell">
    <div class="input_area">
<div class=" highlight hl-ipython3"><pre><span></span><span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">&#39;__main__&#39;</span><span class="p">:</span>
    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;hello&#39;</span><span class="p">)</span>
    <span class="n">unittest</span><span class="o">.</span><span class="n">main</span><span class="p">()</span>
</pre></div>

    </div>
</div>
</div>

<div class="output_wrapper">
<div class="output">

<div class="output_area">

<div class="output_subarea output_stream output_stderr output_text">
<pre>E</pre>
</div>
</div>

<div class="output_area">

<div class="output_subarea output_stream output_stdout output_text">
<pre>hello
</pre>
</div>
</div>

<div class="output_area">

<div class="output_subarea output_stream output_stderr output_text">
<pre>
======================================================================
ERROR: /root/ (unittest.loader._FailedTest)
----------------------------------------------------------------------
AttributeError: module &#39;__main__&#39; has no attribute &#39;/root/&#39;

----------------------------------------------------------------------
Ran 1 test in 0.001s

FAILED (errors=1)
</pre>
</div>
</div>

<div class="output_area">

<div class="output_subarea output_text output_error">
<pre>
An exception has occurred, use %tb to see the full traceback.

<span class="ansi-red-fg">SystemExit</span><span class="ansi-red-fg">:</span> True
</pre>
</div>
</div>

<div class="output_area">

<div class="output_subarea output_stream output_stderr output_text">
<pre>/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py:3334: UserWarning: To exit: use &#39;exit&#39;, &#39;quit&#39;, or Ctrl-D.
  warn(&#34;To exit: use &#39;exit&#39;, &#39;quit&#39;, or Ctrl-D.&#34;, stacklevel=1)
</pre>
</div>
</div>

</div>
</div>

</div>
</div>