1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
stages:
extract-devanbu-small-corpus:
cmd: unzip devanbu-small-corpus.zip -d devanbu-small-corpus
deps:
- devanbu-small-corpus.zip
outs:
- devanbu-small-corpus
training:
cmd: jq "$(<to-fuzzy-sharp-config.jq)" config.json > params.json && virtualenv
.venv && source .venv/bin/activate && LANGMODELS_COMMIT_HASH=$(<langmodels-commit-hash.txt)
&& pip install -r https://raw.githubusercontent.com/giganticode/langmodels/$LANGMODELS_COMMIT_HASH/requirements.txt
&& pip install git+https://github.com/giganticode/langmodels@$LANGMODELS_COMMIT_HASH
&& langmodels train --config=$(pwd)/config.json --output-path=$(pwd)/output
--patch="corpus.path=$(pwd)/devanbu-small-corpus" --rewrite-output --save-every-epoch; rm -rf .venv
deps:
- config.json
- devanbu-small-corpus
- langmodels-commit-hash.txt
- to-fuzzy-sharp-config.jq
outs:
- output/epoch_0.pth
- output/epoch_2.pth
- output/epoch_5.pth
- output/epoch_10.pth
- output/epoch_19.pth
- output/epoch_79.pth
- output/epoch_159.pth
- output/config
- output/training_metrics.csv:
cache: false
- output/vocab
evaluation-0:
cmd: virtualenv .venv0 && source .venv0/bin/activate && LANGMODELS_V2_COMMIT_HASH=$(<langmodels-v2-commit-hash.txt)
&& pip install -r https://raw.githubusercontent.com/giganticode/langmodels/$LANGMODELS_V2_COMMIT_HASH/requirements.txt
&& pip install git+https://github.com/giganticode/langmodels@$LANGMODELS_V2_COMMIT_HASH
&& langmodels evaluate $(pwd)/output -p $(pwd)/devanbu-small-corpus/test -o
$(pwd)/evaluation-test-0 -b 64 -e 0; rm -rf .venv0
deps:
- output/epoch_0.pth
- devanbu-small-corpus
- langmodels-v2-commit-hash.txt
outs:
- evaluation-test-0
evaluation-19:
cmd: virtualenv .venv19 && source .venv19/bin/activate && LANGMODELS_V2_COMMIT_HASH=$(<langmodels-v2-commit-hash.txt)
&& pip install -r https://raw.githubusercontent.com/giganticode/langmodels/$LANGMODELS_V2_COMMIT_HASH/requirements.txt
&& pip install git+https://github.com/giganticode/langmodels@$LANGMODELS_V2_COMMIT_HASH
&& langmodels evaluate $(pwd)/output -p $(pwd)/devanbu-small-corpus/test -o
$(pwd)/evaluation-test-19 -b 64 -e 19; rm -rf .venv19
deps:
- output/epoch_19.pth
- devanbu-small-corpus
- langmodels-v2-commit-hash.txt
outs:
- evaluation-test-19
evaluation-79:
cmd: virtualenv .venv79 && source .venv79/bin/activate && LANGMODELS_V2_COMMIT_HASH=$(<langmodels-v2-commit-hash.txt)
&& pip install -r https://raw.githubusercontent.com/giganticode/langmodels/$LANGMODELS_V2_COMMIT_HASH/requirements.txt
&& pip install git+https://github.com/giganticode/langmodels@$LANGMODELS_V2_COMMIT_HASH
&& langmodels evaluate $(pwd)/output -p $(pwd)/devanbu-small-corpus/test -o
$(pwd)/evaluation-test-79 -b 64 -e 79; rm -rf .venv79
deps:
- output/epoch_79.pth
- devanbu-small-corpus
- langmodels-v2-commit-hash.txt
outs:
- evaluation-test-79
evaluation-159:
cmd: virtualenv .venv159 && source .venv159/bin/activate && LANGMODELS_V2_COMMIT_HASH=$(<langmodels-v2-commit-hash.txt)
&& pip install -r https://raw.githubusercontent.com/giganticode/langmodels/$LANGMODELS_V2_COMMIT_HASH/requirements.txt
&& pip install git+https://github.com/giganticode/langmodels@$LANGMODELS_V2_COMMIT_HASH
&& langmodels evaluate $(pwd)/output -p $(pwd)/devanbu-small-corpus/test -o
$(pwd)/evaluation-test-159 -b 64 -e 159; rm -rf .venv159
deps:
- output/epoch_159.pth
- devanbu-small-corpus
- langmodels-v2-commit-hash.txt
outs:
- evaluation-test-159
Tip!
Press p or to see the previous file or,
n or to see the next file