Register
Login
Resources
Docs Blog Datasets Glossary Case Studies Tutorials & Webinars
Product
Data Engine LLMs Platform Enterprise
Pricing Explore
Connect to our Discord channel

dvc.yaml 3.6 KB

You have to be logged in to leave a comment. Sign In
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
  1. stages:
  2. extract-devanbu-small-corpus:
  3. cmd: unzip devanbu-small-corpus.zip -d devanbu-small-corpus
  4. deps:
  5. - devanbu-small-corpus.zip
  6. outs:
  7. - devanbu-small-corpus
  8. training:
  9. cmd: jq "$(<to-fuzzy-sharp-config.jq)" config.json > params.json && virtualenv
  10. .venv && source .venv/bin/activate && LANGMODELS_COMMIT_HASH=$(<langmodels-commit-hash.txt)
  11. && pip install -r https://raw.githubusercontent.com/giganticode/langmodels/$LANGMODELS_COMMIT_HASH/requirements.txt
  12. && pip install git+https://github.com/giganticode/langmodels@$LANGMODELS_COMMIT_HASH
  13. && langmodels train --config=$(pwd)/config.json --output-path=$(pwd)/output
  14. --patch="corpus.path=$(pwd)/devanbu-small-corpus" --rewrite-output --save-every-epoch; rm -rf .venv
  15. deps:
  16. - config.json
  17. - devanbu-small-corpus
  18. - langmodels-commit-hash.txt
  19. - to-fuzzy-sharp-config.jq
  20. outs:
  21. - output/epoch_0.pth
  22. - output/epoch_2.pth
  23. - output/epoch_5.pth
  24. - output/epoch_10.pth
  25. - output/epoch_19.pth
  26. - output/epoch_79.pth
  27. - output/epoch_159.pth
  28. - output/config
  29. - output/training_metrics.csv:
  30. cache: false
  31. - output/vocab
  32. evaluation-0:
  33. cmd: virtualenv .venv0 && source .venv0/bin/activate && LANGMODELS_V2_COMMIT_HASH=$(<langmodels-v2-commit-hash.txt)
  34. && pip install -r https://raw.githubusercontent.com/giganticode/langmodels/$LANGMODELS_V2_COMMIT_HASH/requirements.txt
  35. && pip install git+https://github.com/giganticode/langmodels@$LANGMODELS_V2_COMMIT_HASH
  36. && langmodels evaluate $(pwd)/output -p $(pwd)/devanbu-small-corpus/test -o
  37. $(pwd)/evaluation-test-0 -b 64 -e 0; rm -rf .venv0
  38. deps:
  39. - output/epoch_0.pth
  40. - devanbu-small-corpus
  41. - langmodels-v2-commit-hash.txt
  42. outs:
  43. - evaluation-test-0
  44. evaluation-19:
  45. cmd: virtualenv .venv19 && source .venv19/bin/activate && LANGMODELS_V2_COMMIT_HASH=$(<langmodels-v2-commit-hash.txt)
  46. && pip install -r https://raw.githubusercontent.com/giganticode/langmodels/$LANGMODELS_V2_COMMIT_HASH/requirements.txt
  47. && pip install git+https://github.com/giganticode/langmodels@$LANGMODELS_V2_COMMIT_HASH
  48. && langmodels evaluate $(pwd)/output -p $(pwd)/devanbu-small-corpus/test -o
  49. $(pwd)/evaluation-test-19 -b 64 -e 19; rm -rf .venv19
  50. deps:
  51. - output/epoch_19.pth
  52. - devanbu-small-corpus
  53. - langmodels-v2-commit-hash.txt
  54. outs:
  55. - evaluation-test-19
  56. evaluation-79:
  57. cmd: virtualenv .venv79 && source .venv79/bin/activate && LANGMODELS_V2_COMMIT_HASH=$(<langmodels-v2-commit-hash.txt)
  58. && pip install -r https://raw.githubusercontent.com/giganticode/langmodels/$LANGMODELS_V2_COMMIT_HASH/requirements.txt
  59. && pip install git+https://github.com/giganticode/langmodels@$LANGMODELS_V2_COMMIT_HASH
  60. && langmodels evaluate $(pwd)/output -p $(pwd)/devanbu-small-corpus/test -o
  61. $(pwd)/evaluation-test-79 -b 64 -e 79; rm -rf .venv79
  62. deps:
  63. - output/epoch_79.pth
  64. - devanbu-small-corpus
  65. - langmodels-v2-commit-hash.txt
  66. outs:
  67. - evaluation-test-79
  68. evaluation-159:
  69. cmd: virtualenv .venv159 && source .venv159/bin/activate && LANGMODELS_V2_COMMIT_HASH=$(<langmodels-v2-commit-hash.txt)
  70. && pip install -r https://raw.githubusercontent.com/giganticode/langmodels/$LANGMODELS_V2_COMMIT_HASH/requirements.txt
  71. && pip install git+https://github.com/giganticode/langmodels@$LANGMODELS_V2_COMMIT_HASH
  72. && langmodels evaluate $(pwd)/output -p $(pwd)/devanbu-small-corpus/test -o
  73. $(pwd)/evaluation-test-159 -b 64 -e 159; rm -rf .venv159
  74. deps:
  75. - output/epoch_159.pth
  76. - devanbu-small-corpus
  77. - langmodels-v2-commit-hash.txt
  78. outs:
  79. - evaluation-test-159
Tip!

Press p or to see the previous file or, n or to see the next file

Comments

Loading...