Register
Login
Resources
Docs Blog Datasets Glossary Case Studies Tutorials & Webinars
Product
Data Engine LLMs Platform Enterprise
Pricing Explore
Connect to our Discord channel

Jenkinsfile 4.8 KB

You have to be logged in to leave a comment. Sign In
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
  1. /* groovylint-disable CompileStatic, DuplicateMapLiteral, DuplicateStringLiteral, LineLength, NestedBlockDepth */
  2. pipeline {
  3. agent any
  4. environment {
  5. JENKINS_USER_NAME = 'JenkinsRPPP'
  6. JENKINS_EMAIL = 'jenkins@rppp.com'
  7. GIT_REPO = 'dagshub.com/puneethp/RPPP'
  8. GIT_COMMIT_REV = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
  9. }
  10. stages {
  11. stage('Run inside Docker Image') {
  12. agent {
  13. dockerfile {
  14. additionalBuildArgs '--tag rppp:$BRANCH_NAME'
  15. args '-v $WORKSPACE:/project -w /project -v /extras:/extras -e PYTHONPATH=/project'
  16. }
  17. }
  18. stages {
  19. stage('Run Unit Test') {
  20. steps {
  21. sh 'pytest -vvrxXs'
  22. }
  23. }
  24. stage('Run Linting') {
  25. steps {
  26. sh '''
  27. flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
  28. flake8 . --count --max-complexity=10 --max-line-length=127 --statistics
  29. black . --check --diff
  30. '''
  31. }
  32. }
  33. stage('Setup DVC Creds') {
  34. steps {
  35. withCredentials(
  36. [
  37. usernamePassword(
  38. credentialsId: 'PASSWORD',
  39. passwordVariable: 'PASSWORD',
  40. usernameVariable: 'USER_NAME'),
  41. ]
  42. ) {
  43. sh '''
  44. dvc remote modify origin --local auth basic
  45. dvc remote modify origin --local user $USER_NAME
  46. dvc remote modify origin --local password $PASSWORD
  47. dvc status -r origin
  48. '''
  49. }
  50. }
  51. }
  52. stage('Sync DVC Remotes') {
  53. steps {
  54. sh '''
  55. dvc status
  56. dvc status -r jenkins_local
  57. dvc status -r origin
  58. dvc pull -r jenkins_local || echo 'Some files are missing in local cache!'
  59. dvc pull -r origin
  60. dvc push -r jenkins_local
  61. '''
  62. }
  63. }
  64. stage('Update DVC Pipeline') {
  65. when { changeRequest() }
  66. steps {
  67. sh '''
  68. dvc repro --dry -mP
  69. dvc repro -mP
  70. git branch -a
  71. cat dvc.lock
  72. dvc push -r jenkins_local
  73. dvc push -r origin
  74. '''
  75. sh 'dvc metrics diff --show-md --precision 2 $CHANGE_TARGET'
  76. }
  77. }
  78. stage('Commit back results') {
  79. when { changeRequest() }
  80. steps {
  81. withCredentials(
  82. [
  83. usernamePassword(
  84. credentialsId: 'GIT_PAT',
  85. passwordVariable: 'GIT_PAT',
  86. usernameVariable: 'GIT_USER_NAME'),
  87. ]
  88. ) {
  89. sh '''
  90. git branch -a
  91. git status
  92. if ! git diff --exit-code dvc.lock; then
  93. git add .
  94. git status
  95. git config --local user.email $JENKINS_EMAIL
  96. git config --local user.name $JENKINS_USER_NAME
  97. git commit -m "$GIT_COMMIT_REV: Update dvc.lock and metrics"
  98. git push https://$GIT_USER_NAME:$GIT_PAT@$GIT_REPO HEAD:$CHANGE_BRANCH
  99. else
  100. echo 'Nothing to Commit!'
  101. fi
  102. '''
  103. }
  104. }
  105. }
  106. }
  107. post {
  108. always {
  109. sh '''
  110. rm -r .dvc/config.local || echo 'Config not found! Nothing to worry about!'
  111. '''
  112. }
  113. }
  114. }
  115. }
  116. }
Tip!

Press p or to see the previous file or, n or to see the next file

Comments

Loading...