Register
Login
Resources
Docs Blog Datasets Glossary Case Studies Tutorials & Webinars
Product
Data Engine LLMs Platform Enterprise
Pricing Explore
Connect to our Discord channel

Jenkinsfile 5.0 KB

You have to be logged in to leave a comment. Sign In
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
  1. /* groovylint-disable CompileStatic, DuplicateMapLiteral, DuplicateStringLiteral, LineLength, NestedBlockDepth */
  2. pipeline {
  3. agent any
  4. environment {
  5. GIT_COMMIT_REV = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
  6. JENKINS_USER_NAME = 'JenkinsRPPP'
  7. JENKINS_EMAIL = 'jenkins@rpp.com'
  8. // GIT_COMMIT_MASTER = sh(returnStdout: true, script: 'git rev-parse --short master').trim()
  9. }
  10. stages {
  11. stage('Run inside Docker Image') {
  12. agent {
  13. dockerfile {
  14. args "-v ${env.WORKSPACE}:/project -w /project -v /extras:/extras -e PYTHONPATH=/project"
  15. }
  16. }
  17. stages {
  18. stage('Run Unit Test') {
  19. steps {
  20. sh 'pytest -vvrxXs'
  21. }
  22. }
  23. stage('Run Linting') {
  24. steps {
  25. sh '''
  26. flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
  27. flake8 . --count --max-complexity=10 --max-line-length=127 --statistics
  28. black . --check --diff
  29. '''
  30. }
  31. }
  32. stage('Setup DVC Creds') {
  33. steps {
  34. withCredentials(
  35. [
  36. usernamePassword(
  37. credentialsId: 'PASSWORD',
  38. passwordVariable: 'PASSWORD',
  39. usernameVariable: 'USER_NAME'),
  40. ]
  41. ) {
  42. sh '''
  43. dvc remote modify origin --local auth basic
  44. dvc remote modify origin --local user $USER_NAME
  45. dvc remote modify origin --local password $PASSWORD
  46. dvc status -r origin
  47. '''
  48. }
  49. }
  50. }
  51. stage('Sync DVC Remotes') {
  52. steps {
  53. sh '''
  54. dvc status
  55. dvc status -r jenkins_local
  56. dvc status -r origin
  57. dvc pull -r jenkins_local || echo 'Some files are missing in local cache!'
  58. dvc pull -r origin
  59. dvc push -r jenkins_local
  60. '''
  61. }
  62. }
  63. stage('Update DVC Pipeline') {
  64. when { changeRequest() }
  65. steps {
  66. sh '''
  67. date >> a.txt
  68. dvc repro --dry -mP
  69. dvc repro -mP
  70. git branch -a
  71. cat dvc.lock
  72. dvc push -r jenkins_local
  73. dvc push -r origin
  74. # rm -r /extras/RPPP/repo/$CHANGE_BRANCH || echo 'All clean'
  75. # mkdir -p /extras/RPPP/repo/$CHANGE_BRANCH
  76. # cp -Rf . /extras/RPPP/repo/$CHANGE_BRANCH
  77. '''
  78. sh 'dvc metrics diff --show-md --precision 2 $CHANGE_TARGET'
  79. }
  80. }
  81. stage('Commit back results') {
  82. when { changeRequest() }
  83. steps {
  84. withCredentials(
  85. [
  86. usernamePassword(
  87. credentialsId: 'GIT_PAT',
  88. passwordVariable: 'GIT_PAT',
  89. usernameVariable: 'GIT_USER_NAME'),
  90. ]
  91. ) {
  92. sh '''
  93. env
  94. git branch -a
  95. git status
  96. if ! git diff --exit-code; then
  97. git add .
  98. git status
  99. git config --global user.email $JENKINS_EMAIL
  100. git config --global user.name $JENKINS_USER_NAME
  101. git commit -m '$GIT_COMMIT_REV: Update dvc.lock and metrics'
  102. git push https://$GIT_USER_NAME:$GIT_PAT@github.com/PuneethaPai/RPPP HEAD:$CHANGE_BRANCH
  103. cat ~/.git-credentials || echo 'Nothing Saved/cached'
  104. else
  105. echo 'Nothing to Commit!'
  106. fi
  107. '''
  108. }
  109. }
  110. }
  111. }
  112. }
  113. }
  114. }
Tip!

Press p or to see the previous file or, n or to see the next file

Comments

Loading...