Browse Source

Create pipeline

deanp07 1 year ago
commit
c26b32fa4e
9 changed files with 50 additions and 0 deletions
  1. 9
    0
      .dvc/.gitignore
  2. 2
    0
      .dvc/config
  3. 18
    0
      Stage.dvc
  4. 1
    0
      data/.gitignore
  5. 7
    0
      data/raw.csv.dvc
  6. 2
    0
      metric.csv
  7. 1
    0
      models/.gitignore
  8. 1
    0
      params.yml
  9. 9
    0
      run_something.py

+ 9
- 0
.dvc/.gitignore

@@ -0,0 +1,9 @@
+/config.local
+/updater
+/state-journal
+/state-wal
+/state
+/lock
+/tmp
+/updater.lock
+/cache

+ 2
- 0
.dvc/config

@@ -0,0 +1,2 @@
+['remote "ex-remote"']
+    url = s3://who-moved-my-data

+ 18
- 0
Stage.dvc

@@ -0,0 +1,18 @@
+md5: 6cbf6611b793993867fa600cf935d2d9
+cmd: python3 run_something.py
+deps:
+- md5: c8263e8422925b0872ee1fb7c953742a
+  path: data/raw.csv
+- md5: 1fb0413fe5fcef1b75edc507cda73c3b
+  path: run_something.py
+outs:
+- md5: c915a492d1f4b718a6385564a235b3e7
+  path: models/model.pkl
+  cache: true
+  metric: false
+  persist: false
+- md5: 360838fe749de7973ad06a2bb625293d
+  path: metric.csv
+  cache: false
+  metric: true
+  persist: false

+ 1
- 0
data/.gitignore

@@ -0,0 +1 @@
+/raw.csv

+ 7
- 0
data/raw.csv.dvc

@@ -0,0 +1,7 @@
+md5: df91e183b1037a1e2840bd887a4fae7c
+outs:
+- md5: c8263e8422925b0872ee1fb7c953742a
+  path: raw.csv
+  cache: true
+  metric: false
+  persist: false

+ 2
- 0
metric.csv

@@ -0,0 +1,2 @@
+Name,Value,Timestamp,Step
+"Over",9000,1582363274245,1

+ 1
- 0
models/.gitignore

@@ -0,0 +1 @@
+/model.pkl

+ 1
- 0
params.yml

@@ -0,0 +1 @@
+{}

+ 9
- 0
run_something.py

@@ -0,0 +1,9 @@
+import pickle
+import dagshub
+
+text = "hello world"
+print(text)
+with open("./models/model.pkl", 'wb') as f:
+	pickle.dump(text, f)
+with dagshub.dagshub_logger(metrics_path = 'metric.csv') as logger:
+	logger.log_metrics({'Over': 9000})