martin-fabbri
/
tpu-workflow
mirror of https://github.com/martin-fabbri/tpu-workflow.git


  
1

	
2

	
3

	
4

	
5

	
6

	
7

	
8

	
9

	
10

	
11

	
12

	
13

	
14

	
15

	
16

	
17

	
18

	
19

	
20

	
21

	
22

	
23

	
24

	
25

	
26

	
27

	
28

	
29

	
30

	
31

	
32

	
33

	
34

	
35

	
36

	
37

	
38

	
39

	
40

	
41

	
42

	
43

	
44

	
45

	
46

	
47

	
48

	
49

	
50

	
51

	
52

	
53

	
54

	
55

	
56

	
57

	
58

	
59

	
60

	
61

	
62

	
63

	
64

	
65

	
66

	
67

	
68

	
69

	
70

	
71

	
72

	
73

	
74

	
75

	
76

	
77

	
78

	
79

	
80

	
81

	
82

	
83

	
84

	
85

	
86

	
87

	
88

	
89

	
90

	
91

	
92

	
93

	
94

	
95

	
96

	
97

	
98

	
99

	
100

	
101

	
102

	
103

	
104

	
105

	
106

	
107

	
108

	
109

	
110

	
111

	
112

	
113

	
114

	
115

	
            schema: '2.0'
stages:
  download_file:
    cmd: gsutil cp gs://kaggle-data-tpu/test/test.txt test.txt
    deps:
    - path: gs://kaggle-data-tpu/test/test.txt
      md5: 74e24b46e2336dd063622ee49c96514d
      size: 22
    outs:
    - path: test.txt
      md5: 74e24b46e2336dd063622ee49c96514d
      size: 22
  download_file_2:
    cmd: gsutil cp gs://kaggle-data-tpu/test/test_2.txt test_2.txt
    deps:
    - path: remote://gcs_test/test_2.txt
      md5: 74e24b46e2336dd063622ee49c96514d
      size: 22
    outs:
    - path: test_2.txt
      md5: 74e24b46e2336dd063622ee49c96514d
      size: 22
  test_read_blob_gcs:
    cmd: python src/test.py
    deps:
    - path: remote://gcs_test/test_2.txt
      md5: 74e24b46e2336dd063622ee49c96514d
      size: 22
  test_list_objects_gcs:
    cmd: python src/test_list_blobs.py
    deps:
    - path: remote://gcs_test
      md5: 4c8d119295d08db866947dea99bd3272.dir
      size: 44
      nfiles: 2
  split:
    cmd: python3 src/pipeline/split.py --gcs-path gs://kaggle-data-tpu/images --train-split-path
      data/interim/train_split.json --val-split-path data/interim/val_split.json --images-count-path
      data/interim/images_count.json --validation-split 0.19 --image-size 331
    deps:
    - path: remote://gcs_images
      md5: 61d4627ec881871e7683b161b726e3ee.dir
      size: 1067709259
      nfiles: 3735
    - path: src/pipeline/split.py
      md5: 7e2d9421f9cac18bd18b4d7a55ab5f0d
      size: 3159
    outs:
    - path: data/interim/images_count.json
      md5: cb5f6e38428e031affe368e5e4b64351
      size: 59
    - path: data/interim/train_split.json
      md5: 990315c7d90f4d0d640be37145518a78
      size: 962
    - path: data/interim/val_split.json
      md5: 3a634df5d52902bd59ebd43b98844c22
      size: 222
  train:
    cmd: python src/pipeline/train.py --lr-start 1e-05 --classes-path data/domain/classes.json
      --image-size 331 --epochs 12 --batch-size 16 --train-split-path data/interim/train_split.json
      --val-split-path data/interim/val_split.json --images-count-path data/interim/images_count.json
      --saved-model-path models/xception_model.h5 --train-plot-path plots/train_accuracy_plot.json
      --checkpoint-dir checkpoints --checkpoint-prefix cp.ckpt
    deps:
    - path: data/domain/classes.json
      md5: 536eef1578b80119c679abe0e2fe63c7
      size: 77
    - path: data/interim/images_count.json
      md5: cb5f6e38428e031affe368e5e4b64351
      size: 59
    - path: data/interim/train_split.json
      md5: 990315c7d90f4d0d640be37145518a78
      size: 962
    - path: data/interim/val_split.json
      md5: 3a634df5d52902bd59ebd43b98844c22
      size: 222
    - path: src/pipeline/train.py
      md5: c8004d60d4b563d7f6e91484ce47ef59
      size: 8653
    outs:
    - path: checkpoints/checkpoint
      md5: 04d5dddb0afda887c287a6ba51725e32
      size: 71
    - path: checkpoints/cp.ckpt.data-00000-of-00001
      md5: d2c83af5170baada8a2d7180c1ebcc1e
      size: 250102229
    - path: checkpoints/cp.ckpt.index
      md5: ad47ca60d78377d23dd5d90931c1b647
      size: 40587
    - path: models/xception_model.h5
      md5: f1a02bff5e323e4b1e7e59cf5e50950b
      size: 250635496
    - path: plots/train_accuracy_plot.json
      md5: 302c3d789dddeebe589f774989a8fafc
      size: 1571
  setup:
    cmd: python3 src/pipeline/setup.py
  evaluate:
    cmd: python src/pipeline/evaluate.py --image-size 331 --batch-size 16 --val-split-path
      data/interim/val_split.json --saved-model-path models/xception_model.h5 --evaluate-metrics-path
      metrics/evaluate_metrics.json
    deps:
    - path: data/interim/val_split.json
      md5: 3a634df5d52902bd59ebd43b98844c22
      size: 222
    - path: models/xception_model.h5
      md5: f1a02bff5e323e4b1e7e59cf5e50950b
      size: 250635496
    - path: src/pipeline/evaluate.py
      md5: 8cad7e36c9b7a4e92d9c73d462406d9c
      size: 2263
    outs:
    - path: metrics/evaluate_metrics.json
      md5: a70c54a6c2ce6688688b03143f076c12
      size: 64