Browse Source

add dvc data

taomiao 1 month ago
parent
commit
257fa34994

+ 22 - 0
TensorFlowMnist/README.md

@@ -0,0 +1,22 @@
+# TensorFlowMnist
+
+基于TensorFlow 1.0.1实现的[MNIST手写数字识别](http://yann.lecun.com/exdb/mnist/)
+
+## 数据集
+所有的数据集都在data目录下:
+- train-images-idx3-ubyte.gz:  training set images (9912422 bytes) 
+- train-labels-idx1-ubyte.gz:  training set labels (28881 bytes) 
+- t10k-images-idx3-ubyte.gz:   test set images (1648877 bytes) 
+- t10k-labels-idx1-ubyte.gz:   test set labels (4542 bytes)
+
+## 网络模型的创建与训练
+```train_model.py```: 创建并训练网络模型
+
+## 网络模型的测试
+```test_model.py```: 测试训练好后的网络模型
+
+## 读取数据集
+``` input_data.py```和```mnist_loader.py```
+
+## 小工具:保存MNIST中的每一张图像
+```extract_mnist.py```

BIN
TensorFlowMnist/__pycache__/input_data.cpython-37.pyc


BIN
TensorFlowMnist/__pycache__/mnist_loader.cpython-37.pyc


+ 4 - 0
TensorFlowMnist/data/.gitignore

@@ -0,0 +1,4 @@
+/t10k-images-idx3-ubyte.gz
+/t10k-labels-idx1-ubyte.gz
+/train-images-idx3-ubyte.gz
+/train-labels-idx1-ubyte.gz

+ 7 - 0
TensorFlowMnist/data/t10k-images-idx3-ubyte.gz.dvc

@@ -0,0 +1,7 @@
+md5: 40fb70bdd2d803504f6d3260da98ccb5
+outs:
+- md5: 9fb629c4189551a2d022fa330f9573f3
+  path: t10k-images-idx3-ubyte.gz
+  cache: true
+  metric: false
+  persist: false

+ 7 - 0
TensorFlowMnist/data/t10k-labels-idx1-ubyte.gz.dvc

@@ -0,0 +1,7 @@
+md5: cfbfffd8a0102839abe2440e608ed464
+outs:
+- md5: ec29112dd5afa0611ce80d1b7f02629c
+  path: t10k-labels-idx1-ubyte.gz
+  cache: true
+  metric: false
+  persist: false

+ 7 - 0
TensorFlowMnist/data/train-images-idx3-ubyte.gz.dvc

@@ -0,0 +1,7 @@
+md5: 8306c4f1b4ef661bc48085d80b7756f9
+outs:
+- md5: f68b3c2dcbeaaa9fbdd348bbdeb94873
+  path: train-images-idx3-ubyte.gz
+  cache: true
+  metric: false
+  persist: false

+ 7 - 0
TensorFlowMnist/data/train-labels-idx1-ubyte.gz.dvc

@@ -0,0 +1,7 @@
+md5: b374b3fb3102f2704c570952db8249d2
+outs:
+- md5: d53e105ee54ea40749a09fcbcd1e9432
+  path: train-labels-idx1-ubyte.gz
+  cache: true
+  metric: false
+  persist: false

+ 51 - 0
TensorFlowMnist/extract_mnist.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+#coding: utf-8
+
+"""
+MNIST数据集的原始格式是gz格式的,该脚本的功能是将该数据集中的每一张图片转为.png格式,并保存到指定目录。
+"""
+
+import gzip
+import os
+import numpy
+import PIL.Image as Image
+
+
+def _read32(bytestream):
+    dt = numpy.dtype(numpy.uint32).newbyteorder('>')
+    return numpy.frombuffer(bytestream.read(4), dtype=dt)[0]
+
+
+def extract_and_save_mnist(file_path,save_dir):
+  """
+    file_path:mnist数据集的路径,比如"./data/t10k-images-idx3-ubyte.gz"
+    save_dir:要保存至的目标目录,比如"/home/u2/data"
+  """
+  f=open(file_path, 'rb')
+  print('Extracting', f.name)
+  
+  with gzip.GzipFile(fileobj=f) as bytestream:
+    magic = _read32(bytestream)
+    if magic != 2051:
+      raise ValueError('Invalid magic number %d in MNIST image file: %s' %
+                       (magic, f.name))
+    num_images = _read32(bytestream)
+    rows = _read32(bytestream)
+    cols = _read32(bytestream)
+    buf = bytestream.read(rows * cols * num_images)
+    data = numpy.frombuffer(buf, dtype=numpy.uint8)
+    data = data.reshape(num_images, rows, cols)
+    
+    
+    #依次保存图片
+    for index,d in enumerate(data):
+        imges = Image.fromarray(d).convert('L')
+        abs_path = os.path.join(os.path.abspath(save_dir), str(index)+".png")
+        imges.save(abs_path,'png')
+    
+
+def main():
+    TEST_IMAGES_PATH = 'data/t10k-images-idx3-ubyte.gz'
+    extract_and_save_mnist(TEST_IMAGES_PATH,"./img")
+    
+# main()    

+ 111 - 0
TensorFlowMnist/input_data.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+#coding: utf-8
+
+import numpy as np
+import mnist_loader
+import collections
+
+
+Datasets = collections.namedtuple('Datasets', ['train',  'test'])
+
+class DataSet(object):
+ 
+  def __init__(self,
+               images,
+               labels):
+
+    self._num_examples = images.shape[0]
+    self._images = images
+    self._labels = labels
+    self._epochs_completed = 0
+    self._index_in_epoch = 0
+ 
+  @property
+  def images(self):
+    return self._images
+ 
+  @property
+  def labels(self):
+    return self._labels
+ 
+  @property
+  def num_examples(self):
+    return self._num_examples
+ 
+  @property
+  def epochs_completed(self):
+    return self._epochs_completed
+  
+
+  def mini_batches(self,mini_batch_size):
+    """
+      return: list of tuple(x,y)
+    """
+    # Shuffle the data
+    perm = np.arange(self._num_examples)
+    np.random.shuffle(perm)
+    self._images = self._images[perm]
+    self._labels = self._labels[perm]
+    
+    n = self.images.shape[0]
+    
+    mini_batches = [(self._images[k:k+mini_batch_size],self._labels[k:k+mini_batch_size])
+                    for k in range(0, n, mini_batch_size)]
+    
+    if len(mini_batches[-1]) != mini_batch_size:
+        return mini_batches[:-1]
+    else:
+        return mini_batches
+
+
+  def _next_batch(self, batch_size, fake_data=False):
+    """Return the next `batch_size` examples from this data set."""
+    
+    start = self._index_in_epoch
+    self._index_in_epoch += batch_size
+    if self._index_in_epoch > self._num_examples:
+        # Finished epoch
+        self._epochs_completed += 1
+        # Shuffle the data
+        perm = np.arange(self._num_examples)
+        np.random.shuffle(perm)
+        self._images = self._images[perm]
+        self._labels = self._labels[perm]
+        # Start next epoch
+        start = 0
+        self._index_in_epoch = batch_size
+        assert batch_size <= self._num_examples
+    end = self._index_in_epoch
+    return self._images[start:end], self._labels[start:end]
+
+
+def read_data_sets():
+    """
+    Function:读取训练集(TrainSet)和测试集(TestSet)。
+    
+    Notes
+    ----------
+        TrainSet: include imgs_train and labels_train.
+        TestSet:  include imgs_test and  labels_test.
+        
+        the shape of imgs_train and imgs_test are:(batch_size,height,width). namely (n, 28L, 28L)
+        the shape of labels_train and labels_test are:(batch_size,num_classes). namely (n, 10L)
+    
+    """
+    imgs_train, imgs_test, labels_train, labels_test  =  mnist_loader.read_data_sets()
+    train = DataSet(imgs_train, labels_train)
+    test = DataSet(imgs_test, labels_test)
+    return Datasets(train=train, test=test)
+
+def _test():
+    dataset = read_data_sets()
+    
+    print("dataset.train.images.shape:",dataset.train.images.shape)
+    print("dataset.train.labels.shape:",dataset.train.labels.shape)
+    print("dataset.test.images.shape:",dataset.test.images.shape)
+    print("dataset.test.labels.shape:",dataset.test.labels.shape)
+    
+    print(dataset.test.images[0])
+    print(dataset.test.labels[0])
+
+# _test()

+ 13 - 0
TensorFlowMnist/mnist.dvc

@@ -0,0 +1,13 @@
+md5: 664b382549142fe20d6acd94bb29271a
+cmd: python train_model.py
+deps:
+- md5: 0581019d3a96183ddaca3398b7a21ad1
+  path: train_model.py
+- md5: 46d6a469c3b9b7cb1183eeb12b091a86.dir
+  path: data
+outs:
+- md5: b7040d321ea280a7c23826ba70e5bdeb
+  path: output/mnist-tf1.0.1.pb
+  cache: true
+  metric: false
+  persist: false

+ 108 - 0
TensorFlowMnist/mnist_loader.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+#coding: utf-8
+
+import gzip
+import numpy
+
+def _read32(bytestream):
+  dt = numpy.dtype(numpy.uint32).newbyteorder('>')
+  return numpy.frombuffer(bytestream.read(4), dtype=dt)[0]
+
+
+def _extract_images(f):
+  """Extract the images into a 4D uint8 numpy array [index, y, x, depth].
+
+  Args:
+    f: A file object that can be passed into a gzip reader.
+
+  Returns:
+    data: A 4D unit8 numpy array [index, y, x].
+
+  Raises:
+    ValueError: If the bytestream does not start with 2051.
+
+  """
+  print('Extracting', f.name)
+  with gzip.GzipFile(fileobj=f) as bytestream:
+    magic = _read32(bytestream)
+    if magic != 2051:
+      raise ValueError('Invalid magic number %d in MNIST image file: %s' %
+                       (magic, f.name))
+    num_images = _read32(bytestream)
+    rows = _read32(bytestream)
+    cols = _read32(bytestream)
+    buf = bytestream.read(rows * cols * num_images)
+    data = numpy.frombuffer(buf, dtype=numpy.uint8)
+    data = data.reshape(num_images, rows, cols)
+    data = numpy.multiply(data, 1.0 / 255.0)
+    return data
+
+
+def _dense_to_one_hot(labels_dense, num_classes):
+  """Convert class labels from scalars to one-hot vectors."""
+  num_labels = labels_dense.shape[0]
+  index_offset = numpy.arange(num_labels) * num_classes
+  labels_one_hot = numpy.zeros((num_labels, num_classes))
+  labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1
+  return labels_one_hot
+
+
+def _extract_labels(f, one_hot=False, num_classes=10):
+  """Extract the labels into a 1D uint8 numpy array [index].
+
+  Args:
+    f: A file object that can be passed into a gzip reader.
+    one_hot: Does one hot encoding for the result.
+    num_classes: Number of classes for the one hot encoding.
+
+  Returns:
+    labels: a 1D unit8 numpy array.
+
+  Raises:
+    ValueError: If the bystream doesn't start with 2049.
+  """
+  print('Extracting', f.name)
+  with gzip.GzipFile(fileobj=f) as bytestream:
+    magic = _read32(bytestream)
+    if magic != 2049:
+      raise ValueError('Invalid magic number %d in MNIST label file: %s' %
+                       (magic, f.name))
+    num_items = _read32(bytestream)
+    buf = bytestream.read(num_items)
+    labels = numpy.frombuffer(buf, dtype=numpy.uint8)
+    
+    if one_hot:
+        labels = _dense_to_one_hot(labels, num_classes)
+        
+    return labels
+
+
+
+def read_data_sets():
+  
+    TRAIN_IMAGES = 'data/train-images-idx3-ubyte.gz'
+    TRAIN_LABELS = 'data/train-labels-idx1-ubyte.gz'
+    TEST_IMAGES = 'data/t10k-images-idx3-ubyte.gz'
+    TEST_LABELS = 'data/t10k-labels-idx1-ubyte.gz'
+
+    local_file = TRAIN_IMAGES
+    with open(local_file, 'rb') as f:
+        train_images = _extract_images(f)
+    
+    local_file = TRAIN_LABELS
+    with open(local_file, 'rb') as f:
+        train_labels = _extract_labels(f, one_hot=True)
+    
+    local_file = TEST_IMAGES
+    with open(local_file, 'rb') as f:
+        test_images = _extract_images(f)
+        
+        
+    local_file = TEST_LABELS
+    with open(local_file, 'rb') as f:
+        test_labels = _extract_labels(f, one_hot=True)
+        
+    return  train_images, test_images, train_labels, test_labels  
+    
+    
+# read_data_sets()

+ 1 - 0
TensorFlowMnist/output/.gitignore

@@ -0,0 +1 @@
+/mnist-tf1.0.1.pb

+ 50 - 0
TensorFlowMnist/test_model.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+#coding: utf-8
+
+import tensorflow as tf
+import  numpy as np
+import PIL.Image as Image
+            
+
+def recognize(png_path,pb_file_path):
+    """
+    Function:使用训练完的网络模型进行预测。
+    
+    Parameters
+    ----------
+        png_path:要预测的图片的路径。
+        pb_file_path: pb文件的路径。
+    
+    """    
+    with tf.Graph().as_default():
+        output_graph_def = tf.GraphDef()
+    
+        with open(pb_file_path, "rb") as f:
+            output_graph_def.ParseFromString(f.read()) #rb
+            _ = tf.import_graph_def(output_graph_def, name="")
+    
+        with tf.Session() as sess:
+            tf.global_variables_initializer().run()
+
+            input_x = sess.graph.get_tensor_by_name("input:0")
+            print input_x
+            out_softmax = sess.graph.get_tensor_by_name("out_softmax:0")
+            print out_softmax
+            keep_prob = sess.graph.get_tensor_by_name("keep_prob_placeholder:0")
+            print keep_prob
+            out_label = sess.graph.get_tensor_by_name("output:0")
+            print out_label
+             
+            img_datas  = np.array(Image.open(png_path).convert('L')) 
+            img_out_softmax = sess.run(out_softmax, feed_dict={
+                                   input_x: img_datas,
+                                   keep_prob: 1.0,
+                               })
+             
+            print "img_out_softmax:",img_out_softmax
+            prediction_labels = np.argmax(img_out_softmax, axis=1)
+            print "label:",prediction_labels
+            
+          
+#recognize("/home/tsiangleo/mnist_test_set/1.png","output/mnist-tf1.0.1.pb")
+

+ 219 - 0
TensorFlowMnist/train_model.py

@@ -0,0 +1,219 @@
+#!/usr/bin/env python
+#coding: utf-8
+
+import tensorflow as tf
+import input_data
+from tensorflow.python.framework import graph_util
+
+
+def build_network(height,width):
+    """
+    Function:构建网络模型。
+    
+    Parameters
+    ----------
+        height: Mnist图像的宽。
+        width: Mnist图像的宽。
+    
+    """
+    
+    x = tf.placeholder(tf.float32, [None, height, width], name='input')
+    
+    y_placeholder = tf.placeholder(tf.float32, shape=[None, 10],name='labels_placeholder')
+    
+    keep_prob_placeholder = tf.placeholder(tf.float32, name='keep_prob_placeholder')
+
+    def weight_variable(shape):
+        initial = tf.truncated_normal(shape, stddev=0.1)
+        return tf.Variable(initial)
+
+    def bias_variable(shape):
+        initial = tf.constant(0.1, shape=shape)
+        return tf.Variable(initial)
+  
+    def conv2d(x, W):
+        return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
+
+    def max_pool_2x2(x):
+        return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
+  
+    x_image = tf.reshape(x, [-1,height, width,1])
+  
+    # First Convolutional Layer
+    W_conv1 = weight_variable([5, 5, 1, 32])
+    b_conv1 = bias_variable([32])
+    
+    h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
+    h_pool1 = max_pool_2x2(h_conv1)
+    
+    # Second Convolutional Layer
+    W_conv2 = weight_variable([5, 5, 32, 64])
+    b_conv2 = bias_variable([64])
+    
+    h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
+    h_pool2 = max_pool_2x2(h_conv2)
+
+
+    # Densely Connected Layer
+    W_fc1 = weight_variable([7 * 7 * 64, 1024])
+    b_fc1 = bias_variable([1024])
+    
+    h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])
+    h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
+
+    # Dropout
+    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob_placeholder)
+
+    # Readout Layer
+    W_fc2 = weight_variable([1024, 10])
+    b_fc2 = bias_variable([10])
+    
+    logits = tf.matmul(h_fc1_drop, W_fc2) + b_fc2
+    
+    sofmax_out = tf.nn.softmax(logits,name="out_softmax")
+   
+    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits,labels=y_placeholder))
+    
+    optimize = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(cost)
+    
+    prediction_labels = tf.argmax(sofmax_out, axis=1,name="output")
+    real_labels= tf.argmax(y_placeholder, axis=1)
+    
+    correct_prediction = tf.equal(prediction_labels, real_labels)
+    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
+    #一个Batch中预测正确的次数
+    correct_times_in_batch = tf.reduce_sum(tf.cast(correct_prediction, tf.int32))
+    
+    return dict(
+                keep_prob_placeholder = keep_prob_placeholder,
+                x_placeholder= x,
+                y_placeholder = y_placeholder,
+                optimize = optimize,
+                logits = logits,
+                prediction_labels = prediction_labels,
+                real_labels = real_labels,
+                correct_prediction = correct_prediction,
+                correct_times_in_batch = correct_times_in_batch,
+                cost = cost,
+                accuracy = accuracy,
+    )
+
+def train_network(graph,
+                 dataset,
+                 batch_size,
+                 num_epochs,
+                 pb_file_path,):
+    """
+    Function:训练网络。
+    
+    Parameters
+    ----------
+        graph: 一个dict,build_network函数的返回值。
+        dataset: 数据集
+        batch_size: 
+        num_epochs: 训练轮数。
+        pb_file_path:要生成的pb文件的存放路径。
+    """
+    
+    with tf.Session() as sess:
+    
+        sess.run(tf.global_variables_initializer())
+         
+        print("batch size:",batch_size)
+         
+        #用于控制每epoch_delta轮在train set和test set上计算一下accuracy和cost
+        epoch_delta = 2
+        for epoch_index in range(num_epochs):
+                    
+            #################################
+            #    获取TRAIN set,开始训练网络
+            #################################
+            for (batch_xs,batch_ys) in dataset.train.mini_batches(batch_size):
+                sess.run([graph['optimize']], feed_dict={
+                    graph['x_placeholder']: batch_xs,
+                    graph['y_placeholder']: batch_ys,
+                    graph['keep_prob_placeholder']:0.5,
+                })
+      
+            
+            #每epoch_delta轮在train set和test set上计算一下accuracy和cost
+            if epoch_index % epoch_delta  == 0:
+                #################################
+                #    开始在 train set上计算一下accuracy和cost
+                #################################
+                #记录训练集中有多少个batch
+                total_batches_in_train_set = 0
+                #记录在训练集中预测正确的次数
+                total_correct_times_in_train_set = 0
+                #记录在训练集中的总cost
+                total_cost_in_train_set = 0.
+                for (train_batch_xs,train_batch_ys) in dataset.train.mini_batches(batch_size):
+                    return_correct_times_in_batch = sess.run(graph['correct_times_in_batch'], feed_dict={
+                        graph['x_placeholder']: train_batch_xs,
+                        graph['y_placeholder']: train_batch_ys,
+                        graph['keep_prob_placeholder']:1.0,
+                    })
+                    mean_cost_in_batch = sess.run(graph['cost'], feed_dict={
+                        graph['x_placeholder']: train_batch_xs,
+                        graph['y_placeholder']: train_batch_ys,
+                        graph['keep_prob_placeholder']:1.0,
+                    })
+                      
+                    total_batches_in_train_set += 1
+                    total_correct_times_in_train_set += return_correct_times_in_batch
+                    total_cost_in_train_set  += (mean_cost_in_batch*batch_size)
+                      
+            
+                #################################
+                # 开始在 test set上计算一下accuracy和cost
+                #################################
+                #记录测试集中有多少个batch
+                total_batches_in_test_set = 0
+                #记录在测试集中预测正确的次数
+                total_correct_times_in_test_set = 0
+                #记录在测试集中的总cost
+                total_cost_in_test_set = 0.
+                for (test_batch_xs,test_batch_ys) in dataset.test.mini_batches(batch_size):
+                    return_correct_times_in_batch = sess.run(graph['correct_times_in_batch'], feed_dict={
+                        graph['x_placeholder']: test_batch_xs,
+                        graph['y_placeholder']: test_batch_ys,
+                        graph['keep_prob_placeholder']:1.0,
+                    })
+                    mean_cost_in_batch = sess.run(graph['cost'], feed_dict={
+                        graph['x_placeholder']: test_batch_xs,
+                        graph['y_placeholder']: test_batch_ys,
+                        graph['keep_prob_placeholder']:1.0,
+                    })
+                      
+                    total_batches_in_test_set += 1
+                    total_correct_times_in_test_set += return_correct_times_in_batch
+                    total_cost_in_test_set  += (mean_cost_in_batch*batch_size)
+                 
+                ### summary and print
+                acy_on_test = total_correct_times_in_test_set / float(total_batches_in_test_set * batch_size)
+                acy_on_train = total_correct_times_in_train_set / float(total_batches_in_train_set * batch_size)
+                print('Epoch - {:2d} , acy_on_test:{:6.2f}%({}/{}),loss_on_test:{:6.2f}, acy_on_train:{:6.2f}%({}/{}),loss_on_train:{:6.2f}'.
+                      format(epoch_index, acy_on_test*100.0,total_correct_times_in_test_set,
+                             total_batches_in_test_set * batch_size,total_cost_in_test_set, acy_on_train*100.0,
+                             total_correct_times_in_train_set,total_batches_in_train_set * batch_size,total_cost_in_train_set))    
+            
+            # 每轮训练完后就保存为pb文件
+            constant_graph = graph_util.convert_variables_to_constants(sess, sess.graph_def, ["output"]) #out_softmax
+            with tf.gfile.FastGFile(pb_file_path,mode='wb') as f:
+                f.write(constant_graph.SerializeToString())
+                
+                
+
+def main():
+    
+    batch_size = 20
+    num_epochs = 2
+    
+    #pb文件保存路径
+    pb_file_path = "output/mnist-tf1.0.1.pb"
+
+    g = build_network(height=28, width=28)
+    dataset = input_data.read_data_sets()
+    train_network(g, dataset, batch_size, num_epochs, pb_file_path)
+    
+main()